From 4012da0b75219a4db55498536ec398947843f273 Mon Sep 17 00:00:00 2001 From: Privatae Date: Wed, 28 Oct 2026 02:24:07 +0000 Subject: [PATCH] bastion 0.56.0 --- .gitea/workflows/ci.yml | 6 + .github/workflows/ci.yml | 7 + CHANGELOG.md | 51 ++ README.md | 8 +- client-go/agents.go | 70 +- client-go/coder_egress.go | 95 ++- client-go/coders_linux_test.go | 10 +- client-go/contextsock.go | 92 ++- client-go/contextsock_bounds_test.go | 130 +++ client-go/contextsock_peruser_test.go | 86 ++ client-go/dcn.go | 75 +- client-go/dcn_plan.go | 78 +- client-go/dcn_rootpin_test.go | 322 ++++++++ client-go/dcn_store.go | 199 +++++ client-go/dcn_store_darwin.go | 21 + client-go/dcn_store_linux.go | 48 ++ client-go/dcn_store_other.go | 14 + client-go/egress.go | 125 +-- client-go/egress_http.go | 420 ++++++++++ client-go/egress_local_test.go | 33 +- client-go/egress_network_test.go | 76 ++ client-go/egress_one_request_test.go | 393 +++++++++ client-go/egress_relay.go | 63 +- client-go/egress_relay_test.go | 166 ++++ client-go/harness.go | 63 +- client-go/hostops_test.go | 8 +- client-go/inferencefilter.go | 33 +- client-go/lane_bounds.go | 122 +++ client-go/lane_bounds_helpers_test.go | 126 +++ client-go/lane_bounds_test.go | 481 +++++++++++ client-go/lane_frames_test.go | 197 +++++ client-go/local_host_linux_test.go | 5 +- client-go/local_host_tcp_linux_test.go | 3 +- client-go/mirror.go | 31 +- client-go/modelscope_test.go | 44 +- client-go/modelswap.go | 104 ++- client-go/ownaddr_inside_linux_test.go | 17 +- client-go/ownaddr_test.go | 6 + client-go/pii.go | 6 +- client-go/rootexec.go | 7 +- client-go/run.go | 43 +- client-go/servelane.go | 721 ++++++++++++++++ client-go/servelane_dial_linux_test.go | 131 +++ client-go/servelane_linux_test.go | 992 +++++++++++++++++++++++ client-go/servelane_models_linux_test.go | 240 ++++++ client-go/servelane_netns_linux_test.go | 274 +++++++ client-go/session.go | 43 +- deploy/VERSION | 2 +- deploy/fuzz.sh | 90 +- deploy/test_fuzz.sh | 220 +++++ docs/claims.md | 27 +- docs/host-on-this-machine.md | 32 +- docs/security-model.md | 44 +- 53 files changed, 6430 insertions(+), 270 deletions(-) create mode 100644 client-go/contextsock_bounds_test.go create mode 100644 client-go/contextsock_peruser_test.go create mode 100644 client-go/dcn_rootpin_test.go create mode 100644 client-go/dcn_store.go create mode 100644 client-go/dcn_store_darwin.go create mode 100644 client-go/dcn_store_linux.go create mode 100644 client-go/dcn_store_other.go create mode 100644 client-go/egress_http.go create mode 100644 client-go/egress_network_test.go create mode 100644 client-go/egress_one_request_test.go create mode 100644 client-go/egress_relay_test.go create mode 100644 client-go/lane_bounds.go create mode 100644 client-go/lane_bounds_helpers_test.go create mode 100644 client-go/lane_bounds_test.go create mode 100644 client-go/lane_frames_test.go create mode 100644 client-go/servelane.go create mode 100644 client-go/servelane_dial_linux_test.go create mode 100644 client-go/servelane_linux_test.go create mode 100644 client-go/servelane_models_linux_test.go create mode 100644 client-go/servelane_netns_linux_test.go create mode 100755 deploy/test_fuzz.sh diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 19f83a8..716295a 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -131,8 +131,14 @@ jobs: - name: every fuzz target, 5 seconds each # A short pass on every commit. Run deploy/fuzz.sh with its default of # 20 seconds before changing a wall a host's input goes through. + # fuzz.sh does not fail on Go's own -fuzztime report, on the Go + # releases whose fuzzer reports its deadline as a failure, and fails on + # anything else; test_fuzz.sh checks both, with planted failing + # inputs, before the targets run, and fails once the image's Go no + # longer needs the workaround. run: | docker run --rm --cpus 6 -e GOMAXPROCS=6 -v "$PWD":/src -w /src \ -e GOTOOLCHAIN=local -e CGO_ENABLED=0 "$GO_IMAGE" sh -euc ' apk add -q --no-cache bash + bash deploy/test_fuzz.sh deploy/fuzz.sh 5' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a8022da..e5faf36 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -166,6 +166,13 @@ jobs: git fetch -q --depth=1 "$REPO_URL" "$WANT" git checkout -q --detach FETCH_HEAD test "$(git rev-parse HEAD)" = "$WANT" + - name: the fuzz runner fails on a failing input and on nothing else + # fuzz.sh does not fail on Go's own -fuzztime report, on the Go + # releases whose fuzzer reports its deadline as a failure, and fails on + # anything else; this checks both, with planted failing inputs, before + # the targets run, and fails once the image's Go no longer needs the + # workaround. + run: bash deploy/test_fuzz.sh - name: every fuzz target, 5 seconds each # A short pass on every commit. Run deploy/fuzz.sh with its default of # 20 seconds before changing a wall a host's input goes through. diff --git a/CHANGELOG.md b/CHANGELOG.md index 5432546..7137818 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,57 @@ From 0.39.1 on, a heading marks a version *not published* or *held* once that is settled. The newest entries may not be published yet; the manifest says which version is. Older entries are not marked. +## 0.56.0 + +- **New: a Host with no platform can use the model servers on this + computer.** A Host running on this computer, or in a virtual machine on it, + can reach the model servers here (Ollama, a local language model server, a + personal-data filter) by name through the bastion, and is told which of + them it can use. It starts working once your Host supports it. +- **Changed: for a Host with no platform, model management reaches only this + computer.** Listing, downloading and removing models for such a Host now + reach the model server set here only when that server is on this computer. + One set to another computer is refused, and the bastion's log says which + setting is not used and what to set. Such a Host is also told only of + the model servers it can use, and no longer of the service slots, or of + coder runs when it is in a virtual machine, since neither runs for it. +- **Changed: on a local machine, the internet lane reaches only public + addresses.** A Host can no longer use a local machine's internet lane (the + egress lane) to reach devices on your home or office network at their + private addresses, such as 192.168.x.x or 10.x.x.x, or the private ranges + many VPNs use, as was already the case on an anchored machine. Your own + machines are reached through their approved lanes. A device that has a + public address, as many do over IPv6, is still reached at that address. +- **Changed: plain-HTTP requests through the bastion are checked one per + connection.** On the internet lane and a coder run's web proxy, a + plain-HTTP (not HTTPS) connection now carries one request, which is checked + before it is sent on; anything sent after it closes the connection. Before, + later requests on the same connection reached the destination unchecked. A + malformed or ambiguous request is refused, and every refusal now says why. +- **Safer: file access in a folder you shared stays inside it.** When a Host + lists, reads or moves files in a folder you shared with it, the bastion + stays inside that folder. If the folder, or any folder on its path, is later + swapped for a link to somewhere else, the bastion refuses until the folder + is shared again, and a file or folder inside it swapped for a link is never + followed out of it. Search, which goes through the folder's own search + agent, is not part of this change. +- **Safer: the bastion limits how much it holds of what a Host sends.** On + the internet lane, the terminal and root commands, coder runs, the memory + mirror (the copy of your Host's database kept here) and the bastion's + check-in with its Host, a message from the Host is read only up to a set + size and refused past it, or passed along or dropped as it arrives. A Host + can no longer make the bastion hold a message of any size in memory. A + coder run's web proxy now carries at most 1024 connections at once, and the + memory mirror at most 8. +- **Safer: asking to share a folder with a Host has limits.** Any user on the + computer can send that ask to the bastion, so each ask is now read only up + to a set size, and the number of asks open at once is capped, per user and + in total; past the cap, the ask is told the bastion is busy and to try + again. +- **For contributors: the fuzz check in CI no longer fails on a timing bug in + Go's own fuzzer.** A real failure still fails it. The program itself is + unchanged by this. + ## 0.55.3 - **A pairing change reaches every Host on the machine at once.** Each pairing diff --git a/README.md b/README.md index a5765dd..8ddee6d 100644 --- a/README.md +++ b/README.md @@ -248,6 +248,10 @@ Each point in full: [docs/security-model.md](docs/security-model.md); the link a connection is accepted. ([tunnel](docs/lanes.md#tunnel)) - **A Host on this machine gets the same walls**: mutual TLS against the CA pinned at pairing, and the bearer on every lane. ([page](docs/host-on-this-machine.md)) +- **A Host with no platform reaches your model servers by name, never by + address**: it names `ollama`, `llm` or `pii`, and the bastion reaches the + server set on this machine for it, through the forwards' filter, only when + that address is this machine. ([page](docs/host-on-this-machine.md#the-model-servers-on-this-machine)) - **Scoped tunnel.** The WireGuard peer's only allowed address is the paired host's single tunnel address. ([tunnel](docs/lanes.md#tunnel)) - **No MCP endpoint.** Nothing on this machine drives the host. @@ -269,8 +273,8 @@ Each point in full: [docs/security-model.md](docs/security-model.md); the link a - **Egress for someone else reaches the public internet, never this machine**: a coder run's proxy and an anchored machine's egress lane refuse internal addresses and this machine's own (a coder run reaches a LAN host - you named). A local machine's egress lane refuses this machine and keeps - your LAN. An address that reaches this machine only through a 1:1 NAT is + you named). A local machine's egress lane refuses this machine and your + LAN too; your own machines are reached through their approved lanes. An address that reaches this machine only through a 1:1 NAT is not covered. ([full text](docs/security-model.md)) - **A terminal is checked here and logged.** On a local machine it opens when the host asks; on an anchored one each needs an approval for that session; diff --git a/client-go/agents.go b/client-go/agents.go index 50e97ee..0ba68dd 100644 --- a/client-go/agents.go +++ b/client-go/agents.go @@ -512,6 +512,30 @@ func resolveLLMIPs() map[string]string { // to the anchor host, because approving is the anchor's job. "" = no pairing // identity (tests, tools): mode, anchor and unapproved attempts still travel. func buildHeartbeatBody(self, hostname, platform, version, wgPub, fingerprint, tunnelIP, endpoint string, + savePending bool, mirror *mirrorReport, enforcing map[string]any, peers []peerAnnounce) string { + return buildHeartbeatBodyOn(beatLanes{}, self, hostname, platform, version, wgPub, fingerprint, tunnelIP, endpoint, + savePending, mirror, enforcing, peers) +} + +// beatLanes is which of a pairing's lanes run, so its beat advertises those +// and no others. The zero value is a tunnel pairing: the forwards listen on +// the tunnel's stack and the service slots are reachable there. +type beatLanes struct { + // local: a Host with no platform (run.go runUnixPairing). The forwards and + // the service slots are absent; the model servers ride the serve lane + // (servelane.go), and the beat says which in lane_serves. + local bool + // noCoders: a tcp pairing, whose coder runs have no control plane to use + // (pair_tcp.go coderTransportRefusal), so it is never offered one. + noCoders bool +} + +func beatLanesFor(cfg *Config) beatLanes { + return beatLanes{local: cfg.localTransport(), noCoders: cfg.tcpTransport()} +} + +// buildHeartbeatBodyOn is buildHeartbeatBody for a pairing whose lanes are lanes. +func buildHeartbeatBodyOn(lanes beatLanes, self, hostname, platform, version, wgPub, fingerprint, tunnelIP, endpoint string, savePending bool, mirror *mirrorReport, enforcing map[string]any, peers []peerAnnounce) string { caps := []string{} // WHY SOME CAPABILITIES ARE WITHHELD HERE AND OTHERS ARE NOT — the asymmetry @@ -549,7 +573,17 @@ func buildHeartbeatBody(self, hostname, platform, version, wgPub, fingerprint, t // gets exactly what it got before: scope.filterModels is the identity then. scope := pairingModelScope(self) reachable, models := false, []string(nil) - if inference { + // A local pairing has no forward to advertise: its model servers ride the + // serve lane, and each lane below is advertised only when lane_serves + // carries its service (the target set and this machine's, the server + // answering, the family held). Its probes are of the address the lane + // dials, never the tunnel's guesses. + var served laneServeReport + if lanes.local { + served = laneServes(self) + reachable = containsStr(served.served, "ollama") + models = served.models + } else if inference { reachable, models = ollamaModels(tunnelIP) models = scope.filterModels(models) // Reachable but nothing shared: advertising `embed` would put this box @@ -572,7 +606,15 @@ func buildHeartbeatBody(self, hostname, platform, version, wgPub, fingerprint, t // box running both. llmModels := []string{} llmPort := llmForwardPort - if !inference { + if lanes.local { + // No port listens for it here: the Host reaches the server through the + // serve lane, so llm_port is the "no port" value. + llmPort = 0 + if containsStr(served.served, "llm") { + caps = append(caps, "llm-openai") + llmModels = served.llmModels + } + } else if !inference { // The host stores 0 as NULL; with llm-openai withheld and // llm_models empty nothing on the host resolves a port for this row. llmPort = 0 @@ -598,7 +640,9 @@ func buildHeartbeatBody(self, hostname, platform, version, wgPub, fingerprint, t // probe: an up forwarder with a dead upstream is a distinct state the // host wants before committing a fail-closed write path to it. piiAdvertised, piiOK := false, false - if inference && scope.allowsPII() { + if lanes.local { + piiAdvertised, piiOK = containsStr(served.served, "pii"), served.piiOK + } else if inference && scope.allowsPII() { piiAdvertised, piiOK = piiProbe() } if piiAdvertised { @@ -638,7 +682,7 @@ func buildHeartbeatBody(self, hostname, platform, version, wgPub, fingerprint, t // "build a server on this machine" on it, and an offer the box would refuse // is a control that fails after the user chose it. The slot list rides with // it, withheld (absent, not empty) from a pairing outside the family. - servicesOn := servicesCapable(self) + servicesOn := !lanes.local && servicesCapable(self) if servicesOn { caps = append(caps, serviceCapability) } @@ -646,7 +690,10 @@ func buildHeartbeatBody(self, hostname, platform, version, wgPub, fingerprint, t // only when this machine would accept a run-up from this pairing — Linux, // root, coders on, a runtime, the family, and (anchored) a standing coder // grant. Absent, never empty, otherwise. - coders := coderHeartbeat(self) + var coders map[string]any + if !lanes.noCoders { + coders = coderHeartbeat(self) + } if coders != nil { caps = append(caps, coderCapability) } @@ -774,6 +821,13 @@ func buildHeartbeatBody(self, hostname, platform, version, wgPub, fingerprint, t // absence as a client too old to say. "endpoint_save_pending": savePending, } + // What this pairing serves over the serve lane (servelane.go), on a local + // pairing only: ["ollama","llm","pii"] or fewer, [] when nothing is. Absent + // on a tunnel pairing, which has the forwards instead, and from a bastion + // that predates the lane. + if lanes.local { + body["lane_serves"] = served.served + } // The service slots this pairing may reach, with the tunnel port for each // (service.go). Absent when the capability is withheld, never empty-for-no. if servicesOn { @@ -1004,7 +1058,11 @@ func buildHeartbeatBody(self, hostname, platform, version, wgPub, fingerprint, t } } if piiAdvertised { - body["pii_port"] = piiForwardPort + // pii_port is the tunnel's port; a local pairing has none and reaches + // the sterilizer through the serve lane. + if !lanes.local { + body["pii_port"] = piiForwardPort + } body["pii_ok"] = piiOK } if faAdvertised { diff --git a/client-go/coder_egress.go b/client-go/coder_egress.go index 48798d7..de79381 100644 --- a/client-go/coder_egress.go +++ b/client-go/coder_egress.go @@ -23,6 +23,7 @@ package main import ( "bufio" "context" + "io" "net" "os" "path/filepath" @@ -150,17 +151,32 @@ func coderVettedAddrs(ctx context.Context, hostport string, allowLAN []string) ( } func coderRefuse(c net.Conn, why string) { - _, _ = c.Write([]byte("HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\nConnection: close\r\nX-Privatae-Refused: " + - strings.NewReplacer("\r", " ", "\n", " ").Replace(why) + "\r\n\r\n")) + writeEgressRefusal(c, "403 Forbidden", why, why) } // handleCoderEgressConn serves one connection from a run: HTTP CONNECT or an // absolute-form plain HTTP request, the same parsing as handleEgressConn. // Destinations are never logged; refusals are counted on the run. +// coderEgressMaxHeadBytes bounds a request's line and headers together. +const coderEgressMaxHeadBytes = 64 << 10 + +// coderEgressMaxConns bounds the connections one run's proxy carries at once. +// A whole Chromium profile opens at most 256 sockets at a time (its socket +// pool limit), and package managers far fewer, so four times the browser's +// figure never refuses real work; past it a connection is answered 503 and +// closed, and each one held costs this machine at most its head and two copy +// buffers. +const coderEgressMaxConns = 1024 + func handleCoderEgressConn(client net.Conn, allowLAN []string, refused *int64) { defer client.Close() _ = client.SetReadDeadline(time.Now().Add(30 * time.Second)) - br := bufio.NewReader(client) + // The request line and headers are read under one bound, the head limit + // below has always named: the run's own processes send them, and a line + // that never ends must not grow in this machine's memory before that + // limit is looked at. + headLim := &io.LimitedReader{R: client, N: coderEgressMaxHeadBytes} + br := bufio.NewReader(headLim) line, err := br.ReadString('\n') if err != nil { return @@ -172,7 +188,6 @@ func handleCoderEgressConn(client net.Conn, allowLAN []string, refused *int64) { method, target := parts[0], parts[1] var head strings.Builder head.WriteString(line) - hostHdr := "" for { h, err := br.ReadString('\n') if err != nil { @@ -182,38 +197,41 @@ func handleCoderEgressConn(client net.Conn, allowLAN []string, refused *int64) { if h == "\r\n" || h == "\n" { break } - if head.Len() > 64<<10 { + if head.Len() > coderEgressMaxHeadBytes { return } - if len(h) >= 5 && strings.EqualFold(h[:5], "host:") { - hostHdr = strings.TrimSpace(h[5:]) - } } _ = client.SetReadDeadline(time.Time{}) + // The head is in; what follows is the stream itself. What the reader + // already buffered past the head stays first in line. + headLim.N = 1<<63 - 1 ctx, cancel := context.WithTimeout(context.Background(), egressDialTimeout) defer cancel() + // Read the way the bastion's own egress lane reads it (egress_http.go): a + // CONNECT target through parseEgressAuthority, and plain HTTP as ONE + // request, rebuilt and sent with `Connection: close`, so the run's later + // requests on a keep-alive connection never reach the destination unvetted. connect := strings.EqualFold(method, "CONNECT") hp := "" + var req egressRequest if connect { - hp = ensurePort(target, "443") - } else { - host := hostHdr - if i := strings.Index(target, "://"); i >= 0 { - rest := target[i+3:] - if s := strings.IndexAny(rest, "/?#"); s >= 0 { - host = rest[:s] - } else { - host = rest - } - if at := strings.LastIndexByte(host, '@'); at >= 0 { - host = host[at+1:] - } - } - if host == "" { + host, port, ok := parseEgressAuthority(target, 443) + if !ok { + atomic.AddInt64(refused, 1) + writeEgressRefusal(client, "400 Bad Request", egressNoHost, egressNoHost) return } - hp = ensurePort(host, "80") + hp = egressHostPort(host, port) + } else { + r, bad := parseEgressRequest(head.String()) + if bad != nil { + atomic.AddInt64(refused, 1) + writeEgressRefusal(client, bad.status, bad.body, bad.body) + return + } + req = r + hp = egressHostPort(req.host, req.port) } dialAddrs, why := coderVettedAddrs(ctx, hp, allowLAN) if why != "" { @@ -223,15 +241,14 @@ func handleCoderEgressConn(client net.Conn, allowLAN []string, refused *int64) { } upstream, err := dialEach(ctx, coderDialOut, dialAddrs) if err != nil { - _, _ = client.Write([]byte("HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")) + writeEgressRefusal(client, "502 Bad Gateway", egressUnreachable, "") return } - if connect { - if _, err := client.Write([]byte("HTTP/1.1 200 Connection established\r\n\r\n")); err != nil { - _ = upstream.Close() - return - } - } else if _, err := upstream.Write([]byte(head.String())); err != nil { + if !connect { + oneEgressExchange(br, client, upstream, req) + return + } + if _, err := client.Write([]byte("HTTP/1.1 200 Connection established\r\n\r\n")); err != nil { _ = upstream.Close() return } @@ -277,6 +294,7 @@ func coderListenEgress(runID, dir string, uid int) error { } run := &coderEgressRun{ln: ln} coderEgress[runID] = run + slots := make(chan struct{}, coderEgressMaxConns) go func() { for { c, err := ln.Accept() @@ -285,7 +303,20 @@ func coderListenEgress(runID, dir string, uid int) error { } // allow_lan is read per connection: the owner's edit applies to // running runs at once. - go handleCoderEgressConn(c, loadCoderConfig().AllowLAN, &run.refused) + select { + case slots <- struct{}{}: + go func() { + defer func() { <-slots }() + handleCoderEgressConn(c, loadCoderConfig().AllowLAN, &run.refused) + }() + default: + go func() { + defer c.Close() + _ = c.SetWriteDeadline(time.Now().Add(time.Second)) + why := "this run already has " + strconv.Itoa(coderEgressMaxConns) + " connections open" + writeEgressRefusal(c, "503 Service Unavailable", why, why) + }() + } } }() return nil diff --git a/client-go/coders_linux_test.go b/client-go/coders_linux_test.go index 40682f1..702174b 100644 --- a/client-go/coders_linux_test.go +++ b/client-go/coders_linux_test.go @@ -474,8 +474,14 @@ func TestCoderEgressProxyOverTheSocket(t *testing.T) { } { head, c := ask(req) c.Close() - if !strings.HasPrefix(head, "HTTP/1.1 403") || !strings.Contains(head, "X-Privatae-Refused:") { - t.Fatalf("must be refused with 403 + X-Privatae-Refused: %q -> %q", req, head) + want := "HTTP/1.1 403" + if strings.Contains(req, "user@") { + // Userinfo is refused outright, before any lookup, as the Host's + // egress gate refuses it (parseEgressAuthority); still counted. + want = "HTTP/1.1 400" + } + if !strings.HasPrefix(head, want) || !strings.Contains(head, "X-Privatae-Refused:") { + t.Fatalf("must be refused with %s + X-Privatae-Refused: %q -> %q", want, req, head) } } head, c := ask("CONNECT public.example:443 HTTP/1.1\r\n\r\n") diff --git a/client-go/contextsock.go b/client-go/contextsock.go index 9eaab4c..e58fa26 100644 --- a/client-go/contextsock.go +++ b/client-go/contextsock.go @@ -43,9 +43,11 @@ import ( "encoding/json" "errors" "fmt" + "io" "net" "os" "path/filepath" + "sync" "time" ) @@ -126,18 +128,81 @@ func startContextSock(done <-chan struct{}) { _ = ln.Close() _ = os.Remove(p) }() - go func() { - for { - c, err := ln.Accept() - if err != nil { - return - } - go contextSockServe(c) - } - }() + go contextSockAccept(ln) uilogf("context socket listening on %s\n", p) } +// The socket is 0666 and the daemon runs as root (or SYSTEM), so every user on +// the machine can make it read. These bound what they can make it hold: an ask +// is a folder path and a display name, far under contextAskMaxBytes, and no +// more than contextSockMaxInFlight asks are read at once. Past that a +// connection is told the daemon is busy and closed; the asker tries again. +// +// One person may hold at most contextSockMaxPerUser of those slots, keyed on +// the identity the kernel gives (never on anything the asker says): without +// it, one user opening contextSockMaxInFlight idle connections and dialling +// again as each deadline ran out kept every other user's asks "busy" for as +// long as they liked. A person adds a folder at a time; four at once is far +// past that. +const ( + contextAskMaxBytes = 64 << 10 + contextSockMaxInFlight = 32 +) + +// contextSockMaxPerUser is a var only so the test of the overall cap can lift +// it (every test connection is the same user). +var contextSockMaxPerUser = 4 + +var contextSockUsers = struct { + sync.Mutex + held map[string]int +}{held: map[string]int{}} + +// contextSockTakeUser takes one of user's slots, or reports there is none. +func contextSockTakeUser(user string) bool { + contextSockUsers.Lock() + defer contextSockUsers.Unlock() + if contextSockUsers.held[user] >= contextSockMaxPerUser { + return false + } + contextSockUsers.held[user]++ + return true +} + +func contextSockReleaseUser(user string) { + contextSockUsers.Lock() + defer contextSockUsers.Unlock() + if contextSockUsers.held[user]--; contextSockUsers.held[user] <= 0 { + delete(contextSockUsers.held, user) + } +} + +// contextSockAccept serves ln until it is closed, at most +// contextSockMaxInFlight connections at a time. +func contextSockAccept(ln net.Listener) { + slots := make(chan struct{}, contextSockMaxInFlight) + for { + c, err := ln.Accept() + if err != nil { + return + } + select { + case slots <- struct{}{}: + go func() { + defer func() { <-slots }() + contextSockServe(c) + }() + default: + go func() { + defer c.Close() + _ = c.SetDeadline(time.Now().Add(time.Second)) + _ = json.NewEncoder(c).Encode(contextAskReply{Why: "the bastion is busy " + + "answering other asks on this machine; try again in a moment"}) + }() + } + } +} + func contextSockServe(c net.Conn) { defer c.Close() _ = c.SetDeadline(time.Now().Add(30 * time.Second)) @@ -154,8 +219,15 @@ func contextSockServe(c net.Conn) { return } + if !contextSockTakeUser(user) { + _ = json.NewEncoder(c).Encode(contextAskReply{User: user, Why: "the bastion is busy " + + "answering other asks of yours; try again in a moment"}) + return + } + defer contextSockReleaseUser(user) + var req contextAskRequest - if err := json.NewDecoder(c).Decode(&req); err != nil { + if err := json.NewDecoder(io.LimitReader(c, contextAskMaxBytes)).Decode(&req); err != nil { _ = json.NewEncoder(c).Encode(contextAskReply{User: user, Why: "the request could not be read"}) return } diff --git a/client-go/contextsock_bounds_test.go b/client-go/contextsock_bounds_test.go new file mode 100644 index 0000000..4974de8 --- /dev/null +++ b/client-go/contextsock_bounds_test.go @@ -0,0 +1,130 @@ +package main + +// The context socket is 0666: every user on the machine may connect, and the +// daemon behind it runs as root (or SYSTEM). So what one connection can make +// the daemon hold must be bounded by the daemon, not by the asker. + +import ( + "net" + "path/filepath" + "runtime" + "strings" + "testing" + "time" +) + +// An ask is a folder path and a display name. A client that streams a request +// with no end gets the daemon to read only a bounded amount of it, then a +// reply and a close; before, the decoder kept every byte until the 30 s +// deadline, and nothing capped how many such connections were open. +func TestAContextAskWithNoEndIsReadOnlyToABound(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("no unix-socket peer credentials on windows; contextsock_windows.go says why") + } + sock := filepath.Join(t.TempDir(), "s.sock") + ln, err := net.Listen("unix", sock) + if err != nil { + t.Fatal(err) + } + defer ln.Close() + go func() { + c, err := ln.Accept() + if err != nil { + return + } + contextSockServe(c) + }() + + c, err := net.Dial("unix", sock) + if err != nil { + t.Fatal(err) + } + defer c.Close() + const total = 16 << 20 + chunk := []byte(strings.Repeat("a", 64<<10)) + sent := 0 + if _, err := c.Write([]byte(`{"root":"`)); err != nil { + t.Fatal(err) + } + for sent < total { + _ = c.SetWriteDeadline(time.Now().Add(2 * time.Second)) + n, err := c.Write(chunk) + sent += n + if err != nil { + break + } + } + // The kernel's socket buffers hold a few hundred KiB the daemon never + // read; anything near the whole stream means the daemon read it all. + if sent >= 4<<20 { + t.Fatalf("the daemon accepted %d MiB of one ask with no end; it must stop at a bound", sent>>20) + } +} + +// Asks in flight are capped: a connection past the cap is closed at once, so +// many idle or endless connections cannot multiply the bound above. +func TestContextAsksInFlightAreCapped(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("no unix-socket peer credentials on windows; contextsock_windows.go says why") + } + sock := filepath.Join(t.TempDir(), "s.sock") + ln, err := net.Listen("unix", sock) + if err != nil { + t.Fatal(err) + } + defer ln.Close() + // Every connection here is the same user; lift the per-user share so the + // overall cap is what this test meets (TestOneUserCannotHoldEveryContextSockSlot + // holds the share). + oldShare := contextSockMaxPerUser + contextSockMaxPerUser = contextSockMaxInFlight + t.Cleanup(func() { contextSockMaxPerUser = oldShare }) + go contextSockAccept(ln) + + var held []net.Conn + defer func() { + for _, h := range held { + _ = h.Close() + } + }() + // Each of these sends nothing and holds its slot until its deadline. + for i := 0; i < contextSockMaxInFlight; i++ { + h, err := net.Dial("unix", sock) + if err != nil { + t.Fatal(err) + } + held = append(held, h) + } + // Give the daemon's accept loop time to take every held connection. + time.Sleep(200 * time.Millisecond) + + extra, err := net.Dial("unix", sock) + if err != nil { + t.Fatal(err) + } + defer extra.Close() + _ = extra.SetReadDeadline(time.Now().Add(2 * time.Second)) + buf := make([]byte, 512) + n, rerr := extra.Read(buf) + if ne, ok := rerr.(net.Error); ok && ne.Timeout() { + t.Fatalf("a connection past the cap of %d was left open; it must be refused", contextSockMaxInFlight) + } + if n > 0 && !strings.Contains(string(buf[:n]), "busy") { + t.Fatalf("a connection past the cap got %q; it must be told the daemon is busy", buf[:n]) + } + + // Once a held connection goes, a new ask gets a slot again. + _ = held[0].Close() + held = held[1:] + time.Sleep(200 * time.Millisecond) + again, err := net.Dial("unix", sock) + if err != nil { + t.Fatal(err) + } + defer again.Close() + _ = again.SetReadDeadline(time.Now().Add(500 * time.Millisecond)) + n, rerr = again.Read(buf) + if ne, ok := rerr.(net.Error); !ok || !ne.Timeout() { + t.Fatalf("after a slot was freed, a new ask was not served (read %q, %v)", buf[:n], rerr) + } +} diff --git a/client-go/contextsock_peruser_test.go b/client-go/contextsock_peruser_test.go new file mode 100644 index 0000000..460a7e8 --- /dev/null +++ b/client-go/contextsock_peruser_test.go @@ -0,0 +1,86 @@ +package main + +import ( + "net" + "path/filepath" + "runtime" + "strings" + "testing" + "time" +) + +// One person cannot take every slot: past their own share they are told the +// daemon is busy at once, while slots remain for everyone else. Before, one +// user's idle connections, re-dialled as each deadline ran out, held all of +// them and every other user's ask was refused. +func TestOneUserCannotHoldEveryContextSockSlot(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("no unix-socket peer credentials on windows; contextsock_windows.go says why") + } + const share = 4 // contextSockMaxPerUser + // A test before this one may still be releasing its slots. + if n := contextSockSlotsHeld(t, 0); n != 0 { + t.Fatalf("%d slots still held from an earlier test", n) + } + sock := filepath.Join(t.TempDir(), "s.sock") + ln, err := net.Listen("unix", sock) + if err != nil { + t.Fatal(err) + } + defer ln.Close() + go contextSockAccept(ln) + + var held []net.Conn + defer func() { + for _, h := range held { + _ = h.Close() + } + }() + for i := 0; i < share; i++ { + h, err := net.Dial("unix", sock) + if err != nil { + t.Fatal(err) + } + held = append(held, h) + } + // Wait until the daemon has given those four their slots (a fixed sleep + // lost this race on a loaded machine and handed the 5th one first). + if n := contextSockSlotsHeld(t, share); n != share { + t.Fatalf("the daemon gave %d slots to %d idle connections of one user", n, share) + } + + extra, err := net.Dial("unix", sock) + if err != nil { + t.Fatal(err) + } + defer extra.Close() + _ = extra.SetReadDeadline(time.Now().Add(2 * time.Second)) + buf := make([]byte, 512) + n, rerr := extra.Read(buf) + if ne, ok := rerr.(net.Error); ok && ne.Timeout() { + t.Fatalf("one user held %d idle connections and a %dth was still given a slot: "+ + "one user can take all %d", share, share+1, contextSockMaxInFlight) + } + if !strings.Contains(string(buf[:n]), "busy") { + t.Fatalf("past one user's share the ask got %q; it must be told the daemon is busy", buf[:n]) + } +} + +// contextSockSlotsHeld waits up to 10 s for the per-user slots held across +// all users to reach want, and returns the count it last saw. +func contextSockSlotsHeld(t *testing.T, want int) int { + t.Helper() + n := -1 + for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(10 * time.Millisecond) { + contextSockUsers.Lock() + n = 0 + for _, k := range contextSockUsers.held { + n += k + } + contextSockUsers.Unlock() + if n == want { + break + } + } + return n +} diff --git a/client-go/dcn.go b/client-go/dcn.go index 7276b23..049abd4 100644 --- a/client-go/dcn.go +++ b/client-go/dcn.go @@ -26,6 +26,7 @@ import ( "encoding/base64" "encoding/json" "fmt" + "io/fs" "os" "path/filepath" "runtime" @@ -74,6 +75,10 @@ type dcnGrantRec struct { // the signed bytes (a public key; the terms stay exactly what was // signed). The managed local agent seals the store's catalog to it. SealRecipient string `json:"seal_recipient,omitempty"` + // RootReal is where the granted folder led when this box installed the + // grant: its real path, pinned here (dcn_store.go), outside the signed + // bytes. An op whose root leads anywhere else is refused. + RootReal string `json:"root_real,omitempty"` } // dcnTerms is parsed FROM the verified canonical bytes — never from anything else. @@ -524,6 +529,17 @@ func handleDcnGrantSync(ws *websocket.Conn, dir string, args []string) { dcnRefuse(ws, dir, "grant_sync", payload.StoreID, "", "grant does not verify: "+err.Error()) return } + // The folder this grant names is pinned as it resolves now; a re-sync or a + // renewal of the same root keeps the pin it already has (dcn_store.go). + if prev, held := f.Stores[fmt.Sprint(payload.StoreID)]; held { + prevRoot := "" + if pt, perr := dcnVerifyRec(f.VerifyPubkey, prev); perr == nil { + prevRoot = pt.Root + } + rec.RootReal = dcnPinFor(&prev, prevRoot, terms.Root) + } else { + rec.RootReal = dcnPinFor(nil, "", terms.Root) + } if hn, _ := os.Hostname(); hn != "" && terms.Server != "" && terms.Server != hn { dcnRefuse(ws, dir, "grant_sync", payload.StoreID, "", fmt.Sprintf("grant is for server '%s', this box is '%s'", terms.Server, hn)) @@ -658,15 +674,26 @@ func handleDcnList(ws *websocket.Conn, dir string, args []string) { if terms == nil { return } - base := terms.Root + base := "." if a.Prefix != "" { p, ok := dcnResolvePath(terms.Root, a.Prefix) if !ok { dcnRefuse(ws, dir, "list", storeID, a.Prefix, "prefix escapes the store root") return } - base = p + rel, err := dcnRel(terms.Root, p) + if err != nil { + dcnRefuse(ws, dir, "list", storeID, a.Prefix, "prefix escapes the store root") + return + } + base = filepath.ToSlash(rel) } + r, err := dcnOpenStore(dir, storeID, terms) + if err != nil { + dcnRefuse(ws, dir, "list", storeID, a.Prefix, err.Error()) + return + } + defer r.Close() limit := a.Limit if limit <= 0 || limit > dcnMaxListEntries { limit = dcnMaxListEntries @@ -678,12 +705,14 @@ func handleDcnList(ws *websocket.Conn, dir string, args []string) { } var files []entry truncated := false - _ = filepath.WalkDir(base, func(p string, d os.DirEntry, err error) error { + // Walked through the store's handle: a link that leads out of it is not + // followed, whatever the names do during the walk. + _ = fs.WalkDir(r.FS(), base, func(p string, d fs.DirEntry, err error) error { if err != nil { return nil // unreadable subtree: skip, never abort the listing } - rel, rerr := filepath.Rel(terms.Root, p) - if rerr != nil || rel == "." { + rel := filepath.FromSlash(p) + if rel == "." { return nil } if dcnExcluded(terms.Excludes, rel) { @@ -727,8 +756,23 @@ func handleDcnStat(ws *websocket.Conn, dir string, args []string) { dcnRefuse(ws, dir, "stat", storeID, a.Path, "path escapes the store root") return } - info, err := os.Stat(abs) + rel, err := dcnRel(terms.Root, abs) if err != nil { + dcnRefuse(ws, dir, "stat", storeID, a.Path, err.Error()) + return + } + r, err := dcnOpenStore(dir, storeID, terms) + if err != nil { + dcnRefuse(ws, dir, "stat", storeID, a.Path, err.Error()) + return + } + defer r.Close() + info, err := r.Stat(rel) + if err != nil { + if dcnEscaped(err) { + dcnRefuse(ws, dir, "stat", storeID, a.Path, "path escapes the store root") + return + } if os.IsNotExist(err) { dcnResult(ws, dir, "stat", storeID, a.Path, map[string]any{ "state": "gone", "path": a.Path}) @@ -771,8 +815,25 @@ func handleDcnReadRange(ws *websocket.Conn, dir string, args []string) { dcnRefuse(ws, dir, "read_range", storeID, a.Path, "path escapes the store root") return } - fh, err := os.Open(abs) + rel, err := dcnRel(terms.Root, abs) if err != nil { + dcnRefuse(ws, dir, "read_range", storeID, a.Path, err.Error()) + return + } + r, err := dcnOpenStore(dir, storeID, terms) + if err != nil { + dcnRefuse(ws, dir, "read_range", storeID, a.Path, err.Error()) + return + } + defer r.Close() + // Opened through the store's handle: a name swapped for a link to a file + // outside the store after the check above is refused, never followed. + fh, err := r.Open(rel) + if err != nil { + if dcnEscaped(err) { + dcnRefuse(ws, dir, "read_range", storeID, a.Path, "path escapes the store root") + return + } if os.IsNotExist(err) { dcnResult(ws, dir, "read_range", storeID, a.Path, map[string]any{ "state": "gone", "path": a.Path}) diff --git a/client-go/dcn_plan.go b/client-go/dcn_plan.go index 5c4a6b2..aedf37f 100644 --- a/client-go/dcn_plan.go +++ b/client-go/dcn_plan.go @@ -191,10 +191,28 @@ func dcnReportPath(root string, planID int64) string { return p } -// dcnApplyPlan runs the plan: check, rename in order (each re-checked just -// before it runs; rolling back on the first failure), verify, report. Every -// path it touches has passed dcnCheckPlan. +// dcnApplyPlan is dcnApplyPlanIn on root opened where it leads now (the op +// handler opens the PINNED folder instead, dcn_store.go). func dcnApplyPlan(root string, excludes []string, plan *dcnPlanTerms) map[string]any { + real, err := dcnRealRoot(root) + if err != nil { + return map[string]any{"state": "refused", "mode": "apply", "plan_id": plan.PlanID, + "reason": "the store's folder cannot be opened: " + err.Error()} + } + r, err := dcnOpenRootAt(real) + if err != nil { + return map[string]any{"state": "refused", "mode": "apply", "plan_id": plan.PlanID, "reason": err.Error()} + } + defer r.Close() + return dcnApplyPlanIn(r, root, excludes, plan) +} + +// dcnApplyPlanIn runs the plan: check, rename in order (each re-checked just +// before it runs; rolling back on the first failure), verify, report. Every +// path it touches has passed dcnCheckPlan, and every change is made through +// r, the store's handle, so a folder that becomes a link to somewhere else +// between the check and the rename is refused rather than followed. +func dcnApplyPlanIn(r *os.Root, root string, excludes []string, plan *dcnPlanTerms) map[string]any { problems := dcnCheckPlan(root, excludes, plan.Moves) if len(problems) > 0 { return map[string]any{"state": "refused", "mode": "apply", "plan_id": plan.PlanID, @@ -207,11 +225,11 @@ func dcnApplyPlan(root string, excludes []string, plan *dcnPlanTerms) map[string // and what is NOT (so the user knows exactly what is still displaced). rollback := func() (back, stuck []string) { for i := len(applied) - 1; i >= 0; i-- { - if _, err := os.Lstat(applied[i].src); err == nil { + if _, err := r.Lstat(applied[i].src); err == nil { stuck = append(stuck, applied[i].from) // something now sits at the source; never replace it continue } - if err := os.Rename(applied[i].dst, applied[i].src); err == nil { + if err := r.Rename(applied[i].dst, applied[i].src); err == nil { back = append(back, applied[i].from) } else { stuck = append(stuck, applied[i].from) @@ -230,10 +248,17 @@ func dcnApplyPlan(root string, excludes []string, plan *dcnPlanTerms) map[string return rep } for i, m := range plan.Moves { - src, _ := dcnResolvePath(root, m.From) - dst, _ := cleanRel(root, m.To) - if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { - return fail(i, m, "could not create the target folder: "+err.Error()) + srcAbs, _ := dcnResolvePath(root, m.From) + dstAbs, _ := cleanRel(root, m.To) + src, serr := dcnRel(root, srcAbs) + dst, derr := dcnRel(root, dstAbs) + if serr != nil || derr != nil { + return fail(i, m, "path escapes the store root") + } + if d := filepath.Dir(dst); d != "." { + if err := r.MkdirAll(d, 0o755); err != nil { + return fail(i, m, "could not create the target folder: "+err.Error()) + } } // Re-check the instant before the rename: the world may have changed since // the pre-check (a file appeared at the target; a parent became a symlink). @@ -241,13 +266,12 @@ func dcnApplyPlan(root string, excludes []string, plan *dcnPlanTerms) map[string if _, ok := dcnResolvePath(root, m.To); !ok { return fail(i, m, "target's parent no longer resolves inside the store root") } - if _, err := os.Lstat(dst); err == nil { + if _, err := r.Lstat(dst); err == nil { return fail(i, m, "target appeared since the check (a plan never overwrites)") } - if err := os.Rename(src, dst); err != nil { + if err := r.Rename(src, dst); err != nil { why := err.Error() - var le *os.LinkError - if errors.As(err, &le) && errors.Is(le.Err, syscall.EXDEV) { + if errors.Is(err, syscall.EXDEV) { why = "target is on a different filesystem than the source (a plan only renames; it never copies and removes)" } return fail(i, m, why) @@ -257,14 +281,16 @@ func dcnApplyPlan(root string, excludes []string, plan *dcnPlanTerms) map[string // Verify pass: every target present, every source gone, declared sizes hold. var mismatches []dcnPlanProblem for i, m := range plan.Moves { - src, _ := dcnResolvePath(root, m.From) - dst, _ := cleanRel(root, m.To) - info, err := os.Lstat(dst) + srcAbs, _ := dcnResolvePath(root, m.From) + dstAbs, _ := cleanRel(root, m.To) + src, _ := dcnRel(root, srcAbs) + dst, _ := dcnRel(root, dstAbs) + info, err := r.Lstat(dst) if err != nil { mismatches = append(mismatches, dcnPlanProblem{i, m.From, m.To, "target missing after move"}) continue } - if _, err := os.Lstat(src); err == nil { + if _, err := r.Lstat(src); err == nil { mismatches = append(mismatches, dcnPlanProblem{i, m.From, m.To, "source still present after move"}) } if m.SizeBytes != nil && !info.IsDir() && info.Size() != *m.SizeBytes { @@ -372,9 +398,19 @@ func handleDcnPlanApply(ws *websocket.Conn, dir string, args []string) { // a retry (a lost reply, a host re-run) gets the earlier report back BEFORE // any gate can turn it into a refusal. (The signed expiry is the guard that // does not depend on a file the user can remove.) + store, err := dcnOpenStore(dir, storeID, terms) + if err != nil { + dcnRefuse(ws, dir, op, storeID, planKey, err.Error()) + return + } + defer store.Close() repPath := dcnReportPath(terms.Root, plan.PlanID) + repRel := "" if repPath != "" { - if prev, err := os.ReadFile(repPath); err == nil { + repRel, _ = dcnRel(terms.Root, repPath) + } + if repRel != "" { + if prev, err := store.ReadFile(repRel); err == nil { var old map[string]any if json.Unmarshal(prev, &old) == nil && old["state"] == "ok" { old["note"] = "already applied on this machine; this is the earlier report" @@ -398,13 +434,13 @@ func handleDcnPlanApply(ws *websocket.Conn, dir string, args []string) { return } } - rep := dcnApplyPlan(terms.Root, terms.Excludes, plan) + rep := dcnApplyPlanIn(store, terms.Root, terms.Excludes, plan) if rep["state"] == "ok" && plan.Approval != "" { markApprovalUsed(dir, plan.Approval) // single use: the files have moved } - if repPath != "" { + if repRel != "" { if raw, err := json.MarshalIndent(rep, "", " "); err == nil { - _ = atomicWrite(repPath, raw) // the machine's own record; best-effort + _ = dcnRootAtomicWrite(store, repRel, raw) // the machine's own record; best-effort } } dcnResult(ws, dir, op, storeID, planKey, rep) diff --git a/client-go/dcn_rootpin_test.go b/client-go/dcn_rootpin_test.go new file mode 100644 index 0000000..46ebf09 --- /dev/null +++ b/client-go/dcn_rootpin_test.go @@ -0,0 +1,322 @@ +package main + +// A machine store is the folder the person granted, for as long as the grant +// lives. The bastion often runs as root and the folder belongs to someone who +// is not, so whoever can write in it, or above it, must not be able to point +// a read or a move somewhere else: by turning the granted folder into a link +// after the grant (no race needed), or by swapping a file for a link between +// the check and the open. + +import ( + "crypto/ed25519" + "encoding/base64" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/gorilla/websocket" +) + +// dcnRunOp runs one DCN handler on the far side of a real WebSocket and +// returns what it sent: the O payload, the R text and the exit code. +func dcnRunOp(t *testing.T, h func(ws *websocket.Conn)) (out, errText string, exit int) { + t.Helper() + up := websocket.Upgrader{} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ws, err := up.Upgrade(w, r, nil) + if err != nil { + return + } + defer ws.Close() + h(ws) + })) + defer srv.Close() + ws, _, err := websocket.DefaultDialer.Dial("ws"+strings.TrimPrefix(srv.URL, "http"), nil) + if err != nil { + t.Fatal(err) + } + defer ws.Close() + _ = ws.SetReadDeadline(time.Now().Add(10 * time.Second)) + exit = -1 + for { + _, data, err := ws.ReadMessage() + if err != nil || len(data) == 0 { + return + } + switch data[0] { + case 'O': + out += string(data[1:]) + case 'R': + errText += string(data[1:]) + case 'X': + exit, _ = strconv.Atoi(string(data[1:])) + return + } + } +} + +type dcnStoreFixture struct { + dir string // the pairing's dcn dir + base string + store string // the granted folder + other string // a folder the grant never named + priv ed25519.PrivateKey + pub ed25519.PublicKey +} + +func newDcnStoreFixture(t *testing.T) *dcnStoreFixture { + t.Helper() + fx := &dcnStoreFixture{dir: t.TempDir(), base: t.TempDir()} + fx.store = filepath.Join(fx.base, "store") + fx.other = filepath.Join(fx.base, "other") + for _, d := range []string{fx.store, fx.other, filepath.Join(fx.store, "docs")} { + if err := os.MkdirAll(d, 0o755); err != nil { + t.Fatal(err) + } + } + if err := os.WriteFile(filepath.Join(fx.store, "a.txt"), []byte("MINE"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(fx.other, "a.txt"), []byte("OTHER"), 0o600); err != nil { + t.Fatal(err) + } + fx.pub, fx.priv, _ = ed25519.GenerateKey(nil) + return fx +} + +// sync installs a read grant for the store through the real grant_sync op. +func (fx *dcnStoreFixture) sync(t *testing.T, mask ...string) { + t.Helper() + if len(mask) == 0 { + mask = []string{"read"} + } + rec := signedRec(t, fx.priv, map[string]any{"kind": "dcn_store_grant", "class": "machine", + "root": fx.store, "mask": mask, + "expires": time.Now().Add(24 * time.Hour).UTC().Format(time.RFC3339)}) + payload, _ := json.Marshal(map[string]any{"store_id": 1, "terms_canonical": rec.TermsCanonical, + "signature": rec.Signature, "verify_pubkey": base64.StdEncoding.EncodeToString(fx.pub)}) + _, why, exit := dcnRunOp(t, func(ws *websocket.Conn) { + handleDcnGrantSync(ws, fx.dir, []string{base64.StdEncoding.EncodeToString(payload)}) + }) + if exit != 0 { + t.Fatalf("grant_sync: exit %d: %s", exit, why) + } +} + +func (fx *dcnStoreFixture) op(t *testing.T, h func(*websocket.Conn, string, []string), args map[string]any) (map[string]any, string, int) { + t.Helper() + raw, _ := json.Marshal(args) + out, why, exit := dcnRunOp(t, func(ws *websocket.Conn) { + h(ws, fx.dir, []string{"1", string(raw)}) + }) + var obj map[string]any + _ = json.Unmarshal([]byte(out), &obj) + return obj, why, exit +} + +func (fx *dcnStoreFixture) read(t *testing.T, path string) (string, string, int) { + t.Helper() + obj, why, exit := fx.op(t, handleDcnReadRange, map[string]any{"path": path}) + b, _ := base64.StdEncoding.DecodeString(func() string { s, _ := obj["content_b64"].(string); return s }()) + return string(b), why, exit +} + +// repoint turns the granted folder into a link to the other one. +func (fx *dcnStoreFixture) repoint(t *testing.T) { + t.Helper() + if err := os.Rename(fx.store, fx.store+".was"); err != nil { + t.Fatal(err) + } + if err := os.Symlink(fx.other, fx.store); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } +} + +func TestDcnStoreRepointedAfterTheGrantIsRefused(t *testing.T) { + fx := newDcnStoreFixture(t) + fx.sync(t) + if got, why, exit := fx.read(t, "a.txt"); exit != 0 || got != "MINE" { + t.Fatalf("the granted file must read: exit %d %q %s", exit, got, why) + } + fx.repoint(t) + if got, _, exit := fx.read(t, "a.txt"); exit == 0 || got == "OTHER" { + t.Fatalf("after the granted folder became a link elsewhere, read_range returned %q (exit %d); it must refuse", got, exit) + } + if obj, _, exit := fx.op(t, handleDcnStat, map[string]any{"path": "a.txt"}); exit == 0 { + t.Fatalf("stat through the repointed folder must refuse, got %v", obj) + } + if obj, _, exit := fx.op(t, handleDcnList, map[string]any{}); exit == 0 { + t.Fatalf("list through the repointed folder must refuse, got %v", obj) + } +} + +// A folder inside the store that becomes a link to somewhere else after the +// grant is the same escape one level down. +func TestDcnSubfolderRepointedIsRefused(t *testing.T) { + fx := newDcnStoreFixture(t) + fx.sync(t) + if err := os.RemoveAll(filepath.Join(fx.store, "docs")); err != nil { + t.Fatal(err) + } + if err := os.Symlink(fx.other, filepath.Join(fx.store, "docs")); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + if got, _, exit := fx.read(t, "docs/a.txt"); exit == 0 || got == "OTHER" { + t.Fatalf("a link inside the store to a folder outside it read %q (exit %d)", got, exit) + } +} + +// The race: a file swapped for a link to a file outside the store between the +// check and the open. Run for long enough that the old check-then-open loses. +func TestDcnReadRangeSwapRaceNeverLeavesTheStore(t *testing.T) { + fx := newDcnStoreFixture(t) + fx.sync(t) + target := filepath.Join(fx.store, "a.txt") + realTmp := filepath.Join(fx.store, ".real") + linkTmp := filepath.Join(fx.store, ".link") + var stop atomic.Bool + done := make(chan struct{}) + go func() { + defer close(done) + for !stop.Load() { + _ = os.Symlink(filepath.Join(fx.other, "a.txt"), linkTmp) + _ = os.Rename(linkTmp, target) + _ = os.WriteFile(realTmp, []byte("MINE"), 0o644) + _ = os.Rename(realTmp, target) + } + }() + deadline := time.Now().Add(3 * time.Second) + n := 0 + for time.Now().Before(deadline) { + n++ + if got, _, _ := fx.read(t, "a.txt"); got == "OTHER" { + stop.Store(true) + <-done + t.Fatalf("read %d returned the file outside the store through a swapped link", n) + } + } + stop.Store(true) + <-done +} + +// A move plan applies through the store too: a target folder that leads +// outside the store for an instant between the re-check and the rename must +// not receive the file. +func TestDcnPlanMoveRaceNeverLeavesTheStore(t *testing.T) { + fx := newDcnStoreFixture(t) + if err := os.Remove(filepath.Join(fx.other, "a.txt")); err != nil { + t.Fatal(err) + } + docs := filepath.Join(fx.store, "docs") + var stop atomic.Bool + done := make(chan struct{}) + go func() { + defer close(done) + for !stop.Load() { + _ = os.Rename(docs, docs+".real") + _ = os.Symlink(fx.other, docs+".link") + _ = os.Rename(docs+".link", docs) + _ = os.Rename(docs, docs+".link") + _ = os.Remove(docs + ".link") + _ = os.Rename(docs+".real", docs) + } + }() + defer func() { stop.Store(true); <-done }() + deadline := time.Now().Add(3 * time.Second) + for i := 0; time.Now().Before(deadline); i++ { + _ = os.WriteFile(filepath.Join(fx.store, "a.txt"), []byte("MINE"), 0o644) + plan := &dcnPlanTerms{PlanID: int64(i), StoreID: 1, Root: fx.store, + Moves: []dcnMove{{From: "a.txt", To: "docs/a.txt"}}} + _ = dcnApplyPlan(fx.store, nil, plan) + if _, err := os.Lstat(filepath.Join(fx.other, "a.txt")); err == nil { + t.Fatalf("move %d landed outside the store, through a folder that was a link for an instant", i) + } + _ = os.Remove(filepath.Join(fx.store, "docs.real", "a.txt")) + _ = os.Remove(filepath.Join(docs, "a.txt")) + } +} + +// The plan report is written beside its name and renamed into place. Whoever +// can write in the store could plant a link at that side name; the write must +// never follow it, or the root bastion truncates whatever file the link names +// (an excluded .ssh/authorized_keys included) and fills it with the report. +func TestDcnReportWriteNeverFollowsAPlantedSideName(t *testing.T) { + store := t.TempDir() + victim := filepath.Join(store, ".ssh", "authorized_keys") + if err := os.MkdirAll(filepath.Dir(victim), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(victim, []byte("ssh-ed25519 AAAAexample user@example\n"), 0o600); err != nil { + t.Fatal(err) + } + plans := filepath.Join(store, ".agent", "plans") + if err := os.MkdirAll(plans, 0o700); err != nil { + t.Fatal(err) + } + rel := filepath.Join(".agent", "plans", "7.json") + // Every side name a writer could guess: this process's pid, and a spread + // of others, all pointing at the victim. + for _, pid := range []int{os.Getpid(), 1, 2, 100, 1000} { + side := filepath.Join(store, rel+".tmp-"+strconv.Itoa(pid)) + if err := os.Symlink(filepath.Join("..", "..", ".ssh", "authorized_keys"), side); err != nil { + t.Fatal(err) + } + } + r, err := os.OpenRoot(store) + if err != nil { + t.Fatal(err) + } + defer r.Close() + if err := dcnRootAtomicWrite(r, rel, []byte(`{"state":"ok"}`)); err != nil { + t.Fatalf("report write: %v", err) + } + if got, _ := os.ReadFile(victim); string(got) != "ssh-ed25519 AAAAexample user@example\n" { + t.Fatalf("the report write followed a planted link: the victim now holds %q", got) + } + if got, err := os.ReadFile(filepath.Join(store, rel)); err != nil || string(got) != `{"state":"ok"}` { + t.Fatalf("report not in place: %q %v", got, err) + } + if fi, err := os.Lstat(filepath.Join(store, rel)); err != nil || fi.Mode()&os.ModeSymlink != 0 || fi.Mode().Perm() != 0o600 { + t.Fatalf("report is not a 0600 regular file: %v %v", fi, err) + } +} + +// A grant installed before pins existed has no root_real. It still works, +// pins where its folder leads at first use, keeps that pin, and a repoint +// after that is refused like any other. +func TestDcnGrantFromBeforePinsPinsAtFirstUse(t *testing.T) { + fx := newDcnStoreFixture(t) + fx.sync(t) + f, err := dcnLoadFile(fx.dir) + if err != nil { + t.Fatal(err) + } + rec := f.Stores["1"] + rec.RootReal = "" + f.Stores["1"] = rec + if err := dcnSaveFile(fx.dir, f); err != nil { + t.Fatal(err) + } + if got, why, exit := fx.read(t, "a.txt"); exit != 0 || got != "MINE" { + t.Fatalf("a grant from before pins no longer reads: exit %d %q %s", exit, got, why) + } + f, err = dcnLoadFile(fx.dir) + if err != nil { + t.Fatal(err) + } + want, _ := filepath.EvalSymlinks(fx.store) + if got := f.Stores["1"].RootReal; got != want { + t.Fatalf("first use pinned %q, want %q", got, want) + } + fx.repoint(t) + if got, _, exit := fx.read(t, "a.txt"); exit == 0 || got == "OTHER" { + t.Fatalf("after the first-use pin a repointed folder was read: exit %d %q", exit, got) + } +} diff --git a/client-go/dcn_store.go b/client-go/dcn_store.go new file mode 100644 index 0000000..34d186a --- /dev/null +++ b/client-go/dcn_store.go @@ -0,0 +1,199 @@ +package main + +// dcn_store.go — every byte a machine-store op touches is reached through ONE +// handle on the folder the person granted. +// +// WHY. The bastion is often root (a networked install, the macOS and Windows +// system service), and the granted folder is often someone else's. Checking a +// path and then opening it by name leaves two ways out of the store for +// whoever can write in it or above it: +// +// - the granted folder itself turned into a link to another folder after the +// grant. The old check resolved the root again on every op and judged the +// path against wherever it led NOW, so no race was needed; +// - a file or folder inside the store swapped for a link between the check +// and the open (or the rename, for a move plan). +// +// So: the folder's real path is PINNED at grant sync (rec.RootReal, this box's +// own field, outside the signed bytes), an op whose root now leads elsewhere +// is refused, and the op opens the pinned folder as an os.Root, which never +// follows a link out of it, whatever happens to the names afterwards. That +// handle is then checked to be the folder at the pinned path reached with no +// link at all (dcnNoLinkStat: an open that refuses a link in every component +// on Linux and macOS), so a swap of a folder above the store between the two +// opens is refused too. + +import ( + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strconv" +) + +// errDcnRepointed is the refusal for a granted folder that leads elsewhere now. +var errDcnRepointed = errors.New("the granted folder now leads somewhere other than the folder that was granted; " + + "a new grant for it is the repair") + +// dcnRealRoot is where the grant's root leads now: its links resolved. +func dcnRealRoot(root string) (string, error) { + real, err := filepath.EvalSymlinks(filepath.Clean(root)) + if err != nil { + return "", err + } + if !filepath.IsAbs(real) { + return "", errors.New("the granted folder is not an absolute path") + } + return filepath.Clean(real), nil +} + +// dcnPinFor is the real root a sync records: the one already pinned for the +// same root while the store is held (a re-sync, a renewed grant), else where +// the root leads now ("" when it cannot be resolved yet; pinned at first use). +func dcnPinFor(prev *dcnGrantRec, prevRoot, root string) string { + if prev != nil && prev.RootReal != "" && filepath.Clean(prevRoot) == filepath.Clean(root) { + return prev.RootReal + } + real, err := dcnRealRoot(root) + if err != nil { + return "" + } + return real +} + +// dcnStorePin is the pinned real root of a held store, pinning it now (and +// saving it) for a grant synced before pins existed. +func dcnStorePin(dir string, storeID int64, root string) (string, error) { + grantsMu.Lock() + defer grantsMu.Unlock() + f, err := dcnLoadFile(dir) + if err != nil { + return "", err + } + key := strconv.FormatInt(storeID, 10) + rec, ok := f.Stores[key] + if !ok { + return "", errors.New("no grant for this store on this box") + } + if rec.RootReal != "" { + return rec.RootReal, nil + } + real, err := dcnRealRoot(root) + if err != nil { + return "", err + } + rec.RootReal = real + f.Stores[key] = rec + if err := dcnSaveFile(dir, f); err != nil { + return "", err + } + return real, nil +} + +// dcnOpenStore opens the store a verified grant names, as the folder that was +// granted, or refuses. +func dcnOpenStore(dir string, storeID int64, terms *dcnTerms) (*os.Root, error) { + pin, err := dcnStorePin(dir, storeID, terms.Root) + if err != nil { + return nil, fmt.Errorf("the granted folder cannot be opened: %w", err) + } + real, err := dcnRealRoot(terms.Root) + if err != nil { + return nil, fmt.Errorf("the granted folder cannot be opened: %w", err) + } + if real != pin { + return nil, errDcnRepointed + } + return dcnOpenRootAt(pin) +} + +// dcnOpenRootAt opens real (a resolved, absolute path) as an os.Root and +// checks that the handle is the folder reached at real with no link in any +// component, so nothing above the folder was swapped while it was opened. +func dcnOpenRootAt(real string) (*os.Root, error) { + r, err := os.OpenRoot(real) + if err != nil { + return nil, fmt.Errorf("the granted folder cannot be opened: %w", err) + } + rfi, err := r.Stat(".") + if err != nil { + _ = r.Close() + return nil, fmt.Errorf("the granted folder cannot be opened: %w", err) + } + nfi, err := dcnNoLinkStat(real) + if err != nil || !nfi.IsDir() || !os.SameFile(rfi, nfi) { + _ = r.Close() + return nil, errDcnRepointed + } + return r, nil +} + +// dcnRel is a path the sandbox accepted (cleanRel/dcnResolvePath's joined +// form) as a name inside the root handle. +func dcnRel(root, joined string) (string, error) { + rel, err := filepath.Rel(filepath.Clean(root), joined) + if err != nil || rel == ".." || filepath.IsAbs(rel) || len(rel) >= 3 && rel[:3] == ".."+string(os.PathSeparator) { + return "", errors.New("path escapes the store root") + } + return rel, nil +} + +// dcnEscaped: the root handle refused a name because it leads out of the store. +func dcnEscaped(err error) bool { + var pe *os.PathError + return errors.As(err, &pe) && pe.Err != nil && pe.Err.Error() == "path escapes from parent" +} + +// dcnRootAtomicWrite is atomicWrite inside the store's handle: folders 0700, +// the file 0600, written beside its name and renamed into place. +func dcnRootAtomicWrite(r *os.Root, rel string, data []byte) error { + if d := filepath.Dir(rel); d != "." { + if err := r.MkdirAll(d, 0o700); err != nil { + return err + } + } + // The side name is unguessable and opened O_EXCL, as atomicWrite's + // CreateTemp is: whoever can write in the store could otherwise plant a + // link at a guessable name, and an open that follows it would truncate + // and fill whatever file in the store the link names. O_EXCL refuses any + // existing name, a link included, without following it. + var f *os.File + var tmp string + for try := 0; ; try++ { + var nb [12]byte + if _, err := rand.Read(nb[:]); err != nil { + return err + } + tmp = filepath.Join(filepath.Dir(rel), ".dcn-"+hex.EncodeToString(nb[:])+".tmp") + var err error + f, err = r.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err == nil { + break + } + if !errors.Is(err, fs.ErrExist) || try >= 8 { + return err + } + } + if _, err := f.Write(data); err != nil { + _ = f.Close() + _ = r.Remove(tmp) + return err + } + if err := f.Sync(); err != nil { + _ = f.Close() + _ = r.Remove(tmp) + return err + } + if err := f.Close(); err != nil { + _ = r.Remove(tmp) + return err + } + if err := r.Rename(tmp, rel); err != nil { + _ = r.Remove(tmp) + return err + } + return nil +} diff --git a/client-go/dcn_store_darwin.go b/client-go/dcn_store_darwin.go new file mode 100644 index 0000000..08785e8 --- /dev/null +++ b/client-go/dcn_store_darwin.go @@ -0,0 +1,21 @@ +//go:build darwin + +package main + +import ( + "os" + + "golang.org/x/sys/unix" +) + +// dcnNoLinkStat describes the directory at p opened with O_NOFOLLOW_ANY, which +// fails on a link in any component (macOS 11 and later). +func dcnNoLinkStat(p string) (os.FileInfo, error) { + fd, err := unix.Open(p, unix.O_RDONLY|unix.O_DIRECTORY|unix.O_NOFOLLOW_ANY|unix.O_CLOEXEC, 0) + if err != nil { + return nil, err + } + f := os.NewFile(uintptr(fd), p) + defer f.Close() + return f.Stat() +} diff --git a/client-go/dcn_store_linux.go b/client-go/dcn_store_linux.go new file mode 100644 index 0000000..618e917 --- /dev/null +++ b/client-go/dcn_store_linux.go @@ -0,0 +1,48 @@ +//go:build linux + +package main + +import ( + "errors" + "os" + "strings" + + "golang.org/x/sys/unix" +) + +// dcnNoLinkStat describes the directory at p (absolute, clean) reached one +// component at a time with O_NOFOLLOW, so a link anywhere in the path fails +// instead of being followed. O_PATH: a component the bastion may traverse but +// not list (a home at 0711) still opens. +func dcnNoLinkStat(p string) (os.FileInfo, error) { + if !strings.HasPrefix(p, "/") { + return nil, errors.New("not an absolute path") + } + fd, err := unix.Open("/", unix.O_PATH|unix.O_DIRECTORY|unix.O_CLOEXEC, 0) + if err != nil { + return nil, err + } + for _, c := range strings.Split(p, "/") { + if c == "" { + continue + } + if c == "." || c == ".." { + _ = unix.Close(fd) + return nil, errors.New("not a clean path") + } + next, err := unix.Openat(fd, c, unix.O_PATH|unix.O_NOFOLLOW|unix.O_DIRECTORY|unix.O_CLOEXEC, 0) + _ = unix.Close(fd) + if err != nil { + return nil, err + } + var st unix.Stat_t + if err := unix.Fstat(next, &st); err != nil || st.Mode&unix.S_IFMT != unix.S_IFDIR { + _ = unix.Close(next) + return nil, errors.New("a component is not a directory") + } + fd = next + } + f := os.NewFile(uintptr(fd), p) + defer f.Close() + return f.Stat() +} diff --git a/client-go/dcn_store_other.go b/client-go/dcn_store_other.go new file mode 100644 index 0000000..2d7b700 --- /dev/null +++ b/client-go/dcn_store_other.go @@ -0,0 +1,14 @@ +//go:build !linux && !darwin + +package main + +import "os" + +// dcnNoLinkStat on a platform with no open that refuses a link in every +// component (Windows): the final component is judged without following it. +// The store is still reached only through its os.Root and only while its real +// path matches the pin; what this leaves is a swap of a folder above the +// store, by someone who can rename it, timed between the two opens. +func dcnNoLinkStat(p string) (os.FileInfo, error) { + return os.Lstat(p) +} diff --git a/client-go/egress.go b/client-go/egress.go index 5fd3948..0eb59df 100644 --- a/client-go/egress.go +++ b/client-go/egress.go @@ -6,6 +6,7 @@ import ( "errors" "io" "net" + "net/http" "strings" "time" ) @@ -24,94 +25,110 @@ type egressDialer func(ctx context.Context, network, addr string) (net.Conn, err // egress is refused (403, before any dial) unless its pairing grant carries // the "egress" cap: this lane makes requests leave THIS machine's connection // under its address, and nothing in the mgmt op table gates it. -func handleEgressConn(client net.Conn, dialOut egressDialer, self string) { +// +// It reports whether it took the job: true once the request passed every check +// and was dialled out, whatever the destination then answered; false when the +// worker closed before a request, or the request was malformed or refused. The +// worker pool (runEgressWorkers) parks again at once after a job it took, and +// waits after one it did not. +func handleEgressConn(client net.Conn, dialOut egressDialer, self string) bool { defer client.Close() - br := bufio.NewReader(client) + // The request line and headers are read under a bound, the one Go's own HTTP + // server uses (http.DefaultMaxHeaderBytes, 1 MiB): the host sends them, and a + // head that never ends must not grow in this machine's memory. + headLim := &io.LimitedReader{R: client, N: egressMaxHeadBytes} + br := bufio.NewReader(headLim) line, err := br.ReadString('\n') if err != nil { - return + return false } parts := strings.Fields(strings.TrimSpace(line)) if len(parts) < 2 { - return + return false } method, target := parts[0], parts[1] - // Buffer the head so a plain-HTTP request can be replayed verbatim; capture Host. + // Buffer the head: a plain-HTTP request is parsed from it and rebuilt + // (parseEgressRequest), never replayed verbatim. var head strings.Builder head.WriteString(line) - hostHdr := "" for { h, err := br.ReadString('\n') if err != nil { - return + return false } head.WriteString(h) if h == "\r\n" || h == "\n" { break } - if len(h) >= 5 && strings.EqualFold(h[:5], "host:") { - hostHdr = strings.TrimSpace(h[5:]) - } } + // The head is in; what follows is the stream itself, unbounded. What the + // reader already buffered past the head stays first in line. + headLim.N = 1<<63 - 1 + if why := capsAllow(self, "egress"); why != "" { - _, _ = client.Write([]byte("HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\nX-Privatae-Refused: " + why + "\r\n\r\n")) - return + writeEgressRefusal(client, "403 Forbidden", why, why) + return false } ctx, cancel := context.WithTimeout(context.Background(), egressDialTimeout) defer cancel() if strings.EqualFold(method, "CONNECT") { - dialAddrs, why := egressVettedAddrs(ctx, ensurePort(target, "443")) + // The target is read the way the Host's egress gate reads it + // (parseEgressAuthority): userinfo, an unbracketed IPv6 literal or a + // malformed port is refused, never guessed at. + host, port, ok := parseEgressAuthority(target, 443) + if !ok { + writeEgressRefusal(client, "400 Bad Request", egressNoHost, egressNoHost) + return false + } + dialAddrs, why := egressVettedAddrs(ctx, egressHostPort(host, port)) if why != "" { - _, _ = client.Write([]byte("HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\nX-Privatae-Refused: " + why + "\r\n\r\n")) - return + writeEgressRefusal(client, "403 Forbidden", why, why) + return false } upstream, err := dialEach(ctx, dialOut, dialAddrs) if err != nil { - _, _ = client.Write([]byte("HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\n\r\n")) - return + writeEgressRefusal(client, "502 Bad Gateway", egressUnreachable, "") + return true } if _, err := client.Write([]byte("HTTP/1.1 200 Connection established\r\n\r\n")); err != nil { _ = upstream.Close() - return + return true } splice(br, client, upstream) - return + return true } - // Plain HTTP, absolute-form: "GET http://host/path HTTP/1.1". - host := hostHdr - if i := strings.Index(target, "://"); i >= 0 { - rest := target[i+3:] - if s := strings.IndexByte(rest, '/'); s >= 0 { - host = rest[:s] - } else { - host = rest - } + // Plain HTTP: ONE request per connection, parsed and rebuilt so the + // destination reads the host, port and body the lane vetted, sent with + // `Connection: close` (egress_http.go). Anything the Host sends after that + // request's body closes the connection unforwarded: a keep-alive connection + // can no longer carry later requests past the vetting. + req, refused := parseEgressRequest(head.String()) + if refused != nil { + writeEgressRefusal(client, refused.status, refused.body, refused.body) + return false } - if host == "" { - return - } - dialAddrs, why := egressVettedAddrs(ctx, ensurePort(host, "80")) + dialAddrs, why := egressVettedAddrs(ctx, egressHostPort(req.host, req.port)) if why != "" { - _, _ = client.Write([]byte("HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\nX-Privatae-Refused: " + why + "\r\n\r\n")) - return + writeEgressRefusal(client, "403 Forbidden", why, why) + return false } upstream, err := dialEach(ctx, dialOut, dialAddrs) if err != nil { - _, _ = client.Write([]byte("HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\n\r\n")) - return + writeEgressRefusal(client, "502 Bad Gateway", egressUnreachable, "") + return true } - if _, err := upstream.Write([]byte(head.String())); err != nil { - _ = upstream.Close() - return - } - splice(br, client, upstream) + oneEgressExchange(br, client, upstream, req) + return true } +// egressMaxHeadBytes bounds an egress request's line and headers together. +const egressMaxHeadBytes = http.DefaultMaxHeaderBytes + // ensurePort returns host:port, adding defPort when target has no explicit port. func ensurePort(target, defPort string) string { target = strings.TrimSpace(target) @@ -157,11 +174,14 @@ func splice(clientReader io.Reader, client, upstream net.Conn) { // included), and any address the machine takes as its own (ownaddr.go), such // as a public one on its NIC or a range a `local` route delivers here. // -// What differs is the network around it. On an ANCHORED box the lane reaches -// the public internet and nothing else: private, CGNAT and reserved addresses -// (the LAN, the mesh) are refused too. On a LOCAL box the LAN stays reachable: -// the host there is the one the person paired, and their own network is -// theirs to reach. +// The network around it is refused too, on every box: the lane reaches the +// public internet and nothing else, so private, CGNAT and reserved addresses +// (the LAN, the mesh) are refused, by address and by any address a name +// resolves to. A LOCAL box used to keep its LAN reachable here; it no longer +// does (2026-10-26): the Host's egress gate holds every dial it makes to the +// same rule, and a person's own machines +// are reached through their approved lanes, never through egress. Only the +// wording differs between the two kinds of box. // // Resolution failures refuse too: a name this box cannot resolve is not one // it can vouch for. @@ -245,6 +265,19 @@ func egressVettedAddrs(ctx context.Context, hostport string) ([]string, string) } return nil, "refused: egress to " + host + " — that is this machine; this lane never reaches this machine" } + // The network around this machine, on a local box as on an anchored one + // (above): any one address of the name that is not the public internet + // refuses it. Checked after this machine's own addresses, so those keep + // saying "that is this machine". + if !anchored { + for _, ip := range addrs { + if internalIP(ip) { + return nil, "refused: egress to " + host + " (" + ip.String() + ") — this lane reaches the " + + "public internet only, never this machine's network or the mesh; reach your own " + + "machines through their approved lanes" + } + } + } out := make([]string, 0, len(addrs)) for _, ip := range addrs { out = append(out, net.JoinHostPort(ip.String(), port)) diff --git a/client-go/egress_http.go b/client-go/egress_http.go new file mode 100644 index 0000000..3bf5eb9 --- /dev/null +++ b/client-go/egress_http.go @@ -0,0 +1,420 @@ +package main + +import ( + "bufio" + "errors" + "io" + "net" + "net/http/httputil" + "strconv" + "strings" +) + +// ONE plain-HTTP request per egress connection, read the way the Host's egress +// gate reads it (its strict request parse and its authority parse: the same +// refusals, the same normalised host, the same rebuilt head). +// +// Everything the lane writes to a destination is what it checked, and nothing +// more: +// - the head is parsed strictly and REBUILT, so the destination reads the same +// host, port and body length the lane vetted: never a second reading of the +// Host's bytes (a Host header naming another site, a query or userinfo glued +// to the host, a padded Transfer-Encoding); +// - the request goes out with `Connection: close`, its body is framed by +// Content-Length or chunked encoding, and a byte the Host sends after the +// body closes the connection with nothing of it forwarded. Before this the +// lane wrote the first head and then spliced the whole connection, so every +// later request on a keep-alive connection reached the destination unread. +// +// An honest client sends its next request only after the response, which the +// destination closes behind `Connection: close`, so it opens a new connection +// and meets the vetting again. + +// egressHostChars are the characters a host name may have here (lower-cased). +const egressHostChars = "abcdefghijklmnopqrstuvwxyz0123456789-._" + +// egressTokenChars are RFC 9110 token characters: methods and header names. +const egressTokenChars = "!#$%&'*+-.^_`|~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + +// egressHopByHop are headers that belong to one hop. The lane speaks for itself +// on the next one, and a proxy credential never goes on to the destination. +var egressHopByHop = map[string]bool{ + "connection": true, "proxy-connection": true, "keep-alive": true, "upgrade": true, + "te": true, "trailer": true, "proxy-authorization": true, "proxy-authenticate": true, +} + +// egressTrailerLimit bounds the trailer section of a chunked body, which is +// read to its end and dropped (the gate's limit). +const egressTrailerLimit = 8192 + +// egressRefusal is a request the lane refuses before it dials anything. +type egressRefusal struct{ status, body string } + +func egressBad(body string) *egressRefusal { + return &egressRefusal{"400 Bad Request", body} +} + +const egressNoHost = "The bastion could not read where this request is going (no host in the " + + "request line or in a Host header), so it sent nothing." + +// egressUnreachable is the body of a 502: the destination passed the vetting, +// and no address of it answered from this machine. +const egressUnreachable = "The bastion could not reach that destination from this machine's own " + + "connection, so nothing was sent." + +// egressHostPort is host and port as the vetting takes them. +func egressHostPort(host string, port int) string { + return net.JoinHostPort(host, strconv.Itoa(port)) +} + +// writeEgressRefusal answers the Host with a status and a short plain-text +// body saying why, and closes the exchange. why, when set, also goes in +// X-Privatae-Refused, the header the Host already reads. +func writeEgressRefusal(w io.Writer, status, body, why string) { + body += "\n" + var b strings.Builder + b.WriteString("HTTP/1.1 " + status + "\r\n") + b.WriteString("Content-Type: text/plain; charset=utf-8\r\n") + b.WriteString("Content-Length: " + strconv.Itoa(len(body)) + "\r\n") + b.WriteString("Connection: close\r\n") + if why != "" { + b.WriteString("X-Privatae-Refused: " + strings.NewReplacer("\r", " ", "\n", " ").Replace(why) + "\r\n") + } + b.WriteString("\r\n") + b.WriteString(body) + _, _ = io.WriteString(w, b.String()) +} + +// parseEgressAuthority: (host, port, true) from "host", "host:port", "[v6]" or +// "[v6]:port", or false when the lane can't read it exactly as the destination +// would. Refused rather than guessed: userinfo ("a@b"), an unbracketed IPv6 +// literal, characters no host name has, a non-numeric or out-of-range port. +// The host is lower-cased and loses one trailing dot. +func parseEgressAuthority(s string, defaultPort int) (string, int, bool) { + s = strings.TrimSpace(s) + if s == "" || strings.Contains(s, "@") { + return "", 0, false + } + var host string + port := defaultPort + digits := func(p string) bool { + if len(p) == 0 || len(p) > 5 { + return false + } + for _, c := range p { + if c < '0' || c > '9' { + return false + } + } + return true + } + if strings.HasPrefix(s, "[") { + end := strings.IndexByte(s, ']') + if end < 0 { + return "", 0, false + } + // An IPv6 literal, IPv4-mapped forms included (the vetting judges the + // address they map to); a bare IPv4 address in brackets or a zone is + // not one. + inner := s[1:end] + ip := net.ParseIP(inner) + if ip == nil || !strings.Contains(inner, ":") || strings.Contains(inner, "%") { + return "", 0, false + } + host = ip.String() + rest := s[end+1:] + switch { + case rest == "": + case strings.HasPrefix(rest, ":") && digits(rest[1:]): + port, _ = strconv.Atoi(rest[1:]) + default: + return "", 0, false + } + } else { + if strings.Count(s, ":") > 1 { + return "", 0, false + } + h, p, hasPort := strings.Cut(s, ":") + if hasPort { + if !digits(p) { + return "", 0, false + } + port, _ = strconv.Atoi(p) + } + h = strings.ToLower(h) + h = strings.TrimSuffix(h, ".") + if h == "" || strings.HasPrefix(h, ".") || strings.Contains(h, "..") { + return "", 0, false + } + for _, c := range h { + if !strings.ContainsRune(egressHostChars, c) { + return "", 0, false + } + } + host = h + } + if port <= 0 || port >= 65536 { + return "", 0, false + } + return host, port, true +} + +// fmtEgressAuthority writes host[:port] back, the port left out when it is the +// scheme's default. +func fmtEgressAuthority(host string, port, defaultPort int) string { + h := host + if strings.Contains(host, ":") { + h = "[" + host + "]" + } + if port == defaultPort { + return h + } + return h + ":" + strconv.Itoa(port) +} + +// egressRequest is the one plain-HTTP request the lane forwards. +type egressRequest struct { + host string + port int + head []byte // rebuilt, CRLF, ends at its blank line + chunked bool + length int64 +} + +func egressTokenOK(s string) bool { + if s == "" { + return false + } + for _, c := range s { + if !strings.ContainsRune(egressTokenChars, c) { + return false + } + } + return true +} + +// parseEgressRequest reads the one request in head (the request line and +// headers through their blank line, as handleEgressConn read them) and +// rebuilds it, or refuses it. CONNECT never comes here. +func parseEgressRequest(head string) (egressRequest, *egressRefusal) { + lines := strings.Split(head, "\n") + // The read stops at the first blank line, so the head ends "\n", blank, "\n": + // drop the empty tail after the last "\n" and the blank line itself. + if len(lines) < 3 { + return egressRequest{}, egressBad("The request ended before its headers did, so the bastion sent nothing.") + } + lines = lines[:len(lines)-2] + for i, ln := range lines { + lines[i] = strings.TrimSuffix(ln, "\r") + if strings.ContainsAny(lines[i], "\r\x00") { + return egressRequest{}, egressBad("The request's headers carry a stray carriage return or NUL, so the bastion sent nothing.") + } + } + parts := strings.Split(lines[0], " ") + if len(parts) != 3 || parts[0] == "" || parts[1] == "" || parts[2] == "" || !egressTokenOK(parts[0]) { + return egressRequest{}, egressBad("The request line is not 'METHOD target HTTP/1.x', so the bastion sent nothing.") + } + method, rawTarget, version := strings.ToUpper(parts[0]), parts[1], parts[2] + if version != "HTTP/1.1" && version != "HTTP/1.0" { + return egressRequest{}, egressBad("The bastion forwards HTTP/1.0 and HTTP/1.1 only, so it sent nothing.") + } + type header struct{ name, value string } + var headers []header + for _, ln := range lines[1:] { + if strings.HasPrefix(ln, " ") || strings.HasPrefix(ln, "\t") { + return egressRequest{}, egressBad("A header is folded onto a second line (obsolete line folding), so the bastion sent nothing.") + } + name, value, ok := strings.Cut(ln, ":") + if !ok || !egressTokenOK(name) { + return egressRequest{}, egressBad("A header line is not 'Name: value' (no space may come before the colon), so the bastion sent nothing.") + } + headers = append(headers, header{name, strings.Trim(value, " \t")}) + } + values := func(n string) []string { + var out []string + for _, h := range headers { + if strings.EqualFold(h.name, n) { + out = append(out, h.value) + } + } + return out + } + + hosts := values("host") + if len(hosts) > 1 { + return egressRequest{}, egressBad("The request carries more than one Host header, so the bastion sent nothing.") + } + var host, target, authority string + var port int + if scheme, after, abs := strings.Cut(rawTarget, "://"); abs { + if strings.ToLower(scheme) != "http" { + // https:// in absolute form asks the lane to make the TLS connection, + // which it does not do: it used to send the request in plain text to + // port 80. Every client sends https as a CONNECT. + return egressRequest{}, egressBad("The bastion forwards plain http:// requests only; send https as a CONNECT. Nothing was sent.") + } + cut := len(after) + for _, sep := range []string{"/", "?", "#"} { + if i := strings.Index(after, sep); i >= 0 && i < cut { + cut = i + } + } + var ok bool + if host, port, ok = parseEgressAuthority(after[:cut], 80); !ok { + return egressRequest{}, egressBad(egressNoHost) + } + path, _, _ := strings.Cut(after[cut:], "#") + if !strings.HasPrefix(path, "/") { + path = "/" + path + } + authority = fmtEgressAuthority(host, port, 80) + target = "http://" + authority + path + } else { + if !strings.HasPrefix(rawTarget, "/") && rawTarget != "*" { + return egressRequest{}, egressBad(egressNoHost) + } + hostHdr := "" + if len(hosts) == 1 { + hostHdr = hosts[0] + } + var ok bool + if host, port, ok = parseEgressAuthority(hostHdr, 80); !ok { + return egressRequest{}, egressBad(egressNoHost) + } + authority = fmtEgressAuthority(host, port, 80) + target = rawTarget + } + + // Body framing (RFC 9112 section 6). Every reading two parsers could disagree + // on is refused: both length headers, an unknown or doubled coding, a coding + // on HTTP/1.0, Content-Length values that differ. + req := egressRequest{host: host, port: port} + te, cl := values("transfer-encoding"), values("content-length") + if len(te) > 0 { + var codings []string + for _, v := range te { + for _, c := range strings.Split(v, ",") { + codings = append(codings, strings.ToLower(strings.TrimSpace(c))) + } + } + if len(cl) > 0 || version == "HTTP/1.0" || len(codings) != 1 || codings[0] != "chunked" { + return egressRequest{}, egressBad("The request's body length is ambiguous (Transfer-Encoding with " + + "Content-Length, on HTTP/1.0, or a coding other than chunked), so the bastion sent nothing.") + } + req.chunked = true + } else if len(cl) > 0 { + seen := map[string]bool{} + for _, v := range cl { + for _, c := range strings.Split(v, ",") { + seen[strings.TrimSpace(c)] = true + } + } + var one string + for k := range seen { + one = k + } + n, err := strconv.ParseInt(one, 10, 64) + if len(seen) != 1 || one == "" || len(one) > 18 || err != nil || n < 0 || strings.TrimLeft(one, "0123456789") != "" { + return egressRequest{}, egressBad("The request's Content-Length is not one number, so the bastion sent nothing.") + } + req.length = n + } + + named := map[string]bool{} + for _, v := range values("connection") { + for _, t := range strings.Split(v, ",") { + named[strings.ToLower(strings.TrimSpace(t))] = true + } + } + var b strings.Builder + b.WriteString(method + " " + target + " " + version + "\r\n") + b.WriteString("Host: " + authority + "\r\n") + for _, h := range headers { + lk := strings.ToLower(h.name) + if egressHopByHop[lk] || named[lk] || lk == "host" || lk == "content-length" || lk == "transfer-encoding" { + continue + } + b.WriteString(h.name + ": " + h.value + "\r\n") + } + if req.chunked { + b.WriteString("Transfer-Encoding: chunked\r\n") + } else if len(cl) > 0 { + b.WriteString("Content-Length: " + strconv.FormatInt(req.length, 10) + "\r\n") + } + b.WriteString("Connection: close\r\n\r\n") + req.head = []byte(b.String()) + return req, nil +} + +// forwardEgressBody copies the one request body from src to dst, framed as the +// rebuilt head says. A chunked body goes out re-encoded (plain sizes, no +// extensions, trailers read and dropped), so the destination parses exactly the +// body this parser found. +func forwardEgressBody(dst io.Writer, src *bufio.Reader, req egressRequest) error { + if !req.chunked { + if req.length == 0 { + return nil + } + _, err := io.CopyN(dst, src, req.length) + return err + } + cw := httputil.NewChunkedWriter(dst) + if _, err := io.Copy(cw, httputil.NewChunkedReader(src)); err != nil { + return err + } + // The trailer section, read to its blank line within a bound, and dropped. + read := 0 + for { + line, err := src.ReadSlice('\n') + read += len(line) + if err != nil || read > egressTrailerLimit { + return errors.New("chunked body: bad trailer section") + } + if string(line) == "\r\n" || string(line) == "\n" { + break + } + } + if err := cw.Close(); err != nil { + return err + } + _, err := io.WriteString(dst, "\r\n") + return err +} + +// oneEgressExchange writes the one rebuilt request and its body to upstream, +// and carries the destination's answer back until the destination closes. +// Nothing the Host sends after the body is forwarded: its first byte closes +// both sides. +func oneEgressExchange(clientReader *bufio.Reader, client, upstream net.Conn, req egressRequest) { + answered := make(chan struct{}) + go func() { + _, _ = io.Copy(client, upstream) + close(answered) + _ = client.Close() + }() + defer func() { + _ = upstream.Close() + <-answered + }() + if _, err := upstream.Write(req.head); err != nil { + return + } + if err := forwardEgressBody(upstream, clientReader, req); err != nil { + return + } + // The request is complete. A further byte is a second request on this + // connection: never forwarded, and the connection closes. End of stream + // from the Host is passed on, as the splice did. + go func() { + var one [1]byte + if n, _ := clientReader.Read(one[:]); n > 0 { + _ = upstream.Close() + _ = client.Close() + return + } + if cw, ok := upstream.(interface{ CloseWrite() error }); ok { + _ = cw.CloseWrite() + } + }() + <-answered +} diff --git a/client-go/egress_local_test.go b/client-go/egress_local_test.go index 5306267..2711be2 100644 --- a/client-go/egress_local_test.go +++ b/client-go/egress_local_test.go @@ -7,8 +7,10 @@ package main // The egress lane on a local (unanchored) box. It refuses this machine and // its link-local addresses, so a host cannot reach this machine's services -// (ollama above all) around the switches that govern them, and it keeps the -// person's LAN. An anchored box is unchanged: the public internet only. +// (ollama above all) around the switches that govern them, and, since +// 2026-10-26, the network around it as well: the LAN and the mesh +// (egress_network_test.go). An anchored box is unchanged: the public internet +// only. import ( "bufio" @@ -57,13 +59,14 @@ func localEgressFixture(t *testing.T) { "self.example.test": {"192.168.50.7"}, "nas.example.test": {"192.168.50.30"}, "dual.example.test": {"2001:db8::8", "198.51.100.8"}, + "site.example.test": {"198.51.100.9"}, }) } // MUTATION: restore the early return for an unanchored box in egressVettedAddrs. // RESULT: red — 127.0.0.1:11434 is handed to the dial, and a name is dialled // as a name instead of the address that was vetted. -func TestLocalEgressRefusesThisMachineAndKeepsTheLAN(t *testing.T) { +func TestLocalEgressRefusesThisMachineAndTheLAN(t *testing.T) { localEgressFixture(t) ctx := context.Background() for _, hp := range []string{ @@ -75,24 +78,22 @@ func TestLocalEgressRefusesThisMachineAndKeepsTheLAN(t *testing.T) { "203.0.113.5:80", // a range a `local` route delivers here "ollama.example.test:11434", "mixed.example.test:443", "self.example.test:11434", "unknown.example.test:443", + // the network around this machine (egress_network_test.go has the rest) + "192.168.50.20:8080", "10.1.2.3:443", "172.16.4.4:22", "[fd12:3456::9]:80", "100.64.0.9:80", + "nas.example.test:443", } { if addr, why := egressVettedAddr(ctx, hp); why == "" || addr != "" { t.Errorf("%s must be refused on a local box, got dial %q", hp, addr) } } for hp, want := range map[string][]string{ - "192.168.50.20:8080": {"192.168.50.20:8080"}, - "10.1.2.3:443": {"10.1.2.3:443"}, - "172.16.4.4:22": {"172.16.4.4:22"}, - "[fd12:3456::9]:80": {"[fd12:3456::9]:80"}, - "100.64.0.9:80": {"100.64.0.9:80"}, "198.51.100.8:443": {"198.51.100.8:443"}, - "nas.example.test:443": {"192.168.50.30:443"}, + "site.example.test:443": {"198.51.100.9:443"}, "dual.example.test:443": {"[2001:db8::8]:443", "198.51.100.8:443"}, } { got, why := egressVettedAddrs(ctx, hp) if why != "" || !reflect.DeepEqual(got, want) { - t.Errorf("%s is the LAN or the internet, so a local box dials %v: got %v %q", hp, want, got, why) + t.Errorf("%s is the internet, so a local box dials %v: got %v %q", hp, want, got, why) } } } @@ -114,8 +115,8 @@ func TestAnchoredEgressStillRefusesTheLAN(t *testing.T) { } // Through the lane itself, with a real dialer: a request for this machine is -// answered 403 and never reaches the listener; a LAN request is dialled at its -// vetted address; a dual-stack name falls back to its second address. +// answered 403 and never reaches the listener; a public name is dialled at +// its vetted address; a dual-stack name falls back to its second address. func TestLocalEgressLaneNeverDialsThisMachine(t *testing.T) { localEgressFixture(t) ln, err := net.Listen("tcp", "127.0.0.1:0") @@ -164,11 +165,11 @@ func TestLocalEgressLaneNeverDialsThisMachine(t *testing.T) { asked = append(asked, addr) return nil, errors.New("stop here") } - if line := ask(recording, "CONNECT nas.example.test:443 HTTP/1.1\r\n\r\n"); !strings.HasPrefix(line, "HTTP/1.1 502") { - t.Fatalf("a LAN destination is dialled (and this dial fails): %q", line) + if line := ask(recording, "CONNECT site.example.test:443 HTTP/1.1\r\n\r\n"); !strings.HasPrefix(line, "HTTP/1.1 502") { + t.Fatalf("a public destination is dialled (and this dial fails): %q", line) } - if !reflect.DeepEqual(asked, []string{"192.168.50.30:443"}) { - t.Fatalf("the vetted LAN address, never the name: %v", asked) + if !reflect.DeepEqual(asked, []string{"198.51.100.9:443"}) { + t.Fatalf("the vetted address, never the name: %v", asked) } asked = nil diff --git a/client-go/egress_network_test.go b/client-go/egress_network_test.go new file mode 100644 index 0000000..ce3b27a --- /dev/null +++ b/client-go/egress_network_test.go @@ -0,0 +1,76 @@ +//go:build !windows + +// Tagged !windows: it borrows localEgressFixture and answerEgressNames +// (egress_local_test.go). + +package main + +// The egress lane on a local (unanchored) box refuses the network around the +// machine as well as the machine itself: the LAN, the mesh and every other +// address that is not the public internet, by address and by any address a +// name resolves to. The Host's egress gate holds every dial to the same rule; +// a person's own machines are reached through their approved lanes. + +import ( + "bufio" + "context" + "errors" + "io" + "net" + "reflect" + "strings" + "testing" + "time" +) + +// MUTATION: drop the local-box internalIP loop at the end of egressVettedAddrs. +// RESULT: red — 192.168.50.20, nas.example.test and the rest are handed to the +// dial, and the lane dials 192.168.50.30. +func TestLocalEgressRefusesTheLANAndTheMesh(t *testing.T) { + localEgressFixture(t) + answerEgressNames(t, map[string][]string{ + "nas.example.test": {"192.168.50.30"}, + "half.example.test": {"198.51.100.8", "10.1.2.3"}, + "mesh.example.test": {"fd12:3456::9"}, + "nat.example.test": {"64:ff9b::a01:203"}, // NAT64 for 10.1.2.3 + "pub.example.test": {"198.51.100.8"}, + }) + ctx := context.Background() + for _, hp := range []string{ + "192.168.50.20:8080", "10.1.2.3:443", "172.16.4.4:22", "[fd12:3456::9]:80", "100.64.0.9:80", + "198.18.0.1:80", "[64:ff9b::a01:203]:443", + "nas.example.test:443", "half.example.test:443", "mesh.example.test:80", "nat.example.test:443", + } { + if addrs, why := egressVettedAddrs(ctx, hp); why == "" || addrs != nil || !strings.Contains(why, "approved lanes") { + t.Errorf("%s is not the public internet and must be refused on a local box, got dial %v %q", hp, addrs, why) + } + } + if got, why := egressVettedAddrs(ctx, "pub.example.test:443"); why != "" || !reflect.DeepEqual(got, []string{"198.51.100.8:443"}) { + t.Fatalf("the public internet passes at its vetted address: %v %q", got, why) + } + + // Through the lane itself: 403, and nothing is dialled. + var asked []string + recording := func(_ context.Context, _, addr string) (net.Conn, error) { + asked = append(asked, addr) + return nil, errors.New("stop here") + } + for _, req := range []string{ + "CONNECT nas.example.test:443 HTTP/1.1\r\n\r\n", + "GET http://10.1.2.3/ HTTP/1.1\r\nHost: 10.1.2.3\r\n\r\n", + "CONNECT half.example.test:443 HTTP/1.1\r\n\r\n", + } { + srv, cli := net.Pipe() + go handleEgressConn(srv, recording, "home.host.example") + _ = cli.SetDeadline(time.Now().Add(5 * time.Second)) + _, _ = io.WriteString(cli, req) + line, _ := bufio.NewReader(cli).ReadString('\n') + cli.Close() + if !strings.HasPrefix(line, "HTTP/1.1 403") { + t.Errorf("a local box must refuse %q, answered %q", req, line) + } + } + if len(asked) != 0 { + t.Fatalf("nothing on the network around this machine is dialled: %v", asked) + } +} diff --git a/client-go/egress_one_request_test.go b/client-go/egress_one_request_test.go new file mode 100644 index 0000000..a977448 --- /dev/null +++ b/client-go/egress_one_request_test.go @@ -0,0 +1,393 @@ +//go:build !windows + +package main + +import ( + "bufio" + "bytes" + "context" + "errors" + "fmt" + "io" + "net" + "net/http" + "strings" + "sync" + "testing" + "time" +) + +// The bastion's egress lanes carry ONE plain-HTTP request per connection, read +// the way the Host's egress gate reads it, and every refusal says +// why in its body. +// +// MUTATIONS, each red: +// - handleEgressConn writes head.String() to upstream and splices, as before: +// TestEgressPlainHTTPCarriesOneRequestPerConnection, ...SendsTheHostItChecked; +// - oneEgressExchange's watcher forwards instead of closing: ...OneRequest...; +// - parseEgressAuthority accepts "@" or cuts the authority at "/" only: +// TestEgressPlainHTTPReadsTheHostAsTheGateDoes; +// - writeEgressRefusal writes Content-Length: 0: TestEgressRefusalsCarryABody. + +// recordingOrigin stands in for a destination that keeps a connection open +// after answering (it ignores Connection: close, as a careless server may), +// and records every byte one connection carried to it. +type recordingOrigin struct { + ln net.Listener + conns chan string + // silent: never answer, so the lane never holds an unread answer when it + // closes (a close with unread data is a reset, which can cost the origin + // bytes it has not read yet). + silent bool +} + +func newSilentOrigin(t *testing.T) *recordingOrigin { + o := newRecordingOrigin(t) + o.silent = true + return o +} + +func newRecordingOrigin(t *testing.T) *recordingOrigin { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + o := &recordingOrigin{ln: ln, conns: make(chan string, 16)} + t.Cleanup(func() { _ = ln.Close() }) + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + go o.serve(c) + } + }() + return o +} + +func (o *recordingOrigin) serve(c net.Conn) { + defer c.Close() + var mu sync.Mutex + var got bytes.Buffer + answered := false + buf := make([]byte, 4096) + for { + _ = c.SetReadDeadline(time.Now().Add(800 * time.Millisecond)) + n, err := c.Read(buf) + mu.Lock() + got.Write(buf[:n]) + if !o.silent && !answered && bytes.Contains(got.Bytes(), []byte("\r\n\r\n")) { + answered = true + _, _ = io.WriteString(c, "HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok") + } + mu.Unlock() + if err != nil { + break + } + } + o.conns <- got.String() +} + +// next is everything the origin's next connection carried, once it ended. +func (o *recordingOrigin) next(t *testing.T) string { + t.Helper() + s, ok := o.nextWithin(8 * time.Second) + if !ok { + t.Fatal("the destination never saw its connection end") + } + return s +} + +func (o *recordingOrigin) nextWithin(d time.Duration) (string, bool) { + select { + case s := <-o.conns: + return s, true + case <-time.After(d): + return "", false + } +} + +func (o *recordingOrigin) addr() string { return o.ln.Addr().String() } + +// A keep-alive connection carries one request to the destination, never two: +// neither a request sent after the first answer (a client reusing the +// connection) nor one pipelined behind the first. +func TestEgressPlainHTTPCarriesOneRequestPerConnection(t *testing.T) { + first := "GET http://198.51.100.30/first HTTP/1.1\r\nHost: 198.51.100.30\r\n\r\n" + second := "GET http://198.51.100.30/second HTTP/1.1\r\nHost: 198.51.100.30\r\n\r\n" + + t.Run("reused after the answer", func(t *testing.T) { + o := newRecordingOrigin(t) + var asked []string + cli, cleanup := dialHandler(t, redirectDial(o.addr(), &asked)) + defer cleanup() + _, _ = io.WriteString(cli, first) + br := bufio.NewReader(cli) + resp, err := http.ReadResponse(br, nil) + if err != nil { + t.Fatalf("the first request got no answer: %v", err) + } + body, _ := io.ReadAll(resp.Body) + if string(body) != "ok" { + t.Fatalf("the first answer was %q", body) + } + _, _ = io.WriteString(cli, second) + got := o.next(t) + if strings.Contains(got, "/second") { + t.Fatalf("a second request on one connection reached the destination unvetted:\n%s", got) + } + if !strings.Contains(got, "\r\nConnection: close\r\n") { + t.Fatalf("the request went out without Connection: close:\n%s", got) + } + _ = cli.SetReadDeadline(time.Now().Add(3 * time.Second)) + if rest, _ := io.ReadAll(br); len(rest) != 0 { + t.Fatalf("the lane answered the second request: %q", rest) + } + if len(asked) != 1 { + t.Fatalf("one connection, one dial: %v", asked) + } + }) + + t.Run("pipelined", func(t *testing.T) { + o := newRecordingOrigin(t) + var asked []string + cli, cleanup := dialHandler(t, redirectDial(o.addr(), &asked)) + defer cleanup() + _, _ = io.WriteString(cli, first+second) + got := o.next(t) + if !strings.Contains(got, "GET http://198.51.100.30/first HTTP/1.1\r\n") { + t.Fatalf("the first request did not reach the destination:\n%s", got) + } + if strings.Contains(got, "/second") { + t.Fatalf("a pipelined second request reached the destination unvetted:\n%s", got) + } + }) +} + +// The destination reads the host the lane vetted, in the one Host header; hop +// headers, those Connection names and a proxy credential stay on this hop. +func TestEgressPlainHTTPSendsTheHostItChecked(t *testing.T) { + o := newRecordingOrigin(t) + var asked []string + cli, cleanup := dialHandler(t, redirectDial(o.addr(), &asked)) + defer cleanup() + _, _ = io.WriteString(cli, "GET http://198.51.100.31/p?q=1 HTTP/1.1\r\n"+ + "Host: inside.example.test\r\nProxy-Authorization: Basic eDp5\r\n"+ + "Connection: keep-alive, X-Hop\r\nX-Hop: 1\r\nX-Keep: 2\r\n\r\n") + got := o.next(t) + head, _, _ := strings.Cut(got, "\r\n\r\n") + want := "GET http://198.51.100.31/p?q=1 HTTP/1.1\r\nHost: 198.51.100.31\r\nX-Keep: 2\r\nConnection: close" + if head != want { + t.Fatalf("the destination got\n%q\nwant\n%q", head, want) + } + if len(asked) != 1 || asked[0] != "198.51.100.31:80" { + t.Fatalf("the lane must dial the address it vetted: %v", asked) + } +} + +// stubEgressNames answers names without DNS and records each name exactly as +// the lane asked for it. +func stubEgressNames(t *testing.T, m map[string]string) *[]string { + t.Helper() + var mu sync.Mutex + var asked []string + old := egressResolve + t.Cleanup(func() { egressResolve = old }) + egressResolve = func(_ context.Context, host string) ([]net.IPAddr, error) { + mu.Lock() + asked = append(asked, host) + mu.Unlock() + ip, ok := m[host] + if !ok { + return nil, errors.New("no such host") + } + return []net.IPAddr{{IP: net.ParseIP(ip)}}, nil + } + return &asked +} + +// The host is read as the gate reads it: the authority ends at the first "/", +// "?" or "#", is lower-cased and loses one trailing dot; userinfo, an +// unbracketed IPv6 literal, a bad port, https in absolute form, two Host +// headers and an ambiguous body length are refused with 400 and a body, before +// anything is resolved or dialled. +func TestEgressPlainHTTPReadsTheHostAsTheGateDoes(t *testing.T) { + for _, c := range []struct { + req, wantName, wantLine string + }{ + {"GET http://public.example.test?@10.0.0.1/ HTTP/1.1\r\n\r\n", "public.example.test", + "GET http://public.example.test/?@10.0.0.1/ HTTP/1.1"}, + {"GET http://public.example.test#@10.0.0.1/ HTTP/1.1\r\n\r\n", "public.example.test", + "GET http://public.example.test/ HTTP/1.1"}, + {"GET http://Public.Example.Test./x HTTP/1.1\r\n\r\n", "public.example.test", + "GET http://public.example.test/x HTTP/1.1"}, + {"GET /x HTTP/1.1\r\nHost: Public.Example.Test:8080\r\n\r\n", "public.example.test", + "GET /x HTTP/1.1"}, + } { + o := newRecordingOrigin(t) + names := stubEgressNames(t, map[string]string{"public.example.test": "198.51.100.40"}) + var asked []string + cli, cleanup := dialHandler(t, redirectDial(o.addr(), &asked)) + _, _ = io.WriteString(cli, c.req) + got, _ := o.nextWithin(4 * time.Second) + cleanup() + if len(*names) != 1 || (*names)[0] != c.wantName { + t.Errorf("%q: the lane resolved %q; want only %q", c.req, *names, c.wantName) + } + if line, _, _ := strings.Cut(got, "\r\n"); line != c.wantLine { + t.Errorf("%q: the destination got the request line %q; want %q", c.req, line, c.wantLine) + } + } + + for _, req := range []string{ + "GET http://user:pw@public.example.test/ HTTP/1.1\r\n\r\n", + "GET http://198.51.100.41:80@public.example.test/ HTTP/1.1\r\n\r\n", + "GET http://2001:db8::1/ HTTP/1.1\r\n\r\n", + "GET http://public.example.test:80x/ HTTP/1.1\r\n\r\n", + "GET https://public.example.test/ HTTP/1.1\r\n\r\n", + "GET / HTTP/1.1\r\nHost: public.example.test\r\nHost: 198.51.100.41\r\n\r\n", + "GET / HTTP/1.1\r\n\r\n", + "POST http://public.example.test/ HTTP/1.1\r\nContent-Length: 3\r\nTransfer-Encoding: chunked\r\n\r\n", + "POST http://public.example.test/ HTTP/1.1\r\nContent-Length: 3\r\nContent-Length: 4\r\n\r\n", + "POST http://public.example.test/ HTTP/1.1\r\nTransfer-Encoding: gzip, chunked\r\n\r\n", + "GET http://public.example.test/ HTTP/1.1\r\n X-Folded: 1\r\n\r\n", + "CONNECT user@public.example.test:443 HTTP/1.1\r\n\r\n", + "CONNECT 2001:db8::1:443 HTTP/1.1\r\n\r\n", + "CONNECT [198.51.100.41]:443 HTTP/1.1\r\n\r\n", + } { + names := stubEgressNames(t, map[string]string{"public.example.test": "198.51.100.40"}) + var asked []string + cli, cleanup := dialHandler(t, redirectDial("127.0.0.1:9", &asked)) + _, _ = io.WriteString(cli, req) + resp, err := http.ReadResponse(bufio.NewReader(cli), nil) + if err != nil { + t.Errorf("%q: no answer: %v", req, err) + cleanup() + continue + } + body, _ := io.ReadAll(resp.Body) + cleanup() + if resp.StatusCode != 400 || len(body) == 0 || int64(len(body)) != resp.ContentLength { + t.Errorf("%q: answered %d with body %q; want 400 with a body saying why", req, resp.StatusCode, body) + } + if len(*names) != 0 || len(asked) != 0 { + t.Errorf("%q: refused requests resolve and dial nothing; resolved %q, dialled %q", req, *names, asked) + } + } +} + +// A request body goes out framed as its head says, chunked bodies re-encoded, +// and nothing after it. +func TestEgressPlainHTTPBodyIsFramed(t *testing.T) { + for _, c := range []struct{ req, wantBody string }{ + {"POST http://198.51.100.42/u HTTP/1.1\r\nHost: 198.51.100.42\r\nContent-Length: 5\r\n\r\nhello" + + "GET http://198.51.100.42/smuggled HTTP/1.1\r\n\r\n", "hello"}, + {"POST http://198.51.100.42/u HTTP/1.1\r\nHost: 198.51.100.42\r\nTransfer-Encoding: chunked\r\n\r\n" + + "5;ext=1\r\nhello\r\n0\r\nX-Trailer: 1\r\n\r\nGET /smuggled HTTP/1.1\r\n\r\n", "5\r\nhello\r\n0\r\n\r\n"}, + } { + o := newSilentOrigin(t) + var asked []string + cli, cleanup := dialHandler(t, redirectDial(o.addr(), &asked)) + _, _ = io.WriteString(cli, c.req) + got := o.next(t) + cleanup() + _, body, _ := strings.Cut(got, "\r\n\r\n") + if body != c.wantBody { + t.Errorf("the destination got the body %q; want %q", body, c.wantBody) + } + } +} + +// A refusal says why in its body, not only in a header: a 403 from the vetting +// and a 502 from a dial that found no answer, on both request forms, and the +// coder-run proxy's refusals too. +func TestEgressRefusalsCarryABody(t *testing.T) { + failDial := func(ctx context.Context, network, addr string) (net.Conn, error) { + return nil, fmt.Errorf("no route") + } + okDial := func(ctx context.Context, network, addr string) (net.Conn, error) { + return nil, fmt.Errorf("must not dial %s", addr) + } + check := func(name string, resp *http.Response, err error, want int) { + t.Helper() + if err != nil { + t.Errorf("%s: no answer: %v", name, err) + return + } + body, _ := io.ReadAll(resp.Body) + if resp.StatusCode != want || len(body) < 10 || int64(len(body)) != resp.ContentLength { + t.Errorf("%s: answered %d with body %q (Content-Length %d); want %d with a short body saying why", + name, resp.StatusCode, body, resp.ContentLength, want) + } + } + for _, c := range []struct { + name string + dial egressDialer + req string + want int + }{ + {"plain HTTP to this machine", okDial, "GET http://127.0.0.1:9/ HTTP/1.1\r\n\r\n", 403}, + {"CONNECT to this machine", okDial, "CONNECT 127.0.0.1:9 HTTP/1.1\r\n\r\n", 403}, + {"plain HTTP, no answer", failDial, "GET http://198.51.100.43/ HTTP/1.1\r\n\r\n", 502}, + {"CONNECT, no answer", failDial, "CONNECT 198.51.100.43:443 HTTP/1.1\r\n\r\n", 502}, + } { + cli, cleanup := dialHandler(t, c.dial) + _, _ = io.WriteString(cli, c.req) + resp, err := http.ReadResponse(bufio.NewReader(cli), nil) + check(c.name, resp, err, c.want) + cleanup() + } + + oldDial := coderDialOut + t.Cleanup(func() { coderDialOut = oldDial }) + coderDialOut = failDial + for _, c := range []struct { + name, req string + want int + }{ + {"coder run to a private address", "GET http://10.0.0.1/ HTTP/1.1\r\nHost: 10.0.0.1\r\n\r\n", 403}, + {"coder run, no answer", "GET http://93.184.216.34/ HTTP/1.1\r\nHost: 93.184.216.34\r\n\r\n", 502}, + } { + boxEnd, runEnd := net.Pipe() + var refused int64 + go handleCoderEgressConn(boxEnd, nil, &refused) + _ = runEnd.SetDeadline(time.Now().Add(5 * time.Second)) + _, _ = io.WriteString(runEnd, c.req) + resp, err := http.ReadResponse(bufio.NewReader(runEnd), nil) + check(c.name, resp, err, c.want) + _ = runEnd.Close() + } +} + +// The coder-run proxy carries one request per connection too: the run's own +// processes are the sender there. +func TestCoderEgressCarriesOneRequestPerConnection(t *testing.T) { + o := newRecordingOrigin(t) + oldDial := coderDialOut + t.Cleanup(func() { coderDialOut = oldDial }) + coderDialOut = func(ctx context.Context, network, addr string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, network, o.addr()) + } + boxEnd, runEnd := net.Pipe() + var refused int64 + go handleCoderEgressConn(boxEnd, nil, &refused) + _ = runEnd.SetDeadline(time.Now().Add(8 * time.Second)) + _, _ = io.WriteString(runEnd, "GET http://93.184.216.34/first HTTP/1.1\r\nHost: 93.184.216.34\r\n\r\n") + br := bufio.NewReader(runEnd) + resp, err := http.ReadResponse(br, nil) + if err != nil { + t.Fatalf("the first request got no answer: %v", err) + } + _, _ = io.ReadAll(resp.Body) + _, _ = io.WriteString(runEnd, "GET http://93.184.216.34/second HTTP/1.1\r\nHost: inside.example.test\r\n\r\n") + got := o.next(t) + _ = runEnd.Close() + if strings.Contains(got, "/second") || strings.Contains(got, "inside.example.test") { + t.Fatalf("a second request on one connection reached the destination unvetted:\n%s", got) + } + if !strings.Contains(got, "\r\nConnection: close\r\n") { + t.Fatalf("the request went out without Connection: close:\n%s", got) + } +} diff --git a/client-go/egress_relay.go b/client-go/egress_relay.go index a07af78..b2dc576 100644 --- a/client-go/egress_relay.go +++ b/client-go/egress_relay.go @@ -14,35 +14,60 @@ import ( const egressWorkerCount = 4 +// egressRefusedWait is how long a worker waits before it parks again after a +// job it did not take (malformed, refused) or after its WebSocket closed while +// parked. A job it took parks the worker again at once, so the person's own +// traffic is never slowed by it; a Host that answers every worker with a job +// this machine refuses costs it at most egressWorkerCount dials, handshakes +// and refusals a second, not as many as the Host can send. The serve lane's +// workers wait the same (serveRefusedWait). +var egressRefusedWait = time.Second + // wsConn adapts a worker WebSocket into a net.Conn so the existing handleEgressConn drives // it unchanged. The host frames the proxied client stream as binary messages: 'D'+payload = // data, 'E' = end-of-stream. Reads surface the host's bytes; writes go back as 'D' frames; // Close/CloseWrite send 'E'. +// +// A 'D' frame is STREAMED into the caller's buffer, never read whole: the host picks a +// frame's length, and one frame of any length must cost this machine no more memory +// than the read it is asked for. type wsConn struct { - ws *websocket.Conn - rbuf []byte - wmu sync.Mutex + ws *websocket.Conn + cur io.Reader // the rest of the 'D' frame being read, or nil + wmu sync.Mutex } func (c *wsConn) Read(p []byte) (int, error) { - for len(c.rbuf) == 0 { - mt, data, err := c.ws.ReadMessage() + for { + if c.cur != nil { + n, err := c.cur.Read(p) + if err == io.EOF { + c.cur = nil + err = nil + } + if n > 0 || err != nil { + return n, err + } + continue + } + mt, r, err := c.ws.NextReader() if err != nil { return 0, err } - if mt != websocket.BinaryMessage || len(data) == 0 { - continue + if mt != websocket.BinaryMessage { + continue // NextReader discards what is left of it } - switch data[0] { + var tag [1]byte + if _, err := io.ReadFull(r, tag[:]); err != nil { + continue // an empty frame + } + switch tag[0] { case 'D': - c.rbuf = data[1:] + c.cur = r case 'E': return 0, io.EOF } } - n := copy(p, c.rbuf) - c.rbuf = c.rbuf[n:] - return n, nil } func (c *wsConn) Write(p []byte) (int, error) { @@ -80,19 +105,25 @@ func (c *wsConn) SetWriteDeadline(t time.Time) error { return c.ws.SetWriteDeadl // bastion's tunnel IP) and serves one egress job per worker, reconnecting after each. Each // worker authenticates with the bastion's bearer, then blocks until the host dispatches a // request; handleEgressConn then dials it out the USER's connection (dialOut) and relays. +// A worker reconnects at once after a job it took, and after egressRefusedWait after one +// it refused or a close while parked. The func returned waits until every worker has +// exited, which is once pctx is done and each one's job has ended. // // pctx is the pairing's context (run.go): when the anchor revokes a pairing the // workers must exit, not re-dial a closed tunnel every 3s for the process's life. func runEgressWorkers(pctx context.Context, self string, wsURL string, tlsCfg *tls.Config, dialHost func(ctx context.Context) (net.Conn, error), bearer string, - dialOut egressDialer, n int) { + dialOut egressDialer, n int) (wait func()) { dialer := &websocket.Dialer{ NetDialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { return dialHost(ctx) }, TLSClientConfig: tlsCfg, HandshakeTimeout: 20 * time.Second, } + var wg sync.WaitGroup for i := 0; i < n; i++ { + wg.Add(1) go func() { + defer wg.Done() for pctx.Err() == nil { ctx, cancel := context.WithTimeout(pctx, 25*time.Second) ws, _, err := dialer.DialContext(ctx, wsURL, http.Header{}) @@ -108,11 +139,15 @@ func runEgressWorkers(pctx context.Context, self string, wsURL string, tlsCfg *t } stopKA := armWSKeepalive(ws) // ping/read-deadline: a dead tunnel unblocks the park unpark := context.AfterFunc(pctx, func() { _ = ws.Close() }) - handleEgressConn(&wsConn{ws: ws}, dialOut, self) // blocks until a job, serves it, then returns + took := handleEgressConn(&wsConn{ws: ws}, dialOut, self) // blocks until a job, serves it, then returns unpark() stopKA() _ = ws.Close() + if !took { + sleepCtx(pctx, egressRefusedWait) + } } }() } + return wg.Wait } diff --git a/client-go/egress_relay_test.go b/client-go/egress_relay_test.go new file mode 100644 index 0000000..a718814 --- /dev/null +++ b/client-go/egress_relay_test.go @@ -0,0 +1,166 @@ +//go:build !windows + +// The egress worker pool (egress_relay.go: runEgressWorkers) against a +// stand-in Host's /bastion-egress: how fast workers park again after a job +// they took and after one they refused. Uses localEgressFixture +// (egress_local_test.go), so it carries the same build tag. + +package main + +import ( + "context" + "io" + "log" + "net" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/gorilla/websocket" +) + +// egressHost is a Host's /bastion-egress as the pool meets it: every worker +// that parks is counted and handed the job job(n) (n is its place in line), +// and what the bastion answered is read to its end. +type egressHost struct { + parked atomic.Int32 + ok atomic.Int32 // answers that began "HTTP/1.1 200" +} + +func startEgressHost(t *testing.T, job func(n int32) string) (*egressHost, string, func(context.Context) (net.Conn, error)) { + t.Helper() + h := &egressHost{} + up := websocket.Upgrader{} + srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + c, err := up.Upgrade(w, r, nil) + if err != nil { + return + } + defer c.Close() + if _, _, err := c.ReadMessage(); err != nil { // BEARER + return + } + n := h.parked.Add(1) + if err := c.WriteMessage(websocket.BinaryMessage, append([]byte{'D'}, job(n)...)); err != nil { + return + } + _ = c.SetReadDeadline(time.Now().Add(5 * time.Second)) + var answer strings.Builder + for { + _, m, err := c.ReadMessage() + if err != nil || len(m) == 0 || m[0] == 'E' { + break + } + answer.Write(m[1:]) + } + if strings.HasPrefix(answer.String(), "HTTP/1.1 200") { + h.ok.Add(1) + } + })) + srv.Config.ErrorLog = log.New(io.Discard, "", 0) + srv.Start() + t.Cleanup(srv.Close) + addr := srv.Listener.Addr().String() + dial := func(ctx context.Context) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, "tcp", addr) + } + return h, "ws://" + addr + "/bastion-egress", dial +} + +// runEgressPool runs egressWorkerCount workers for the test's life, and at its +// end stops them and waits until each has exited: a worker still vetting a job +// would otherwise read the fixture's stubs while the test's cleanup puts them +// back. +func runEgressPool(t *testing.T, wsURL string, dialHost func(context.Context) (net.Conn, error), dialOut egressDialer) { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + wait := runEgressWorkers(ctx, "home.host.example", wsURL, nil, dialHost, "test-bearer", dialOut, egressWorkerCount) + t.Cleanup(func() { + cancel() + done := make(chan struct{}) + go func() { wait(); close(done) }() + select { + case <-done: + case <-time.After(10 * time.Second): + t.Error("the egress workers did not exit 10 s after the pool stopped") + } + }) +} + +// SECURITY (this machine's resources): a Host that answers every parked egress +// worker with a job this machine refuses (a request for this machine, a line +// that is no request) cannot make it dial, handshake and refuse in a loop as +// fast as the Host answers. Each worker that refused waits before it parks +// again, so the pool costs at most egressWorkerCount dials a second. +// +// MUTATION: in runEgressWorkers, park again at once whatever handleEgressConn +// reports (as the pool did). +// RESULT: red — hundreds of workers park in the 2 s window. +func TestARefusingHostCannotSpinTheEgressWorkers(t *testing.T) { + localEgressFixture(t) + var dialled atomic.Int32 + dialOut := func(ctx context.Context, network, addr string) (net.Conn, error) { + dialled.Add(1) + return nil, io.EOF + } + h, wsURL, dialHost := startEgressHost(t, func(n int32) string { + if n%2 == 0 { + return "NOT-A-REQUEST\r\n" + } + return "CONNECT localhost:443 HTTP/1.1\r\nHost: localhost:443\r\n\r\n" + }) + runEgressPool(t, wsURL, dialHost, dialOut) + time.Sleep(2 * time.Second) + // egressWorkerCount park at the start, then each worker once a second: + // about 12 in 2 s. A pool that parks again at once makes hundreds. + n := h.parked.Load() + t.Logf("%d workers parked in 2 s", n) + if limit := int32(egressWorkerCount * 4); n > limit { + t.Fatalf("%d egress workers parked in 2 s on a Host that refuses every job (at most %d): the pool re-dials in a loop", n, limit) + } + if n < egressWorkerCount { + t.Fatalf("only %d workers parked: the pool did not run, so the count proves nothing", n) + } + if d := dialled.Load(); d != 0 { + t.Fatalf("a refused job was dialled out %d times", d) + } +} + +// The person's own traffic is not slowed: a worker that took a job parks again +// at once, so 40 jobs through 4 workers finish in a fraction of the time one +// wait per job would take (40 / 4 x egressRefusedWait = 10 s). +// +// MUTATION: in runEgressWorkers, wait egressRefusedWait after every job. +// RESULT: red — 40 jobs need about 10 s. +func TestEgressWorkersParkAgainAtOnceAfterAJobTheyTook(t *testing.T) { + localEgressFixture(t) + dialOut := func(ctx context.Context, network, addr string) (net.Conn, error) { + if addr != "198.51.100.9:443" { + t.Errorf("dialled %q, not the address the request named", addr) + } + ours, theirs := net.Pipe() + _ = theirs.Close() // the destination accepts and ends at once + return ours, nil + } + h, wsURL, dialHost := startEgressHost(t, func(int32) string { + return "CONNECT 198.51.100.9:443 HTTP/1.1\r\nHost: 198.51.100.9:443\r\n\r\n" + }) + start := time.Now() + runEgressPool(t, wsURL, dialHost, dialOut) + const jobs = 40 + deadline := start.Add(10 * time.Second) + for h.ok.Load() < jobs && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + } + took := time.Since(start) + t.Logf("%d accepted jobs answered in %v", h.ok.Load(), took) + if got := h.ok.Load(); got < jobs { + t.Fatalf("only %d of %d accepted jobs were answered 200 in %v", got, jobs, took) + } + if limit := 3 * egressRefusedWait; took > limit { + t.Fatalf("%d accepted jobs took %v (over %v): a worker that took a job waited before it parked again", jobs, took, limit) + } +} diff --git a/client-go/harness.go b/client-go/harness.go index 981da78..c97cb5f 100644 --- a/client-go/harness.go +++ b/client-go/harness.go @@ -252,16 +252,39 @@ func harnessGate(self, dcnDir string, spec harnessSpec) string { // `BEARER lane=session`); a harness worker refuses it so a shell can never hold // an agent worker for hours. func handleHarnessConn(ws *websocket.Conn, self, dcnDir string) { - mt, data, err := ws.ReadMessage() - if err != nil || mt != websocket.BinaryMessage || len(data) == 0 { + // The spec is read up to harnessSpecMaxBytes (lane_bounds.go) and refused + // past it; a frame of any other tag is not read at all. + tag, body, tooBig, err := readLaneJob(ws, func(t byte) int64 { + if t == 'H' { + return harnessSpecMaxBytes + } + return 0 + }) + if err != nil { return } - if data[0] == 'S' { + if tag == 'S' { sendHarness(ws, 'E', []byte("this is a harness worker; sessions go to a worker that said lane=session\n")) sendHarnessExit(ws, 1) return } - if data[0] != 'H' { + if tag != 'H' { + return + } + if tooBig { + // Gated like a spec that does not parse (below): a pairing the caps + // refuse learns nothing more from a frame that was too large. + if why := capsAllow(self, "harness:run"); why != "" { + b, _ := json.Marshal(map[string]any{"ok": false, "error": why}) + sendHarness(ws, 'J', b) + sendHarnessExit(ws, 126) + return + } + b, _ := json.Marshal(map[string]any{"ok": false, "error": fmt.Sprintf( + "the harness spec is more than the %d MiB a job on this lane may be; nothing of it was read as a job", + harnessSpecMaxBytes>>20)}) + sendHarness(ws, 'J', b) + sendHarnessExit(ws, 1) return } // The gate depends on WHAT is asked: a service-slot job (service.go) is the @@ -270,7 +293,7 @@ func handleHarnessConn(ws *websocket.Conn, self, dcnDir string) { // before its parse error is reported, exactly as it always was, so an // unauthorized pairing learns nothing from a malformed frame. var spec harnessSpec - perr := json.Unmarshal(data[1:], &spec) + perr := json.Unmarshal(body, &spec) gateOp := "harness:run" if perr == nil && spec.Mode == "service" { gateOp = serviceGateOp(spec.ServiceOp) @@ -332,12 +355,10 @@ func handleHarnessConn(ws *websocket.Conn, self, dcnDir string) { go func() { b, c := runHarness(spec); resCh <- harnessRes{b, c} }() closed := make(chan struct{}) go func() { - for { - if _, _, err := ws.ReadMessage(); err != nil { // host close or drop - close(closed) - return - } - } + // Nothing the host sends during a run is read as anything: each frame + // is dropped as it arrives (lane_bounds.go), never held whole. + _ = drainLane(ws) // returns on a host close or drop + close(closed) }() ticker := time.NewTicker(8 * time.Second) defer ticker.Stop() @@ -951,7 +972,10 @@ func driveHarnessAgent(rt, dir string, spec harnessSpec, maxMin int) (agentRes m return nil, fmt.Errorf("harness /run: %v", err) } defer resp.Body.Close() - rb, _ := io.ReadAll(resp.Body) + rb, err := readHarnessRunReply(resp.Body) + if err != nil { + return nil, err + } var res map[string]interface{} if err := json.Unmarshal(rb, &res); err != nil { return nil, fmt.Errorf("harness /run bad json (HTTP %d): %s", resp.StatusCode, truncate(string(rb), 200)) @@ -959,6 +983,21 @@ func driveHarnessAgent(rt, dir string, spec harnessSpec, maxMin int) (agentRes m return res, nil } +// readHarnessRunReply reads the agent container's answer to /run. The +// container writes it, and it goes back to the host in one lane message, so it +// is read to the most that message carries (uvicornWSMaxSize) and no further: +// a reply that never ends must not grow in this machine's memory. +func readHarnessRunReply(body io.Reader) ([]byte, error) { + rb, err := io.ReadAll(io.LimitReader(body, uvicornWSMaxSize+1)) + if err != nil { + return nil, fmt.Errorf("harness /run: reading the reply: %v", err) + } + if len(rb) > uvicornWSMaxSize { + return nil, fmt.Errorf("harness /run: the reply is more than the %d MiB the lane to the host carries", uvicornWSMaxSize>>20) + } + return rb, nil +} + // harnessAgentUID is the user the harness image runs its agent as, the owner // of its home /home/privatae, where the credential folder is mounted. const harnessAgentUID = 10005 diff --git a/client-go/hostops_test.go b/client-go/hostops_test.go index 7f67bbe..cfe5408 100644 --- a/client-go/hostops_test.go +++ b/client-go/hostops_test.go @@ -1010,15 +1010,15 @@ func TestAnchoredEgressRefusesLoopbackAndPrivateDestinations(t *testing.T) { if addr, why := egressVettedAddr(ctx, "localhost:80"); why == "" || addr != "" { t.Fatalf("a refused name yields no address: %q %q", addr, why) } - // A local box refuses this machine too, and keeps its LAN - // (egress_local_test.go has the rest). + // A local box refuses this machine too, and the network around it + // (egress_local_test.go and egress_network_test.go have the rest). t.Setenv("XDG_CONFIG_HOME", t.TempDir()) stubOwnAddrs(t, loopbackOnly, localRange()) if why := egressInternalRefusal(ctx, "127.0.0.1:11434"); why == "" { t.Fatal("a local box must refuse its own ollama on the egress lane") } - if addr, why := egressVettedAddr(ctx, "192.168.1.1:80"); why != "" || addr != "192.168.1.1:80" { - t.Fatalf("a local box keeps its LAN: %q %q", addr, why) + if addr, why := egressVettedAddr(ctx, "192.168.1.1:80"); why == "" || addr != "" { + t.Fatalf("a local box refuses its LAN on the egress lane: %q %q", addr, why) } } diff --git a/client-go/inferencefilter.go b/client-go/inferencefilter.go index 212b7a8..71575c6 100644 --- a/client-go/inferencefilter.go +++ b/client-go/inferencefilter.go @@ -133,18 +133,39 @@ func inferenceFilterNeeded(self string, lane inferenceLane, scope modelScope) bo // // With nothing to enforce it is the original raw splice, byte for byte. func proxyInference(in net.Conn, upstream string, lane inferenceLane, self string) { + proxyInferenceVia(in, dialUpstream(upstream), lane, self, spliceConnVia) +} + +// upstreamDial opens the connection to the service behind a forward. +type upstreamDial func() (net.Conn, error) + +// dialUpstream is the forwards' dial: TCP to the address, 5 s. +func dialUpstream(upstream string) upstreamDial { + return func() (net.Conn, error) { return net.DialTimeout("tcp", upstream, 5*time.Second) } +} + +// proxyInferenceVia is proxyInference with the dial, and the pass-through for +// a connection with nothing to enforce, in the caller's hands. The serve lane +// (servelane.go) uses it so its connections take the same decision and the +// same filter as the tunnel's, while the dial goes to the address it vetted +// and is watched by its idle bound. +func proxyInferenceVia(in net.Conn, dial upstreamDial, lane inferenceLane, self string, pass func(net.Conn, upstreamDial)) { if inferenceFilterNeeded(self, lane, pairingModelScope(self)) { - serveInferenceFiltered(in, upstream, lane, self) + serveInferenceFilteredVia(in, dial, lane, self) return } - spliceConn(in, upstream) + pass(in, dial) } // spliceConn is what the forwards did before this file existed, unchanged and // in one place now that three call sites share it. func spliceConn(in net.Conn, upstream string) { + spliceConnVia(in, dialUpstream(upstream)) +} + +func spliceConnVia(in net.Conn, dial upstreamDial) { defer in.Close() - out, err := net.DialTimeout("tcp", upstream, 5*time.Second) + out, err := dial() if err != nil { return } @@ -163,8 +184,12 @@ func spliceConn(in net.Conn, upstream string) { // grant on every request below, so there is nowhere for a caller — including a // test — to hand it one the grant does not say. func serveInferenceFiltered(in net.Conn, upstream string, lane inferenceLane, self string) { + serveInferenceFilteredVia(in, dialUpstream(upstream), lane, self) +} + +func serveInferenceFilteredVia(in net.Conn, dial upstreamDial, lane inferenceLane, self string) { defer in.Close() - out, err := net.DialTimeout("tcp", upstream, 5*time.Second) + out, err := dial() if err != nil { writeFilterError(in, nil, http.StatusBadGateway, "the service behind this lane is not answering") return diff --git a/client-go/lane_bounds.go b/client-go/lane_bounds.go new file mode 100644 index 0000000..c72266e --- /dev/null +++ b/client-go/lane_bounds.go @@ -0,0 +1,122 @@ +package main + +// What a paired host sends down a worker lane is read at a size this machine +// chose, never at one the host chose: the host picks a frame's length, and a +// frame of any length must not be held whole in this machine's memory. The +// egress lane (egress_relay.go) and the context socket (contextsock.go) follow +// the same rule. +// +// - A job's first frame (the H spec on a harness worker, the S spec on a +// session worker) is read up to its lane's bound and refused past it, by +// name, before any of it is parsed. +// - What a lane carries after that is streamed: terminal input into the pty +// (session.go), the replica's bytes into its socket (mirror.go). Frames that +// nothing reads (a harness run and a root command, which read nothing after +// their spec) are dropped as they arrive. + +import ( + "errors" + "fmt" + "io" + "net/http" + + "github.com/gorilla/websocket" +) + +const ( + // harnessSpecMaxBytes bounds the H frame. Its bulk is the worktree tar + // (worktree_b64). The largest tree this lane hands back is + // maxReturnedTreeBytes (9 MiB, so 12 MiB in base64) inside one message of + // uvicornWSMaxSize (16 MiB), the most the host takes in (harness.go). A host + // that sends that tree into its next run needs those 12 MiB, plus the + // prompt and the run's credentials. Twice the lane's message size holds + // that with 20 MiB to spare. + harnessSpecMaxBytes = 2 * uvicornWSMaxSize + + // sessionSpecMaxBytes bounds the S frame: a signed grant (op ids and the + // person's own words), ids, a public key, and for session:root:exec the + // command, itself held to rootExecCommandMax (64 KiB), which is at most + // 384 KiB even when JSON escapes every byte as \u00XX. 1 MiB is the bound + // Go's own HTTP server puts on a request head. + sessionSpecMaxBytes = http.DefaultMaxHeaderBytes + + // sessionInputMaxBytes bounds one frame on an open terminal. The host relays + // what the person's browser sent it, one message each, and its own server + // takes a message up to uvicornWSMaxSize; the tag is one byte more. Input is + // streamed into the pty, so this caps a frame's length, not memory. + sessionInputMaxBytes = uvicornWSMaxSize + 1 + + // sessionResizeMaxBytes bounds the body of an R frame, {"cols":N,"rows":N}. + sessionResizeMaxBytes = 4 << 10 + + // heartbeatReplyMaxBytes bounds the host's answer to a heartbeat. It + // carries directives and the lists of grants this box should hold, each + // entry an id and a signature of about 130 bytes, so 4 MiB is tens of + // thousands of grants. Nothing in it is bulk: an update names a version, + // and the binary comes from the release manifest, not from this reply. + heartbeatReplyMaxBytes = 4 << 20 +) + +// errLaneNotJob is a first frame that is not a binary frame with a tag. +var errLaneNotJob = errors.New("the first frame is not a job") + +// readLaneJob reads a worker's first frame: its tag, and its body up to +// maxFor(tag) bytes. A body past its bound is never kept: tooBig reports it, +// and body is nil. A tag the caller does not take should get a bound of 0; +// its body is then not read at all. +func readLaneJob(ws *websocket.Conn, maxFor func(tag byte) int64) (tag byte, body []byte, tooBig bool, err error) { + mt, r, err := ws.NextReader() + if err != nil { + return 0, nil, false, err + } + if mt != websocket.BinaryMessage { + return 0, nil, false, errLaneNotJob + } + var t [1]byte + if _, err := io.ReadFull(r, t[:]); err != nil { + return 0, nil, false, errLaneNotJob // an empty frame + } + max := maxFor(t[0]) + if max <= 0 { + return t[0], nil, false, nil + } + body, err = io.ReadAll(io.LimitReader(r, max+1)) + if err != nil { + return t[0], nil, false, err + } + if int64(len(body)) > max { + return t[0], nil, true, nil + } + return t[0], body, false, nil +} + +// drainLane reads and drops every frame until the socket fails, holding none +// of them, and returns that failure. A lane that reads nothing after its job +// still has to read, to answer pings and to notice the host going away. +func drainLane(ws *websocket.Conn) error { + for { + _, r, err := ws.NextReader() + if err != nil { + return err + } + if _, err := io.Copy(io.Discard, r); err != nil { + return err + } + } +} + +// readHeartbeatReply reads the host's answer to a heartbeat up to +// heartbeatReplyMaxBytes. ok is false for a reply past the bound or one that +// failed to arrive: such a beat is treated as unanswered, so none of its +// directives runs and it does not count as the host answering. +func readHeartbeatReply(resp *http.Response) (body []byte, ok bool) { + body, err := io.ReadAll(io.LimitReader(resp.Body, heartbeatReplyMaxBytes+1)) + if err != nil { + return nil, false + } + if len(body) > heartbeatReplyMaxBytes { + fmt.Printf("heartbeat: the host's reply is more than %d MiB; ignored\n", heartbeatReplyMaxBytes>>20) + return nil, false + } + return body, true +} diff --git a/client-go/lane_bounds_helpers_test.go b/client-go/lane_bounds_helpers_test.go new file mode 100644 index 0000000..cb3b0bf --- /dev/null +++ b/client-go/lane_bounds_helpers_test.go @@ -0,0 +1,126 @@ +package main + +// The helpers lane_bounds.go and harness.go added for the reads that were +// inline before (the heartbeat reply in run.go, the agent's /run reply in +// driveHarnessAgent, the drains in harness.go and rootexec.go). They have no +// counterpart on main to run against. + +import ( + "bytes" + "io" + "net/http" + "net/http/httptest" + "runtime" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/gorilla/websocket" +) + +// boxLaneHost stands up the HOST side of one worker socket (host runs on the +// upgraded server end) and returns the box side, dialled. +func boxLaneHost(t *testing.T, host func(c *websocket.Conn)) *websocket.Conn { + t.Helper() + up := websocket.Upgrader{} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + c, err := up.Upgrade(w, r, nil) + if err != nil { + return + } + defer c.Close() + host(c) + })) + t.Cleanup(srv.Close) + ws, _, err := websocket.DefaultDialer.Dial("ws"+strings.TrimPrefix(srv.URL, "http"), nil) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = ws.Close() }) + return ws +} + +// endless is a body that never ends; served counts what was taken from it. +type endless struct{ served *atomic.Int64 } + +func (e endless) Read(p []byte) (int, error) { + for i := range p { + p[i] = 'z' + } + e.served.Add(int64(len(p))) + return len(p), nil +} + +func TestHeartbeatReplyIsReadOnlyToItsBound(t *testing.T) { + var served atomic.Int64 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + _, _ = io.Copy(w, io.LimitReader(endless{&served}, 64<<20)) + })) + defer srv.Close() + resp, err := http.Get(srv.URL) + if err != nil { + t.Fatal(err) + } + body, ok := readHeartbeatReply(resp) + resp.Body.Close() + if ok || body != nil { + t.Fatalf("a 64 MiB heartbeat reply was taken (%d bytes)", len(body)) + } + + small := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(`{"ok":true,"dcn_grants":{"stores":[]}}`)) + })) + defer small.Close() + resp, err = http.Get(small.URL) + if err != nil { + t.Fatal(err) + } + body, ok = readHeartbeatReply(resp) + resp.Body.Close() + if !ok || !strings.Contains(string(body), "dcn_grants") { + t.Fatalf("an ordinary reply was not taken: ok=%v %q", ok, body) + } +} + +func TestHarnessRunReplyIsReadOnlyToItsBound(t *testing.T) { + var served atomic.Int64 + if _, err := readHarnessRunReply(endless{&served}); err == nil || !strings.Contains(err.Error(), "more than the 16 MiB") { + t.Fatalf("an endless /run reply: %v", err) + } + if n := served.Load(); n > 17<<20 { + t.Fatalf("an endless /run reply was read to %d MiB", n>>20) + } + big := append([]byte(`{"ok":true,"pad":"`), bytes.Repeat([]byte{'a'}, 15<<20)...) + big = append(big, `"}`...) + if rb, err := readHarnessRunReply(bytes.NewReader(big)); err != nil || len(rb) != len(big) { + t.Fatalf("a 15 MiB reply the lane carries: %d bytes, %v", len(rb), err) + } +} + +// A drained lane holds no frame whole, and ends when the host goes away. +func TestDrainLaneHoldsNoFrame(t *testing.T) { + const frame = 64 << 20 + payload := bytes.Repeat([]byte{'d'}, frame) + start := make(chan struct{}) + box := boxLaneHost(t, func(c *websocket.Conn) { + <-start + _ = c.WriteMessage(websocket.BinaryMessage, payload) + }) + runtime.GC() + var before, after runtime.MemStats + runtime.ReadMemStats(&before) + close(start) + done := make(chan error, 1) + go func() { done <- drainLane(box) }() + select { + case <-done: + case <-time.After(20 * time.Second): + t.Fatal("drainLane did not return after the host went away") + } + runtime.ReadMemStats(&after) + if grew := after.TotalAlloc - before.TotalAlloc; grew > 8<<20 { + t.Fatalf("draining a %d MiB frame allocated %d MiB", frame>>20, grew>>20) + } +} diff --git a/client-go/lane_bounds_test.go b/client-go/lane_bounds_test.go new file mode 100644 index 0000000..9ecf575 --- /dev/null +++ b/client-go/lane_bounds_test.go @@ -0,0 +1,481 @@ +//go:build !windows + +package main + +// What a paired host sends down the harness, session, root-exec and mirror +// lanes, and what a coder run sends its egress proxy, is read at the size this +// machine chose (lane_bounds.go). Every test here drives only code that main +// already has, so each one also runs against main, where it fails. + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "runtime" + "strings" + "sync" + "sync/atomic" + "syscall" + "testing" + "time" + + "github.com/gorilla/websocket" +) + +// hostSendsAJob is a host that writes one job frame while it reads the box's +// replies, and reports whether its write completed and what came back. +type hostResult struct { + werr error + frames map[byte][]byte +} + +func hostSendsAJob(frame []byte, out chan<- hostResult) func(c *websocket.Conn) { + return hostSendsAJobAs(func(c *websocket.Conn) error { + return c.WriteMessage(websocket.BinaryMessage, frame) + }, out) +} + +// writeInPieces writes msg as ONE message carried by many small frames: the +// writer flushes a continuation frame each time its 4 KiB buffer fills +// (gorilla's default write buffer), so a bound that counted a frame instead +// of the message would let all of it through. +func writeInPieces(msg []byte) func(c *websocket.Conn) error { + return func(c *websocket.Conn) error { + w, err := c.NextWriter(websocket.BinaryMessage) + if err != nil { + return err + } + for off := 0; off < len(msg); off += 1 << 10 { + if _, err := w.Write(msg[off:min(off+1<<10, len(msg))]); err != nil { + return err + } + } + return w.Close() + } +} + +func hostSendsAJobAs(write func(c *websocket.Conn) error, out chan<- hostResult) func(c *websocket.Conn) { + return func(c *websocket.Conn) { + got := map[byte][]byte{} + readDone := make(chan struct{}) + go func() { + defer close(readDone) + _ = c.SetReadDeadline(time.Now().Add(20 * time.Second)) + for { + _, d, err := c.ReadMessage() + if err != nil || len(d) == 0 { + return + } + got[d[0]] = append(got[d[0]], d[1:]...) + if d[0] == 'X' { + return + } + } + }() + werr := write(c) + <-readDone + out <- hostResult{werr, got} + } +} + +// jsonFrame is tag + a JSON object whose field `pad` carries n bytes of 'a'. +func jsonFrame(tag byte, head string, n int) []byte { + b := make([]byte, 0, len(head)+n+16) + b = append(b, tag) + b = append(b, head...) + b = append(b, `,"pad":"`...) + b = append(b, bytes.Repeat([]byte{'a'}, n)...) + b = append(b, `"}`...) + return b +} + +// A harness spec far past the bound is not read to its end: the box answers +// that it is too large, names the bound, and closes, so the host's write of +// the rest fails. On main the box read all of it into memory, then ran it. +func TestHarnessWorkerStopsReadingASpecPastItsBound(t *testing.T) { + const frame = 96 << 20 + out := make(chan hostResult, 1) + box := boxLaneHost(t, hostSendsAJob(jsonFrame('H', `{"harness_id":"big","mode":"echo","prompt":"ping"`, frame), out)) + handleHarnessConn(box, "", "") + _ = box.Close() + r := <-out + if r.werr == nil { + t.Fatalf("the box read the whole %d MiB spec (it replied %.200q); it must stop at its bound", frame>>20, r.frames['J']) + } + if !strings.Contains(string(r.frames['J']), "more than the 32 MiB") || string(r.frames['X']) != "1" { + t.Fatalf("want a refusal naming the 32 MiB bound and exit 1: J=%.200q X=%q", r.frames['J'], r.frames['X']) + } +} + +// The bound counts the whole message, not one frame of it: a spec past the +// bound, sent as one message in thousands of 4 KiB frames, is refused the same +// way and not read to its end. On main the box read all of it. +func TestHarnessWorkerBoundCountsEveryFrameOfTheSpec(t *testing.T) { + const frame = 40 << 20 + out := make(chan hostResult, 1) + spec := jsonFrame('H', `{"harness_id":"pieces","mode":"echo","prompt":"ping"`, frame) + box := boxLaneHost(t, hostSendsAJobAs(writeInPieces(spec), out)) + handleHarnessConn(box, "", "") + _ = box.Close() + r := <-out + if r.werr == nil { + t.Fatalf("the box read the whole %d MiB spec sent in small frames (it replied %.200q); it must stop at its bound", frame>>20, r.frames['J']) + } + if !strings.Contains(string(r.frames['J']), "more than the 32 MiB") || string(r.frames['X']) != "1" { + t.Fatalf("want a refusal naming the 32 MiB bound and exit 1: J=%.200q X=%q", r.frames['J'], r.frames['X']) + } +} + +// The same for a session spec sent in small frames. +func TestSessionWorkerBoundCountsEveryFrameOfTheSpec(t *testing.T) { + env, _, _ := sessionTestEnv(t, []string{sessionOpOpen}, nil) + withSink(t) + const frame = 64 << 20 + out := make(chan hostResult, 1) + spec := jsonFrame('S', `{"op":"session:open","session_id":"sess-pieces"`, frame) + box := boxLaneHost(t, hostSendsAJobAs(writeInPieces(spec), out)) + handleSessionWorkerConn(box, "", env) + _ = box.Close() + r := <-out + if r.werr == nil { + t.Fatalf("the box read the whole %d MiB session spec sent in small frames (E=%q X=%q); it must stop at its bound", + frame>>20, r.frames['E'], r.frames['X']) + } + if !strings.Contains(string(r.frames['E']), "more than the 1024 KiB") || string(r.frames['X']) != "1" { + t.Fatalf("want a refusal naming the bound and exit 1: E=%q X=%q", r.frames['E'], r.frames['X']) + } +} + +// A spec the lane can carry still runs: a tree as large as the most this lane +// hands back (12 MiB in base64), with room on top, past the 16 MiB a host +// takes in. +func TestHarnessWorkerTakesALargeSpecTheLaneCarries(t *testing.T) { + out := make(chan hostResult, 1) + box := boxLaneHost(t, hostSendsAJob(jsonFrame('H', `{"harness_id":"tree","mode":"echo","prompt":"ping","worktree_b64":"dGFy"`, 20<<20), out)) + handleHarnessConn(box, "", "") + r := <-out + if r.werr != nil { + t.Fatalf("the host could not send a 20 MiB spec: %v", r.werr) + } + var res map[string]any + if err := json.Unmarshal(r.frames['J'], &res); err != nil || res["ok"] != true || res["echo"] != "ping" { + t.Fatalf("a 20 MiB spec did not run: J=%.300q X=%q", r.frames['J'], r.frames['X']) + } +} + +// The same on a session worker: an S frame far past its 1 MiB bound is +// refused by name and not read to its end. +func TestSessionWorkerStopsReadingASpecPastItsBound(t *testing.T) { + env, _, _ := sessionTestEnv(t, []string{sessionOpOpen}, nil) + withSink(t) // a box that wrongly took the spec opens no real shell + const frame = 64 << 20 + out := make(chan hostResult, 1) + box := boxLaneHost(t, hostSendsAJob(jsonFrame('S', `{"op":"session:open","session_id":"sess-big"`, frame), out)) + handleSessionWorkerConn(box, "", env) + _ = box.Close() + r := <-out + if r.werr == nil { + t.Fatalf("the box read the whole %d MiB session spec (E=%q X=%q); it must stop at its bound", + frame>>20, r.frames['E'], r.frames['X']) + } + if !strings.Contains(string(r.frames['E']), "more than the 1024 KiB") || string(r.frames['X']) != "1" { + t.Fatalf("want a refusal naming the bound and exit 1: E=%q X=%q", r.frames['E'], r.frames['X']) + } +} + +// A session spec near its bound (a grant, and room the size of a root command +// escaped six times over) still opens a terminal. +func TestSessionWorkerOpensASpecNearItsBound(t *testing.T) { + env, terms, sig := sessionTestEnv(t, []string{sessionOpOpen}, nil) + withFake(t) + // Here the test is the host: it takes the server end, and the box runs on + // the dialled end. + hostEnd := make(chan *websocket.Conn, 1) + hold := make(chan struct{}) + t.Cleanup(func() { close(hold) }) + boxEnd := boxLaneHost(t, func(c *websocket.Conn) { hostEnd <- c; <-hold }) + b := <-hostEnd + head := fmt.Sprintf(`{"op":"session:open","session_id":"sess-near","grant_terms_b64":%q,"grant_sig_b64":%q`, terms, sig) + done := make(chan struct{}) + go func() { defer close(done); handleSessionWorkerConn(boxEnd, "", env) }() + if err := b.WriteMessage(websocket.BinaryMessage, jsonFrame('S', head, 900<<10)); err != nil { + t.Fatal(err) + } + if err := b.WriteMessage(websocket.BinaryMessage, []byte("Ihello\n")); err != nil { + t.Fatal(err) + } + got := readUntil(t, b, 'O', 5*time.Second) + if string(got['O']) != "hello\n" { + t.Fatalf("a 900 KiB spec did not open a terminal: %q", got) + } + _ = b.Close() + <-done +} + +// sinkProc is a terminal that counts what is typed into it and prints nothing. +type sinkProc struct { + n atomic.Int64 + killed chan struct{} + once sync.Once +} + +func (p *sinkProc) Read(b []byte) (int, error) { <-p.killed; return 0, io.EOF } +func (p *sinkProc) Write(b []byte) (int, error) { p.n.Add(int64(len(b))); return len(b), nil } +func (p *sinkProc) Resize(c, r int) error { return nil } +func (p *sinkProc) Wait() int { <-p.killed; return 0 } +func (p *sinkProc) Kill() { p.once.Do(func() { close(p.killed) }) } +func (p *sinkProc) User() string { return "sink" } +func (p *sinkProc) Shell() string { return "/bin/sink" } + +func withSink(t *testing.T) *sinkProc { + t.Helper() + sp := &sinkProc{killed: make(chan struct{})} + old := sessionSpawn + sessionSpawn = func(op, shell string, cols, rows int) (sessionProc, error) { return sp, nil } + t.Cleanup(func() { sessionSpawn = old; sp.Kill() }) + return sp +} + +func waitFor(d time.Duration, cond func() bool) bool { + end := time.Now().Add(d) + for time.Now().Before(end) { + if cond() { + return true + } + time.Sleep(5 * time.Millisecond) + } + return cond() +} + +// Terminal input is streamed into the pty: a 16 MiB paste, the most a host +// relays in one frame, all arrives, and the box never holds it whole. +func TestSessionInputIsStreamedIntoTheTerminal(t *testing.T) { + env, terms, sig := sessionTestEnv(t, []string{sessionOpOpen}, nil) + sp := withSink(t) + ws := sessionDial(t, env) + sendSpec(t, ws, sessionSpec{Op: sessionOpOpen, GrantB64: terms, SigB64: sig, SessionID: "sess-paste"}) + const paste = 16 << 20 + frame := append([]byte{'I'}, bytes.Repeat([]byte{'x'}, paste)...) + runtime.GC() + var before, after runtime.MemStats + runtime.ReadMemStats(&before) + if err := ws.WriteMessage(websocket.BinaryMessage, frame); err != nil { + t.Fatal(err) + } + if !waitFor(20*time.Second, func() bool { return sp.n.Load() >= paste }) { + t.Fatalf("the terminal got %d of %d bytes", sp.n.Load(), paste) + } + runtime.ReadMemStats(&after) + if grew := after.TotalAlloc - before.TotalAlloc; grew > 8<<20 { + t.Fatalf("typing a %d MiB paste allocated %d MiB: the frame was held whole", paste>>20, grew>>20) + } +} + +// One frame of input past the most a host relays ends the session, and the +// terminal never receives all of it. +func TestSessionInputFramePastItsBoundEndsTheSession(t *testing.T) { + env, terms, sig := sessionTestEnv(t, []string{sessionOpOpen}, nil) + sp := withSink(t) + ws := sessionDial(t, env) + sendSpec(t, ws, sessionSpec{Op: sessionOpOpen, GrantB64: terms, SigB64: sig, SessionID: "sess-huge"}) + const frame = 17 << 20 + go func() { + _ = ws.WriteMessage(websocket.BinaryMessage, append([]byte{'I'}, bytes.Repeat([]byte{'x'}, frame)...)) + }() + ended := func() bool { + select { + case <-sp.killed: + return true + default: + return false + } + } + if !waitFor(20*time.Second, ended) { + t.Fatalf("a %d MiB input frame did not end the session; the terminal got %d bytes", frame>>20, sp.n.Load()) + } + if n := sp.n.Load(); n >= frame { + t.Fatalf("the terminal received all %d bytes of a frame past the bound", n) + } +} + +// The mirror forwarder streams a host frame into the replica's socket: reading +// 4 KiB of a 64 MiB frame does not need the frame held whole. +func TestMirrorForwarderStreamsAHostFrame(t *testing.T) { + const frame = 64 << 20 + payload := bytes.Repeat([]byte{'m'}, frame) + start := make(chan struct{}) + up := websocket.Upgrader{} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + c, err := up.Upgrade(w, r, nil) + if err != nil { + return + } + defer c.Close() + _, _, _ = c.ReadMessage() // the BEARER hello + <-start + _ = c.WriteMessage(websocket.BinaryMessage, payload) + time.Sleep(3 * time.Second) + })) + defer srv.Close() + addr := srv.Listener.Addr().String() + dialer := &websocket.Dialer{NetDialContext: func(ctx context.Context, n, a string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, "tcp", addr) + }} + boxEnd, replica := net.Pipe() + defer replica.Close() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go pipeMirrorConn(ctx, boxEnd, dialer, "ws://"+addr+"/", "b") + buf := make([]byte, 4096) + runtime.GC() + var before, after runtime.MemStats + runtime.ReadMemStats(&before) + close(start) + _ = replica.SetReadDeadline(time.Now().Add(20 * time.Second)) + if _, err := io.ReadFull(replica, buf); err != nil || buf[0] != 'm' { + t.Fatalf("the replica read: %v", err) + } + runtime.ReadMemStats(&after) + if grew := after.TotalAlloc - before.TotalAlloc; grew > 8<<20 { + t.Fatalf("passing 4 KiB of a %d MiB host frame to the replica allocated %d MiB: the frame was held whole", + frame>>20, grew>>20) + } +} + +// The forwarder carries at most eight connections at once. It listens on a +// fixed port (5433), so the test needs that port free: run it in its own +// network namespace on a machine where something already holds it. +func TestMirrorForwarderCapsConnectionsInFlight(t *testing.T) { + probe, err := net.Listen("tcp", "127.0.0.1:5433") + if err != nil { + t.Skipf("127.0.0.1:5433 is in use here (%v); run this test in its own network namespace", err) + } + _ = probe.Close() + var live, most atomic.Int64 + release := make(chan struct{}) + up := websocket.Upgrader{} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + c, err := up.Upgrade(w, r, nil) + if err != nil { + return + } + defer c.Close() + n := live.Add(1) + for { + m := most.Load() + if n <= m || most.CompareAndSwap(m, n) { + break + } + } + <-release + live.Add(-1) + })) + defer srv.Close() + defer close(release) + addr := srv.Listener.Addr().String() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go serveMirrorForward(ctx, "127.0.0.1", "127.0.0.0/8", "ws://"+addr+"/", nil, + func(ctx context.Context) (net.Conn, error) { return (&net.Dialer{}).DialContext(ctx, "tcp", addr) }, "b") + var conns []net.Conn + defer func() { + for _, c := range conns { + _ = c.Close() + } + }() + if !waitFor(5*time.Second, func() bool { + c, err := net.Dial("tcp", "127.0.0.1:5433") + if err == nil { + conns = append(conns, c) + } + return err == nil + }) { + t.Fatal("the forwarder never listened") + } + for i := 1; i < 12; i++ { + c, err := net.Dial("tcp", "127.0.0.1:5433") + if err != nil { + t.Fatal(err) + } + conns = append(conns, c) + } + waitFor(3*time.Second, func() bool { return most.Load() >= 12 }) + if m := most.Load(); m != 8 { + t.Fatalf("12 local connections made %d WebSockets to the host at once; want 8", m) + } +} + +// A coder run's egress proxy reads a request head under a bound: a request +// line that never ends is not read to its end. +func TestCoderEgressHeadIsBounded(t *testing.T) { + boxEnd, runEnd := net.Pipe() + var refused int64 + done := make(chan struct{}) + go func() { defer close(done); handleCoderEgressConn(boxEnd, nil, &refused) }() + chunk := bytes.Repeat([]byte{'a'}, 64<<10) + _, _ = runEnd.Write([]byte("GET http://example.invalid/")) + sent := 0 + var werr error + for sent < 8<<20 { + _ = runEnd.SetWriteDeadline(time.Now().Add(5 * time.Second)) + var n int + n, werr = runEnd.Write(chunk) + sent += n + if werr != nil { + break + } + } + _ = runEnd.Close() + <-done + if werr == nil { + t.Fatalf("the proxy read %d MiB of a request line with no end; it must stop at a bound", sent>>20) + } +} + +// A coder run's proxy carries at most 1024 connections at once; the next is +// answered 503 at once, never left waiting. +func TestCoderEgressCapsConnectionsInFlight(t *testing.T) { + dir := t.TempDir() + run := "lane-bounds-test" + if err := coderListenEgress(run, dir, os.Getuid()); err != nil { + t.Fatal(err) + } + defer coderCloseEgress(run) + sock := filepath.Join(dir, coderEgressSock) + var conns []net.Conn + defer func() { + for _, c := range conns { + _ = c.Close() + } + }() + for i := 0; i < 1024; i++ { + c, err := net.Dial("unix", sock) + if err != nil { + if errors.Is(err, syscall.EMFILE) || errors.Is(err, syscall.ENFILE) { + t.Skipf("this process may not open %d sockets: %v", 2*1024, err) + } + t.Fatal(err) + } + conns = append(conns, c) + } + // Every one of them is idle in its head, so the proxy holds them all. + time.Sleep(200 * time.Millisecond) + extra, err := net.Dial("unix", sock) + if err != nil { + t.Fatal(err) + } + conns = append(conns, extra) + _ = extra.SetReadDeadline(time.Now().Add(3 * time.Second)) + line, _ := io.ReadAll(io.LimitReader(extra, 256)) + if !strings.HasPrefix(string(line), "HTTP/1.1 503") { + t.Fatalf("connection 1025 of one run got %q; want a 503 at once", line) + } +} diff --git a/client-go/lane_frames_test.go b/client-go/lane_frames_test.go new file mode 100644 index 0000000..cec18c8 --- /dev/null +++ b/client-go/lane_frames_test.go @@ -0,0 +1,197 @@ +package main + +// What the host sends down a lane is read at the size the bastion asks for, +// never at the size the host chooses: one frame of any length must not be +// held whole in this machine's memory, and an egress request head that never +// ends must not be either. + +import ( + "bytes" + "context" + "errors" + "io" + "net" + "net/http" + "net/http/httptest" + "runtime" + "strconv" + "strings" + "testing" + "time" + + "github.com/gorilla/websocket" +) + +func TestEgressLaneDoesNotHoldAWholeHostFrame(t *testing.T) { + const frame = 64 << 20 + payload := make([]byte, frame+1) + payload[0] = 'D' + for i := 1; i < len(payload); i++ { + payload[i] = 'a' + } + start := make(chan struct{}) + up := websocket.Upgrader{} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ws, err := up.Upgrade(w, r, nil) + if err != nil { + return + } + defer ws.Close() + <-start + _ = ws.WriteMessage(websocket.BinaryMessage, payload) + time.Sleep(2 * time.Second) + })) + defer srv.Close() + ws, _, err := websocket.DefaultDialer.Dial("ws"+strings.TrimPrefix(srv.URL, "http"), nil) + if err != nil { + t.Fatal(err) + } + defer ws.Close() + c := &wsConn{ws: ws} + buf := make([]byte, 4096) + runtime.GC() + var before, after runtime.MemStats + runtime.ReadMemStats(&before) + close(start) + n, err := c.Read(buf) + runtime.ReadMemStats(&after) + if err != nil || n == 0 || buf[0] != 'a' { + t.Fatalf("read: n=%d err=%v", n, err) + } + if grew := after.TotalAlloc - before.TotalAlloc; grew > 8<<20 { + t.Fatalf("reading 4 KiB of a %d MiB host frame allocated %d MiB: the frame was held whole", frame>>20, grew>>20) + } +} + +func TestEgressRequestHeadIsBounded(t *testing.T) { + bastionEnd, hostEnd := net.Pipe() + done := make(chan struct{}) + go func() { + defer close(done) + handleEgressConn(bastionEnd, func(ctx context.Context, network, addr string) (net.Conn, error) { + return nil, errors.New("no dial in this test") + }, "test") + }() + chunk := []byte(strings.Repeat("a", 64<<10)) + _, _ = hostEnd.Write([]byte("GET http://example.invalid/")) + sent := 0 + var werr error + for sent < 8<<20 { + _ = hostEnd.SetWriteDeadline(time.Now().Add(5 * time.Second)) + var n int + n, werr = hostEnd.Write(chunk) + sent += n + if werr != nil { + break + } + } + _ = hostEnd.Close() + <-done + if werr == nil { + t.Fatalf("the bastion read %d MiB of a request line with no end; it must stop at a bound", sent>>20) + } +} + +// Streaming a frame must not change what the reader sees: every byte of every +// 'D' frame in order, with empty frames, text frames and unknown tags skipped, +// a frame boundary never visible, and 'E' as the end, read through a buffer +// far smaller than the frames. +func TestEgressLaneFramesReadInOrderThroughASmallBuffer(t *testing.T) { + var want []byte + frames := [][]byte{{'D'}, {}, {'X', 'z', 'z'}} + for i, size := range []int{1, 7, 4096, 70000, 3} { + f := []byte{'D'} + for j := 0; j < size; j++ { + f = append(f, byte('a'+(i+j)%26)) + } + want = append(want, f[1:]...) + frames = append(frames, f, []byte{'D'}) + } + up := websocket.Upgrader{} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ws, err := up.Upgrade(w, r, nil) + if err != nil { + return + } + defer ws.Close() + _ = ws.WriteMessage(websocket.TextMessage, []byte("Dnot data")) + for _, f := range frames { + _ = ws.WriteMessage(websocket.BinaryMessage, f) + } + _ = ws.WriteMessage(websocket.BinaryMessage, []byte{'E'}) + time.Sleep(time.Second) + })) + defer srv.Close() + ws, _, err := websocket.DefaultDialer.Dial("ws"+strings.TrimPrefix(srv.URL, "http"), nil) + if err != nil { + t.Fatal(err) + } + defer ws.Close() + c := &wsConn{ws: ws} + var got []byte + buf := make([]byte, 13) + for { + n, err := c.Read(buf) + if n == 0 && err == nil { + t.Fatal("Read returned 0, nil") + } + got = append(got, buf[:n]...) + if err == io.EOF { + break + } + if err != nil { + t.Fatalf("read: %v", err) + } + } + if !bytes.Equal(got, want) { + t.Fatalf("read %d bytes, want %d; equal prefix %d", len(got), len(want), commonPrefix(got, want)) + } +} + +func commonPrefix(a, b []byte) int { + i := 0 + for i < len(a) && i < len(b) && a[i] == b[i] { + i++ + } + return i +} + +// A large upload through the lane still arrives whole: the head bound applies +// to the head alone, and the bytes the reader buffered past it go upstream +// first, in order. +func TestEgressLargeUploadPassesTheHeadBound(t *testing.T) { + bastionEnd, hostEnd := net.Pipe() + upBastion, upFar := net.Pipe() + done := make(chan struct{}) + go func() { + defer close(done) + handleEgressConn(bastionEnd, func(ctx context.Context, network, addr string) (net.Conn, error) { + return upBastion, nil + }, "") + }() + body := make([]byte, 3*egressMaxHeadBytes) + for i := range body { + body[i] = byte(i % 251) + } + head := "POST http://198.51.100.20/upload HTTP/1.1\r\nHost: 198.51.100.20\r\nContent-Length: " + + strconv.Itoa(len(body)) + "\r\n\r\n" + // Head and body in ONE write, so the reader holds body bytes when the head ends. + go func() { + _, _ = hostEnd.Write(append([]byte(head), body...)) + }() + // The lane rebuilds a plain-HTTP head (egress_http.go): this request comes + // out the same, with `Connection: close` added as its last header. + want := head[:len(head)-len("\r\n")] + "Connection: close\r\n\r\n" + got := make([]byte, len(want)+len(body)) + _ = upFar.SetReadDeadline(time.Now().Add(20 * time.Second)) + if _, err := io.ReadFull(upFar, got); err != nil { + t.Fatalf("upstream got %v", err) + } + if string(got[:len(want)]) != want || !bytes.Equal(got[len(want):], body) { + t.Fatalf("upstream did not receive the head (%q) and the %d-byte body intact", + got[:len(want)], len(body)) + } + _ = hostEnd.Close() + _ = upFar.Close() + <-done +} diff --git a/client-go/local_host_linux_test.go b/client-go/local_host_linux_test.go index c49bce7..72de3c9 100644 --- a/client-go/local_host_linux_test.go +++ b/client-go/local_host_linux_test.go @@ -493,7 +493,7 @@ func startLaneHostOn(t *testing.T, raw net.Listener, cert tls.Certificate, clien } } } - for _, p := range []string{"/bastion-harness", "/bastion-mgmt", "/bastion-egress"} { + for _, p := range []string{"/bastion-harness", "/bastion-mgmt", "/bastion-egress", "/bastion-serve"} { mux.HandleFunc(p, ws) } srv := &http.Server{Handler: mux, ErrorLog: log.New(io.Discard, "", 0)} @@ -622,7 +622,8 @@ func TestTheUnixLaneCarriesTheBearerAndTheSameProtocol(t *testing.T) { h.mu.Unlock() if maxPark != harnessWorkerCount || count["/bastion-harness"] < harnessWorkerCount || count["/bastion-harness session"] != sessionWorkerCount || - count["/bastion-mgmt"] != mgmtWorkerCount || count["/bastion-egress"] != egressWorkerCount { + count["/bastion-mgmt"] != mgmtWorkerCount || count["/bastion-egress"] != egressWorkerCount || + count["/bastion-serve"] != serveWorkerCount { t.Fatalf("the worker pools over the socket: %v, parked at once %d", count, maxPark) } select { diff --git a/client-go/local_host_tcp_linux_test.go b/client-go/local_host_tcp_linux_test.go index b83f99b..787f635 100644 --- a/client-go/local_host_tcp_linux_test.go +++ b/client-go/local_host_tcp_linux_test.go @@ -417,7 +417,8 @@ func TestTheTCPLaneCarriesTheCertTheBearerAndTheSameProtocol(t *testing.T) { h.mu.Unlock() if maxPark != harnessWorkerCount || count["/bastion-harness"] < harnessWorkerCount || count["/bastion-harness session"] != sessionWorkerCount || - count["/bastion-mgmt"] != mgmtWorkerCount || count["/bastion-egress"] != egressWorkerCount { + count["/bastion-mgmt"] != mgmtWorkerCount || count["/bastion-egress"] != egressWorkerCount || + count["/bastion-serve"] != serveWorkerCount { t.Fatalf("the worker pools over the port: %v, parked at once %d", count, maxPark) } for _, n := range sni { diff --git a/client-go/mirror.go b/client-go/mirror.go index 2e3482b..3915766 100644 --- a/client-go/mirror.go +++ b/client-go/mirror.go @@ -491,6 +491,12 @@ func ensureMirrorNet(rt, name string) (string, string, error) { // container and a BASE_BACKUP of the user's memory. Nothing here is defence in // depth; this is the depth. // +// mirrorForwardMaxConns bounds the connections the forwarder carries at once. +// The replica holds at most two (`pg_basebackup -X stream` while it seeds; one +// walreceiver after), so four times that leaves room for a diagnostic session +// and a reconnect that overlaps the connection it replaces. +const mirrorForwardMaxConns = 8 + // One WebSocket per accepted connection, because `pg_basebackup -X stream` // holds two replication connections at once and multiplexing them would mean // inventing a framing layer for no gain. @@ -507,6 +513,11 @@ func serveMirrorForward(ctx context.Context, gw, subnet, wsURL string, allowed = n } } + // Connections carried at once, across both listeners. Loopback is let in + // for diagnostics, so any local user can connect, and each connection is a + // WebSocket to the host and two pumps here; past the cap one is closed at + // once and its client tries again. + slots := make(chan struct{}, mirrorForwardMaxConns) for _, addr := range []string{ net.JoinHostPort(gw, strconv.Itoa(mirrorForwardPort)), net.JoinHostPort("127.0.0.1", strconv.Itoa(mirrorForwardPort)), @@ -527,7 +538,15 @@ func serveMirrorForward(ctx context.Context, gw, subnet, wsURL string, _ = c.Close() continue } - go pipeMirrorConn(ctx, c, dialer, wsURL, bearer) + select { + case slots <- struct{}{}: + go func() { + defer func() { <-slots }() + pipeMirrorConn(ctx, c, dialer, wsURL, bearer) + }() + default: + _ = c.Close() + } } }(ln) } @@ -578,15 +597,17 @@ func pipeMirrorConn(ctx context.Context, local net.Conn, dialer *websocket.Diale done <- struct{}{} }() go func() { + // Each host frame is streamed into the replica's socket as it arrives, + // never held whole: the host picks a frame's length (lane_bounds.go). for { - mt, data, err := ws.ReadMessage() + mt, r, err := ws.NextReader() if err != nil { break } - if mt != websocket.BinaryMessage || len(data) == 0 { - continue + if mt != websocket.BinaryMessage { + continue // NextReader drops what is left of it } - if _, werr := local.Write(data); werr != nil { + if _, err := io.Copy(local, r); err != nil { break } } diff --git a/client-go/modelscope_test.go b/client-go/modelscope_test.go index 716abd6..98ca9c7 100644 --- a/client-go/modelscope_test.go +++ b/client-go/modelscope_test.go @@ -19,7 +19,9 @@ import ( "net/http" "net/http/httptest" "os" + "slices" "strings" + "sync" "testing" "time" ) @@ -186,17 +188,29 @@ func TestFilterVerdictPolicy(t *testing.T) { // fakeOllama answers the handful of routes the filter cares about and records // what actually reached it — the point of most of these assertions is that a -// refused request never arrives at all. +// refused request never arrives at all. Its handler runs on the server's +// goroutines, beside the test's (and a pairing's beat probes it on its own), +// so the record is read through requests(), under the lock. type fakeOllama struct { srv *httptest.Server + mu sync.Mutex seen []string } +// requests is a copy of what reached the fake so far. +func (f *fakeOllama) requests() []string { + f.mu.Lock() + defer f.mu.Unlock() + return slices.Clone(f.seen) +} + func newFakeOllama(t *testing.T) *fakeOllama { t.Helper() f := &fakeOllama{} f.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() f.seen = append(f.seen, r.Method+" "+r.URL.Path) + f.mu.Unlock() switch r.URL.Path { case "/api/tags": _, _ = w.Write([]byte(`{"models":[{"name":"qwen4-embed:latest","size":42},` + @@ -264,8 +278,8 @@ func TestFilteredForwardRefusesAnotherModelAndNeverForwardsIt(t *testing.T) { if b := bodyOf(t, resp); !strings.Contains(b, "llama3:70b") || !strings.Contains(b, "qwen4-embed") { t.Fatalf("the refusal must name what was asked for and what is shared: %s", b) } - if len(up.seen) != 0 { - t.Fatalf("a refused request reached the upstream: %v", up.seen) + if len(up.requests()) != 0 { + t.Fatalf("a refused request reached the upstream: %v", up.requests()) } resp = roundTrip(t, addr, laneOllama, tMember, "POST", "/api/embed", @@ -273,8 +287,8 @@ func TestFilteredForwardRefusesAnotherModelAndNeverForwardsIt(t *testing.T) { if resp.StatusCode != http.StatusOK { t.Fatalf("the shared model must work: %d", resp.StatusCode) } - if len(up.seen) != 1 || up.seen[0] != "POST /api/embed" { - t.Fatalf("the allowed request must reach the upstream once: %v", up.seen) + if len(up.requests()) != 1 || up.requests()[0] != "POST /api/embed" { + t.Fatalf("the allowed request must reach the upstream once: %v", up.requests()) } } @@ -295,8 +309,8 @@ func TestFilteredForwardRefusesTheStoreMutatingHalf(t *testing.T) { t.Fatalf("%s: status %d, want 403", p, resp.StatusCode) } } - if len(up.seen) != 0 { - t.Fatalf("a store-mutating request reached ollama: %v", up.seen) + if len(up.requests()) != 0 { + t.Fatalf("a store-mutating request reached ollama: %v", up.requests()) } // ...and using a model is untouched by that refusal. if resp := roundTrip(t, addr, laneOllama, tMember, "POST", "/api/embed", @@ -344,8 +358,8 @@ func TestAnchoredBoxRefusesModelChangesThroughTheForwardForTheAnchor(t *testing. t.Fatalf("%s for the anchor: status %d, want 403", p, resp.StatusCode) } } - if len(up.seen) != 0 { - t.Fatalf("a store-mutating request from the anchor reached ollama: %v", up.seen) + if len(up.requests()) != 0 { + t.Fatalf("a store-mutating request from the anchor reached ollama: %v", up.requests()) } if resp := roundTrip(t, addr, laneOllama, tAnchor, "POST", "/api/embed", `{"model":"anything"}`); resp.StatusCode != http.StatusOK { t.Fatalf("the anchor still USES models: %d", resp.StatusCode) @@ -581,8 +595,8 @@ func TestOversizedBodyIsRefusedNotForwarded(t *testing.T) { if resp.StatusCode != http.StatusForbidden { t.Fatalf("a request naming no model must be refused: %d", resp.StatusCode) } - if len(up.seen) != 0 { - t.Fatalf("it reached the upstream anyway: %v", up.seen) + if len(up.requests()) != 0 { + t.Fatalf("it reached the upstream anyway: %v", up.requests()) } } @@ -612,8 +626,8 @@ func TestAScopedPairingCannotRenameAModelIntoItsScope(t *testing.T) { t.Fatalf("%s: the refusal must say why: %s", p, b) } } - if len(up.seen) != 0 { - t.Fatalf("a rename reached ollama: %v", up.seen) + if len(up.requests()) != 0 { + t.Fatalf("a rename reached ollama: %v", up.requests()) } } @@ -803,8 +817,8 @@ func TestAnEndlessHeaderIsCutOff(t *testing.T) { t.Fatal("an endless header was read without limit") } client.Close() - if len(up.seen) != 0 { - t.Fatalf("it reached the upstream: %v", up.seen) + if len(up.requests()) != 0 { + t.Fatalf("it reached the upstream: %v", up.requests()) } } diff --git a/client-go/modelswap.go b/client-go/modelswap.go index 661c013..c97f9e6 100644 --- a/client-go/modelswap.go +++ b/client-go/modelswap.go @@ -419,38 +419,53 @@ func placedByThisHost(root, model string) bool { // ── talking to the model daemon ──────────────────────────────────────────── var ( - modelDaemonMu sync.Mutex - modelDaemonUp bool - modelDaemonAt time.Time + modelDaemonMu sync.Mutex + modelDaemonUp bool + modelDaemonAt time.Time + modelDaemonAddr string // the address the cached answer is for ) // modelDaemonReachable probes the SAME address the rest of this binary uses // (localOllamaAddr, run.go:714-719) so the box can never advertise management // of a daemon the forwards do not point at. -func modelDaemonReachable() bool { +func modelDaemonReachable() bool { return modelDaemonReachableAt(localOllamaAddr()) } + +// modelDaemonReachableAt is modelDaemonReachable's probe of one address. The +// cached answer is for the address it was asked of, so a pairing whose +// address is the vetted one (modelDaemonTarget) never reads another's answer. +func modelDaemonReachableAt(addr string) bool { modelDaemonMu.Lock() defer modelDaemonMu.Unlock() - if !modelDaemonAt.IsZero() && time.Since(modelDaemonAt) < modelDaemonCacheFor { + if !modelDaemonAt.IsZero() && modelDaemonAddr == addr && time.Since(modelDaemonAt) < modelDaemonCacheFor { return modelDaemonUp } cl := &http.Client{Timeout: 3 * time.Second} - resp, err := cl.Get("http://" + localOllamaAddr() + "/api/tags") + resp, err := cl.Get("http://" + addr + "/api/tags") up := false if err == nil { up = resp.StatusCode == http.StatusOK resp.Body.Close() } - modelDaemonUp, modelDaemonAt = up, time.Now() + modelDaemonUp, modelDaemonAt, modelDaemonAddr = up, time.Now(), addr return up } // resetModelDaemonCache exists for the tests. func resetModelDaemonCache() { modelDaemonMu.Lock() - modelDaemonUp, modelDaemonAt = false, time.Time{} + modelDaemonUp, modelDaemonAt, modelDaemonAddr = false, time.Time{}, "" modelDaemonMu.Unlock() } +// modelSelf is the pairing a model:* job came from, or "" when the job was +// driven without one (modelDaemonTarget then keeps OLLAMA_ADDR, as before). +func modelSelf(ws *websocket.Conn) string { + if a := mgmtAccountOf(ws); a != nil { + return a.self + } + return "" +} + // modelSwapCapable answers the heartbeat's question: could this box have a // model placed on it BY THIS PAIRING. Two conditions, and the caps one is // first so a pairing outside the family costs no probe — the rule the @@ -459,13 +474,19 @@ func modelSwapCapable(self string) bool { if capsAllow(self, modelGateOp) != "" { return false } - return modelDaemonReachable() + // A Host with no platform is offered management only of a model server on + // this machine, and a refused one is not probed at all (modelDaemonTarget). + addr, why := modelDaemonTarget(self) + if why != "" { + return false + } + return modelDaemonReachableAt(addr) } // modelPostJSON posts one JSON body and returns (status, body). The body is // read whatever the status is: a streaming ollama reports a FAILED pull inside // a 200 (measured, see the header), so the caller needs both. -func modelPostJSON(method, path string, payload any, timeout time.Duration) (int, []byte, error) { +func modelPostJSON(addr, method, path string, payload any, timeout time.Duration) (int, []byte, error) { var rdr io.Reader if payload != nil { b, err := json.Marshal(payload) @@ -474,7 +495,7 @@ func modelPostJSON(method, path string, payload any, timeout time.Duration) (int } rdr = bytes.NewReader(b) } - req, err := http.NewRequest(method, "http://"+localOllamaAddr()+path, rdr) + req, err := http.NewRequest(method, "http://"+addr+path, rdr) if err != nil { return 0, nil, err } @@ -528,8 +549,8 @@ func modelCosine(a, b []float64) float64 { return dot / (math.Sqrt(na) * math.Sqrt(nb)) } -func modelEmbedOnce(model, text string) ([]float64, error) { - status, body, err := modelPostJSON(http.MethodPost, "/api/embed", +func modelEmbedOnce(addr, model, text string) ([]float64, error) { + status, body, err := modelPostJSON(addr, http.MethodPost, "/api/embed", map[string]any{"model": model, "input": []string{text}}, modelProbeTimeout) if err != nil { return nil, err @@ -549,8 +570,8 @@ func modelEmbedOnce(model, text string) ([]float64, error) { return out.Embeddings[0], nil } -func modelGenerateOnce(model, prompt string) (string, error) { - status, body, err := modelPostJSON(http.MethodPost, "/api/generate", +func modelGenerateOnce(addr, model, prompt string) (string, error) { + status, body, err := modelPostJSON(addr, http.MethodPost, "/api/generate", map[string]any{"model": model, "prompt": prompt, "stream": false, "think": false}, modelProbeTimeout) if err != nil { return "", err @@ -573,12 +594,12 @@ func modelGenerateOnce(model, prompt string) (string, error) { // probeEmbedModel proves the model EMBEDS: the same text twice must come back // as one answer, and a different text must move it. Returns the measured width // and "" , or 0 and the reason it is not usable. -func probeEmbedModel(model string) (int, string) { - v1, err := modelEmbedOnce(model, modelProbeTextA) +func probeEmbedModel(addr, model string) (int, string) { + v1, err := modelEmbedOnce(addr, model, modelProbeTextA) if err != nil { return 0, "it was downloaded but does not answer as an embedding model: " + err.Error() } - v2, err := modelEmbedOnce(model, modelProbeTextA) + v2, err := modelEmbedOnce(addr, model, modelProbeTextA) if err != nil { return 0, "it answered once and then failed: " + err.Error() } @@ -587,7 +608,7 @@ func probeEmbedModel(model string) (int, string) { return 0, fmt.Sprintf("it gave two different answers to the SAME text (similarity %.4f) — "+ "nothing routed at it would be reproducible", same) } - v3, err := modelEmbedOnce(model, modelProbeTextB) + v3, err := modelEmbedOnce(addr, model, modelProbeTextB) if err != nil { return 0, "the control text failed: " + err.Error() } @@ -604,15 +625,15 @@ func probeEmbedModel(model string) (int, string) { // probeGenerateModel proves the model GENERATES: a non-empty answer, and a // different prompt must change it (the same constant-answer trap). -func probeGenerateModel(model string) string { - a, err := modelGenerateOnce(model, modelProbePromptA) +func probeGenerateModel(addr, model string) string { + a, err := modelGenerateOnce(addr, model, modelProbePromptA) if err != nil { return "it was downloaded but does not answer as a text model: " + err.Error() } if strings.TrimSpace(a) == "" { return "it answered with nothing at all" } - b, err := modelGenerateOnce(model, modelProbePromptB) + b, err := modelGenerateOnce(addr, model, modelProbePromptB) if err != nil { return "it answered once and then failed: " + err.Error() } @@ -704,7 +725,12 @@ func modelReply(ws *websocket.Conn, v any) { // here — the two together are what makes an honest proposal possible: the host // can see a model is present without being able to claim it may remove it. func handleModelList(ws *websocket.Conn, root string, _ []string) { - status, body, err := modelPostJSON(http.MethodGet, "/api/tags", nil, modelListTimeout) + addr, why := modelDaemonTarget(modelSelf(ws)) + if why != "" { + modelFail(ws, "model:list: "+why) + return + } + status, body, err := modelPostJSON(addr, http.MethodGet, "/api/tags", nil, modelListTimeout) if err != nil { modelFail(ws, "model:list: no model daemon answering on this machine: "+err.Error()) return @@ -974,12 +1000,12 @@ func modelPullRunningNote(j *modelPullJob) string { // runBackupFetch updates job.Got inside its read loop (backupfetch.go:309-311): // progress that only exists when the transfer ends is not progress. nil is // allowed — the direct-call tests pass it. -func modelPullStreaming(model string, maxBytes int64, note func(want, got int64)) (int64, error) { +func modelPullStreaming(addr, model string, maxBytes int64, note func(want, got int64)) (int64, error) { body, err := json.Marshal(map[string]any{"model": model, "stream": true}) if err != nil { return 0, err } - httpReq, err := http.NewRequest(http.MethodPost, "http://"+localOllamaAddr()+"/api/pull", + httpReq, err := http.NewRequest(http.MethodPost, "http://"+addr+"/api/pull", bytes.NewReader(body)) if err != nil { return 0, err @@ -1109,6 +1135,17 @@ func modelPullStart(ws *websocket.Conn, root string, req modelReq) { return } + // WHERE THE PULL GOES, decided once and held for the download and its + // probe: a Host with no platform pulls only onto a model server on this + // machine (modelDaemonTarget). Refused before the approval is read, so a + // pull that cannot go here costs the person nothing. For such a pairing a + // name in OLLAMA_ADDR is looked up here; nothing is dialled. + addr, why := modelDaemonTarget(modelSelf(ws)) + if why != "" { + modelRefuse(ws, req.Model, why) + return + } + // THE APPROVAL GATE. Nothing above this line has touched the network; nothing // below it runs until the user has said yes to THIS proposal on this machine. // The async split moved the download off this frame and deliberately did NOT @@ -1152,7 +1189,7 @@ func modelPullStart(ws *websocket.Conn, root string, req modelReq) { snap := *job modelPullJobsMu.Unlock() - go runModelPull(job, root, req) + go runModelPull(job, root, req, addr) modelReply(ws, snap) } @@ -1167,7 +1204,7 @@ func modelPullStart(ws *websocket.Conn, root string, req modelReq) { // bytes on this disk", which is true the moment the download succeeds. A model // that downloaded but failed its probe must stay droppable, or a bad pull would // leave dead weight on the user's machine that nothing is allowed to remove. -func runModelPull(job *modelPullJob, root string, req modelReq) { +func runModelPull(job *modelPullJob, root string, req modelReq, addr string) { set := func(f func(j *modelPullJob)) { modelPullJobsMu.Lock() f(job) @@ -1201,7 +1238,7 @@ func runModelPull(job *modelPullJob, root string, req modelReq) { // The 200-with-an-error-in-the-body case that stream:false was chosen for is // handled either way: modelBodyError reads EVERY line, so a failure on the // last line of a stream is caught exactly as before. - total, err := modelPullStreaming(req.Model, req.MaxBytes, + total, err := modelPullStreaming(addr, req.Model, req.MaxBytes, func(want, got int64) { set(func(j *modelPullJob) { j.Want, j.Got = want, got }) }) if err != nil { finish("failed", func(j *modelPullJob) { @@ -1242,9 +1279,9 @@ func runModelPull(job *modelPullJob, root string, req modelReq) { var dim int var why string if req.Role == "embed" { - dim, why = probeEmbedModel(req.Model) + dim, why = probeEmbedModel(addr, req.Model) } else { - why = probeGenerateModel(req.Model) + why = probeGenerateModel(addr, req.Model) } if why != "" { finish("done", func(j *modelPullJob) { @@ -1284,7 +1321,12 @@ func handleModelDrop(ws *websocket.Conn, root string, args []string) { "not remove it. Only a model this host pulled here is its to drop.") return } - status, body, err := modelPostJSON(http.MethodDelete, "/api/delete", + addr, why := modelDaemonTarget(modelSelf(ws)) + if why != "" { + modelFail(ws, "model:drop: "+why) + return + } + status, body, err := modelPostJSON(addr, http.MethodDelete, "/api/delete", map[string]any{"model": req.Model}, modelListTimeout) if err != nil { modelFail(ws, "model:drop: the model daemon did not answer: "+err.Error()) diff --git a/client-go/ownaddr_inside_linux_test.go b/client-go/ownaddr_inside_linux_test.go index 02c5541..e71cc88 100644 --- a/client-go/ownaddr_inside_linux_test.go +++ b/client-go/ownaddr_inside_linux_test.go @@ -286,7 +286,8 @@ func TestNetworksInsideThisMachineAreThisMachine(t *testing.T) { } // The local egress lane on an unanchored box: an inside address, by address - // and by name, is refused as this machine; the LAN passes. + // and by name, is refused as this machine; the LAN is refused as the network + // around it; the public internet passes. t.Setenv("XDG_CONFIG_HOME", t.TempDir()) if currentAnchor() != nil || boxMode() != modeLocal { t.Fatal("the namespaced run must be a local, unanchored box") @@ -301,12 +302,16 @@ func TestNetworksInsideThisMachineAreThisMachine(t *testing.T) { t.Errorf("local egress: %s is on a network inside this machine and must be refused as this machine, got %v %q", hp, got, why) } } - for hp, want := range map[string]string{"10.77.0.9:445": "10.77.0.9:445", "192.168.50.9:631": "192.168.50.9:631", - "nas.example.test:445": "10.77.0.9:445", "198.51.100.9:443": "198.51.100.9:443", - "[2001:db8:ff::9]:443": "[2001:db8:ff::9]:443", "192.168.60.9:445": "192.168.60.9:445", - "10.88.0.9:80": "10.88.0.9:80", "10.89.0.9:80": "10.89.0.9:80", "[2001:db8:88::9]:80": "[2001:db8:88::9]:80"} { + for _, hp := range []string{"10.77.0.9:445", "192.168.50.9:631", "nas.example.test:445", "192.168.60.9:445", + "10.88.0.9:80", "10.89.0.9:80"} { + if got, why := egressVettedAddrs(ctx, hp); !strings.Contains(why, "approved lanes") { + t.Errorf("local egress: %s is the LAN and must be refused as the network around this machine, got %v %q", hp, got, why) + } + } + for hp, want := range map[string]string{"198.51.100.9:443": "198.51.100.9:443", + "[2001:db8:ff::9]:443": "[2001:db8:ff::9]:443", "[2001:db8:88::9]:80": "[2001:db8:88::9]:80"} { if got, why := egressVettedAddrs(ctx, hp); why != "" || len(got) != 1 || got[0] != want { - t.Errorf("local egress: %s is the LAN and passes as %s: %v %q", hp, want, got, why) + t.Errorf("local egress: %s leaves this machine for the internet and passes as %s: %v %q", hp, want, got, why) } } // A coder run's proxy refuses the public range on an inside network too diff --git a/client-go/ownaddr_test.go b/client-go/ownaddr_test.go index 3a3d086..ad8ba58 100644 --- a/client-go/ownaddr_test.go +++ b/client-go/ownaddr_test.go @@ -9,6 +9,7 @@ import ( "errors" "net" "strings" + "sync" "testing" ) @@ -33,8 +34,13 @@ func stubOwnAddrs(t *testing.T, ifaces []string, route func(net.IP) (routeKind, } asked := &[]string{} if route != nil { + // Locked: a lane's workers vet their jobs at once, each on its own + // goroutine (egress_relay_test.go). + var mu sync.Mutex ownRouteLookup = func(ip net.IP) (routeKind, error) { + mu.Lock() *asked = append(*asked, ip.String()) + mu.Unlock() return route(ip) } } else { diff --git a/client-go/pii.go b/client-go/pii.go index 049cdc8..72c84c1 100644 --- a/client-go/pii.go +++ b/client-go/pii.go @@ -29,7 +29,11 @@ const piiForwardPort = 18081 // The bearer-authed surface is proxied opaquely — this client never // holds the PII_API_KEY. func piiProbe() (bool, bool) { - a := piiTarget() + return piiProbeAt(piiTarget()) +} + +// piiProbeAt is piiProbe's probe of one address. +func piiProbeAt(a string) (bool, bool) { if a == "" { return false, false } diff --git a/client-go/rootexec.go b/client-go/rootexec.go index 61d13c5..c289030 100644 --- a/client-go/rootexec.go +++ b/client-go/rootexec.go @@ -686,11 +686,8 @@ func handleRootExec(ws *websocket.Conn, spec sessionSpec, self string, env sessi closed := make(chan struct{}) go func() { defer close(closed) - for { - if _, _, err := ws.ReadMessage(); err != nil { - return - } - } + // Each frame is dropped as it arrives (lane_bounds.go), never held whole. + _ = drainLane(ws) }() done := make(chan int, 1) go func() { done <- proc.Wait() }() diff --git a/client-go/run.go b/client-go/run.go index b30d9e9..f12a901 100644 --- a/client-go/run.go +++ b/client-go/run.go @@ -617,8 +617,11 @@ func runPairing(ctx context.Context, cfg *Config) error { req.Header.Set("Content-Type", "application/json") req.Header.Set("Authorization", "Bearer "+cfg.Bearer) if resp, err := upstream.Do(req); err == nil { - body, _ := io.ReadAll(resp.Body) + body, ok := readHeartbeatReply(resp) resp.Body.Close() + if !ok { + return // a reply past its bound is no answer (lane_bounds.go) + } if resp.StatusCode < 300 { // last_ok per pairing (anchor.go): the anchor rotation's // silence path reads this — 14 days with no 2xx from the @@ -727,14 +730,19 @@ func unixLaneDialer(cfg *Config) (func(ctx context.Context) (net.Conn, error), * // parked harness workers, sessions, git, the heartbeat and what its reply // directs. Absent, because they listen on the tunnel's stack or move the // tunnel: the ollama, LLM and PII forwards, the service slots, and endpoint -// rotation and failover. A unix pairing's coder run has its control plane on -// a socket in RunsDir (coder_socket.go), not a tunnel port; a tcp pairing has -// no coder runs. +// rotation and failover. The model servers the forwards reach are served +// over the serve lane instead (servelane.go), by service name. A unix +// pairing's coder run has its control plane on a socket in RunsDir +// (coder_socket.go), not a tunnel port; a tcp pairing has no coder runs. The +// heartbeat advertises only what runs here (agents.go, beatLanes). func runUnixPairing(ctx context.Context, cfg *Config) error { logf := func(format string, a ...any) { fmt.Printf("["+cfg.HostName+"] "+format, a...) } if !cfg.active() { return errPairingNotActive } + // Before any lane: model management for this pairing (modelDaemonTarget) + // reaches only a model server on this machine, as the serve lane does. + defer markLocalPairing(cfg.HostName)() dialHost, laneTLS, err := localLaneDialer(cfg) if err != nil { return err @@ -762,6 +770,13 @@ func runUnixPairing(ctx context.Context, cfg *Config) error { logf("egress workers up — outbound exits THIS machine's internet via %s on %s\n", wsURL, lane) runEgressWorkers(ctx, cfg.HostName, wsURL, laneTLS.Clone(), dialHost, cfg.Bearer, realDialer.DialContext, egressWorkerCount) }() + // The serve lane: the model servers set on this machine, which the tunnel + // forwards would carry, by service name and never by address. + go func() { + wsURL := fmt.Sprintf("wss://%s:8443/bastion-serve", cfg.HostName) + logf("serve workers up — the model servers set on this machine, by name, via %s on %s\n", wsURL, lane) + runServeWorkers(ctx, cfg.HostName, cfg.dcnDir(), wsURL, laneTLS.Clone(), dialHost, cfg.Bearer, serveWorkerCount) + }() go func() { wsURL := fmt.Sprintf("wss://%s:8443/bastion-mgmt", cfg.HostName) if n := sweepStaleScratch(cfg.contextRoot()); n > 0 { @@ -841,8 +856,11 @@ func runUnixPairing(ctx context.Context, cfg *Config) error { if err != nil { return } - body, _ := io.ReadAll(resp.Body) + body, ok := readHeartbeatReply(resp) resp.Body.Close() + if !ok { + return // a reply past its bound is no answer (lane_bounds.go) + } if resp.StatusCode < 300 { recordPairingOK(cfg.HostName) } @@ -890,7 +908,7 @@ func heartbeatPayload(cfg *Config, dev interface{ IpcGet() (string, error) }, ho for k, v := range pairingEnforcing(cfg.HostName) { enforcing[k] = v } - return buildHeartbeatBody(cfg.HostName, hostname, runtime.GOOS, clientVersion, wgPub, cfg.Fingerprint, cfg.BastionTunnelIP, cfg.Endpoint, + return buildHeartbeatBodyOn(beatLanesFor(cfg), cfg.HostName, hostname, runtime.GOOS, clientVersion, wgPub, cfg.Fingerprint, cfg.BastionTunnelIP, cfg.Endpoint, rotation.isUnsaved(cfg), currentMirrorReport(cfg.HostName), enforcing, // Read fresh on every beat: a host paired a minute ago must // appear as a peer on the NEXT beat, not after a restart. @@ -1124,6 +1142,13 @@ func ollamaModels(tunnelIP string) (bool, []string) { addrs = append(addrs, net.JoinHostPort(tunnelIP, "11434")) } addrs = append(addrs, "127.0.0.1:11434") + return ollamaModelsAt(addrs) +} + +// ollamaModelsAt is ollamaModels' probe of the addresses given, in order. A +// local pairing's beat asks it about the one address its serve lane dials +// (servelane.go), so what the beat says is served is what the lane reaches. +func ollamaModelsAt(addrs []string) (bool, []string) { cl := &http.Client{Timeout: 3 * time.Second} for _, a := range addrs { resp, err := cl.Get("http://" + a + "/api/tags") @@ -1172,7 +1197,11 @@ const llmForwardPort = 18080 // and the operator setting the env IS the opt-in. Same 200-is-alive rule as // /api/tags: a live server with an unparseable list still counts. func openAICompatModels() (bool, []string) { - a := llmTarget() + return openAICompatModelsAt(llmTarget()) +} + +// openAICompatModelsAt is openAICompatModels' probe of one address. +func openAICompatModelsAt(a string) (bool, []string) { if a == "" { return false, nil } diff --git a/client-go/servelane.go b/client-go/servelane.go new file mode 100644 index 0000000..bab9ab4 --- /dev/null +++ b/client-go/servelane.go @@ -0,0 +1,721 @@ +package main + +// The serve lane (/bastion-serve): how a Host with no platform reaches the +// model servers on this machine. +// +// Over the tunnel the Host dials a forward on the tunnel's stack +// (serveOllamaForward and serveLLMForward in run.go, servePIIForward in +// pii.go). A local pairing (a Host on this machine's socket, or in a VM on +// it over TCP) has no tunnel, so those forwards are absent. This lane turns +// the direction round, as egress does: this machine parks worker WebSockets +// on the Host, and the Host hands each one connection. +// +// THE HOST NAMES A SERVICE, NEVER AN ADDRESS. Its first frame on a worker is +// 'S' + {"service":"ollama"}. The address that service reaches is the one set +// on this machine (OLLAMA_ADDR, PRIVATAE_LLM_ADDR, PRIVATAE_PII_ADDR), read by +// the functions the tunnel forwards read, and it must be this machine: a +// server on another computer on the LAN is refused. Nothing the Host sends +// can name, change or widen a target. The bastion answers 'A' and carries +// the connection, or 'E' + a reason and closes, having dialled nothing. +// +// After 'A' the worker is a wsConn ('D' + bytes, 'E' for the end, both ways) +// handed to the forwards' own code: ollama and llm through proxyInference's +// decision and filter (the model scope, /api/pull and /api/delete refused +// without `models`, the list rewrite, its size bounds), pii through the PII +// scope and a pass-through. The `inference` family is checked on every +// connection, as inferenceRefuser does at a forward's accept. +// +// Bounds, per pairing: 16 streams in flight across the services, the 17th +// answered 'E' busy; a stream with no byte either way for 10 minutes is +// closed, on every service; a response's length is not capped, because a +// long generation is the person's own traffic. + +import ( + "context" + "crypto/tls" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/gorilla/websocket" +) + +const ( + // serveWorkerCount workers stay parked per pairing. A replacement is + // dialled as soon as one takes a job, so a long stream never holds up the + // next request. + serveWorkerCount = 2 + // serveMaxInFlight is the streams one pairing may hold open here at once. + // It protects this machine and must never be what slows the person down: + // ollama runs one request per model at a time and queues 512 more, and the + // Host's own client holds at most 10 connections per server, so a request + // waits in the server's queue before it waits here. + serveMaxInFlight = 16 + // serveHelloMax bounds the 'S' frame. A service name and its JSON fit in a + // fraction of it; anything longer is not a request this lane reads. + serveHelloMax = 1024 + // serveTargetLookup bounds resolving a target set by name (localhost:11434). + serveTargetLookup = 2 * time.Second +) + +// serveIdleTimeout closes a stream that has carried no byte either way for +// this long. The filter's own read bound (filteredIdleTimeout) covers a request +// being read; this covers the rest: a server that answers headers and then +// nothing, and a pass-through, which has no bound of its own. A var so a test +// can shorten it. +var serveIdleTimeout = filteredIdleTimeout + +// serveService is one service the lane carries: its name on the wire, where +// its target is read, which forward's code serves it. +type serveService struct { + name string + envVar string // named in a refusal, so the person knows what to set + target func() string + filter bool // through proxyInference's decision and filter + lane inferenceLane // the filter's lane, when filter +} + +// serveServices is every service the lane knows, in the order the beat lists +// them. A name not in it is refused before anything is read or dialled. +var serveServices = []serveService{ + {name: "ollama", envVar: "OLLAMA_ADDR", target: localOllamaAddr, filter: true, lane: laneOllama}, + {name: "llm", envVar: "PRIVATAE_LLM_ADDR", target: llmTarget, filter: true, lane: laneLLM}, + {name: "pii", envVar: "PRIVATAE_PII_ADDR", target: piiTarget}, +} + +func serveServiceNamed(name string) (serveService, bool) { + for _, s := range serveServices { + if s.name == name { + return s, true + } + } + return serveService{}, false +} + +// serveLookupIP resolves a target set by name. A seam so a test can answer. +var serveLookupIP = func(ctx context.Context, host string) ([]net.IPAddr, error) { + return net.DefaultResolver.LookupIPAddr(ctx, host) +} + +// serveVetTarget checks the address set on this machine for a service and +// returns the addresses to dial, or the reason it is refused. Every address +// the target names must be this machine: loopback, an address on its +// interfaces, or one the kernel delivers here (ownaddr.go), which takes in a +// server in a container on a network made on this machine. A server on +// another computer is refused: a device the bastion does not run on is the +// device lane's problem, solved there with pinning. 0.0.0.0 and :: are listen +// addresses, and dialling one reaches this machine, so each is dialled as +// loopback of its family. The addresses dialled are the ones checked, never +// the name again. +// +// Every address checked is returned, in the resolver's order, and the dial +// tries them in turn (serveDial, servePick), as the OS dialler does with a +// name: localhost answers ::1 first on most systems, and ollama listens on +// 127.0.0.1 only by default, so dialling the first address alone fails where +// a plain dial of the name reaches it. +func serveVetTarget(target string) ([]string, string) { + host, port, err := net.SplitHostPort(target) + if err != nil { + return nil, fmt.Sprintf("%q is not host:port", target) + } + if n, err := strconv.Atoi(port); err != nil || n < 1 || n > 65535 { + return nil, fmt.Sprintf("%q has no port between 1 and 65535", target) + } + var ips []net.IP + if ip := net.ParseIP(host); ip != nil { + ips = []net.IP{ip} + } else { + ctx, cancel := context.WithTimeout(context.Background(), serveTargetLookup) + as, err := serveLookupIP(ctx, host) + cancel() + if err != nil || len(as) == 0 { + return nil, fmt.Sprintf("%q could not be resolved on this machine", host) + } + for _, a := range as { + ips = append(ips, a.IP) + } + } + for i, ip := range ips { + switch { + case ip.IsUnspecified() && ip.To4() != nil: + ips[i] = net.IPv4(127, 0, 0, 1) + case ip.IsUnspecified(): + ips[i] = net.IPv6loopback + } + } + for _, ip := range ips { + if ip.IsLoopback() { + continue + } + own, err := ownDestination([]net.IP{ip}) + if err != nil { + return nil, fmt.Sprintf("this machine could not tell whether %s is its own address (%v)", ip, err) + } + if !own { + return nil, fmt.Sprintf("%s is not this machine; the lane reaches only servers on the computer it runs on", ip) + } + } + out := make([]string, 0, len(ips)) + for _, ip := range ips { + a := net.JoinHostPort(ip.String(), port) + if !slices.Contains(out, a) { + out = append(out, a) + } + } + return out, "" +} + +// serveDialTimeout bounds one dial of a service's addresses, all of them +// together, as the forwards' dial (dialUpstream) bounds its one address. +const serveDialTimeout = 5 * time.Second + +// serveDial dials the checked addresses in turn (dialEach: the next starts +// when one fails, or has not answered within the Happy Eyeballs delay) and +// returns the first that answers, with the address it was dialled at. Only +// these addresses are dialled. +func serveDial(addrs []string, timeout time.Duration) (net.Conn, string, error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + var mu sync.Mutex + at := map[net.Conn]string{} + d := &net.Dialer{} + c, err := dialEach(ctx, func(ctx context.Context, network, a string) (net.Conn, error) { + c, err := d.DialContext(ctx, network, a) + if err == nil { + mu.Lock() + at[c] = a + mu.Unlock() + } + return c, err + }, addrs) + if err != nil { + return nil, "", err + } + mu.Lock() + defer mu.Unlock() + return c, at[c], nil +} + +// servePick is the one address an HTTP client here is given for a target +// whose check returned several (the beat's probes, model management): the +// first that accepts a connection, tried in turn as serveDial does, or the +// first checked when none does, so the request fails as it would have. One +// address is returned as it is, with nothing dialled. +func servePick(addrs []string) string { + if len(addrs) == 1 { + return addrs[0] + } + c, a, err := serveDial(addrs, serveTargetLookup) + if err != nil { + return addrs[0] + } + _ = c.Close() + return a +} + +// serveTargetFor is the addresses a service dials here, or why it does not. +// The beat and the lane both ask it, so what the beat says is served is what +// the lane reaches (the rule env.go states for the forwards). +func serveTargetFor(s serveService) ([]string, string) { + t := s.target() + if t == "" { + return nil, "unavailable: no " + s.name + " server is set on this machine (" + s.envVar + ")" + } + addrs, why := serveVetTarget(t) + if why != "" { + return nil, "refused: the " + s.name + " server set on this machine (" + s.envVar + ") is refused: " + why + } + return addrs, "" +} + +// serveInFlight counts each pairing's open streams, keyed by host name, so a +// pairing restarted while its streams still run counts them. +var serveInFlight sync.Map // string -> *atomic.Int32 + +func serveAcquire(self string) bool { + v, _ := serveInFlight.LoadOrStore(self, new(atomic.Int32)) + n := v.(*atomic.Int32) + for { + cur := n.Load() + if cur >= serveMaxInFlight { + return false + } + if n.CompareAndSwap(cur, cur+1) { + return true + } + } +} + +func serveRelease(self string) { + if v, ok := serveInFlight.Load(self); ok { + v.(*atomic.Int32).Add(-1) + } +} + +// serveHello is the 'S' frame's JSON. Fields this build does not know are +// ignored, so a Host may add one without breaking an older bastion. +type serveHello struct { + Service string `json:"service"` +} + +// serveVerdict decides one job before anything is dialled: the service, the +// family, the PII scope, the target. It returns the service and the addresses, +// or the reason the job is refused. +func serveVerdict(self, name string) (serveService, []string, string) { + s, ok := serveServiceNamed(name) + if !ok { + return s, nil, "refused: " + strconv.Quote(truncate(name, 64)) + " is not a service this lane carries " + + "(ollama, llm or pii). The lane takes a service's name and reaches the server set on this machine for it; " + + "it never takes an address." + } + if why := capsAllow(self, "inference"); why != "" { + return s, nil, "refused: " + why + } + if s.name == "pii" && !pairingModelScope(self).allowsPII() { + return s, nil, "refused: this pairing is scoped to named models and the PII sterilizer was not one of them " + + "(approve with 'pii' in the model list to share it)" + } + addrs, why := serveTargetFor(s) + if why != "" { + return s, nil, why + } + return s, addrs, "" +} + +// serveJob serves the one job a parked worker is given. taken is called once +// the job is accepted (its stream counted, before 'A'), so the pool parks a +// replacement while this one works. It reports whether a job was accepted. A +// refused job (malformed, refused, busy) reports false, and its worker waits +// before it parks again: the replacement is not dialled at once, so a Host +// that answers every worker with a frame it knows is refused cannot make this +// machine dial, handshake and write an audit line in a loop as fast as it +// answers. +func serveJob(ws *websocket.Conn, self, dcnDir string, taken func()) bool { + mt, r, err := ws.NextReader() + if err != nil { + return false + } + refuse := func(service, why string) { + _ = ws.WriteMessage(websocket.BinaryMessage, append([]byte{'E'}, why...)) + serveAudit(dcnDir, map[string]any{"service": service, "decision": "refused", "reason": why}) + } + raw, err := io.ReadAll(io.LimitReader(r, serveHelloMax+1)) + if err != nil { + return false + } + var hello serveHello + if mt != websocket.BinaryMessage || len(raw) == 0 || raw[0] != 'S' || len(raw) > serveHelloMax || + json.Unmarshal(raw[1:], &hello) != nil { + // Malformed: answered, not audited (as a malformed terminal request is not). + _ = ws.WriteMessage(websocket.BinaryMessage, []byte("Erefused: the first frame on this lane is 'S' and "+ + "{\"service\":\"\"}, at most 1024 bytes")) + return false + } + s, addrs, why := serveVerdict(self, hello.Service) + if why != "" { + refuse(truncate(hello.Service, 64), why) + return false + } + if !serveAcquire(self) { + refuse(s.name, fmt.Sprintf("busy: this pairing already has %d streams open on this machine, "+ + "which is as many as it may hold; try again when one ends", serveMaxInFlight)) + return false + } + defer serveRelease(self) + taken() + if err := ws.WriteMessage(websocket.BinaryMessage, []byte{'A'}); err != nil { + return true + } + w := newServeWatch(serveIdleTimeout) + in := w.wrap(&wsConn{ws: ws}) + var reachedMu sync.Mutex + reached := "" + dial := func() (net.Conn, error) { + out, a, err := serveDial(addrs, serveDialTimeout) + if err != nil { + return nil, err + } + reachedMu.Lock() + reached = a + reachedMu.Unlock() + return w.wrap(out), nil + } + if s.filter { + proxyInferenceVia(in, dial, s.lane, self, servePass) + } else { + servePass(in, dial) + } + w.stop() + fromHost, toHost := w.counts() + reachedMu.Lock() + target := reached + reachedMu.Unlock() + if target == "" { + target = strings.Join(addrs, " ") // none answered, or none was dialled + } + serveAudit(dcnDir, map[string]any{"service": s.name, "target": target, "decision": "served", + "bytes_from_host": fromHost, "bytes_to_host": toHost, "idle_closed": w.idleClosed()}) + return true +} + +// servePass is the lane's pass-through: egress's splice, which keeps a +// half-close (the Host's 'E' ends what it sends, and the answer still comes +// back), where the tunnel's spliceConn ends both directions with the first. +func servePass(in net.Conn, dial upstreamDial) { + out, err := dial() + if err != nil { + _ = in.Close() + return + } + splice(in, in, out) +} + +// serveWatch is the idle bound of one stream. Both ends are wrapped; any byte +// read or written on either one counts, and once none has for the bound, +// both are closed, which unblocks whichever copy or read is waiting. +type serveWatch struct { + bound time.Duration + last atomic.Int64 // unix nanoseconds of the last byte + fromHost atomic.Int64 + toHost atomic.Int64 + mu sync.Mutex + conns []net.Conn + host net.Conn + done chan struct{} + once sync.Once + idled atomic.Bool +} + +func newServeWatch(bound time.Duration) *serveWatch { + w := &serveWatch{bound: bound, done: make(chan struct{})} + w.last.Store(time.Now().UnixNano()) + go w.run() + return w +} + +func (w *serveWatch) run() { + tick := w.bound / 4 + if tick < 10*time.Millisecond { + tick = 10 * time.Millisecond + } + if tick > 5*time.Second { + tick = 5 * time.Second + } + t := time.NewTicker(tick) + defer t.Stop() + for { + select { + case <-w.done: + return + case <-t.C: + if time.Since(time.Unix(0, w.last.Load())) >= w.bound { + w.idled.Store(true) + w.mu.Lock() + for _, c := range w.conns { + _ = c.Close() + } + w.mu.Unlock() + return + } + } + } +} + +func (w *serveWatch) stop() { w.once.Do(func() { close(w.done) }) } + +func (w *serveWatch) idleClosed() bool { return w.idled.Load() } + +func (w *serveWatch) counts() (fromHost, toHost int64) { return w.fromHost.Load(), w.toHost.Load() } + +// wrap watches c. The first conn wrapped is the Host's: what is read from it +// came from the Host, what is written to it goes back. +func (w *serveWatch) wrap(c net.Conn) net.Conn { + w.mu.Lock() + defer w.mu.Unlock() + if w.host == nil { + w.host = c + } + w.conns = append(w.conns, c) + return &serveWatched{Conn: c, w: w, host: w.host == c} +} + +type serveWatched struct { + net.Conn + w *serveWatch + host bool +} + +func (c *serveWatched) Read(p []byte) (int, error) { + n, err := c.Conn.Read(p) + if n > 0 { + c.w.last.Store(time.Now().UnixNano()) + if c.host { + c.w.fromHost.Add(int64(n)) + } + } + return n, err +} + +func (c *serveWatched) Write(p []byte) (int, error) { + n, err := c.Conn.Write(p) + if n > 0 { + c.w.last.Store(time.Now().UnixNano()) + if c.host { + c.w.toHost.Add(int64(n)) + } + } + return n, err +} + +// CloseWrite passes a half-close through, so splice can end one direction. +func (c *serveWatched) CloseWrite() error { + if cw, ok := c.Conn.(interface{ CloseWrite() error }); ok { + return cw.CloseWrite() + } + return c.Conn.Close() +} + +// serveAudit writes one line per job to the pairing's audit log, the file +// dcnAudit writes, and to the system log. +func serveAudit(dir string, rec map[string]any) { + if dir == "" { + return + } + rec["ts"] = time.Now().UTC().Format(time.RFC3339) + rec["op"] = "serve" + raw, _ := json.Marshal(rec) + auditToSystemLog(auditFilePairingD, auditPairingOfDir(dir), raw) + if err := os.MkdirAll(dir, 0o700); err != nil { + return + } + fh, err := os.OpenFile(filepath.Join(dir, auditFilePairingD), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o600) + if err != nil { + return + } + defer fh.Close() + _, _ = fh.Write(append(raw, '\n')) +} + +// serveRetryMax is the longest a worker waits before dialling again while the +// Host answers the lane's upgrade and refuses it. A Host that predates the +// lane does that (an unknown WebSocket route is a 404, or a 403 from a server +// that closes it before accepting), and two workers dialling it every 3 s +// for good would be work for nothing on the person's machine. A Host that +// cannot be reached at all (restarting) is dialled every 3 s, as egress is. +var serveRetryMax = time.Minute + +// serveRefusedWait is how long a worker waits before it parks again after its +// WebSocket closed while parked, or after its job was refused. With +// serveWorkerCount workers, a Host that refuses or misuses every job costs +// this machine at most that many dials a second. +var serveRefusedWait = time.Second + +// localPairings counts the runs of each pairing with a Host that has no +// platform (runUnixPairing), by host name, while they run. A count, not a +// flag, so a run that is still stopping when the same pairing starts again +// cannot unmark the new one. +var localPairings sync.Map // string -> *atomic.Int32 + +// markLocalPairing records that self runs as a pairing with a Host that has no +// platform, until the returned func is called. +func markLocalPairing(self string) func() { + v, _ := localPairings.LoadOrStore(self, new(atomic.Int32)) + n := v.(*atomic.Int32) + n.Add(1) + var once sync.Once + return func() { once.Do(func() { n.Add(-1) }) } +} + +// isLocalPairing reports whether self is running as a pairing with a Host that +// has no platform. +func isLocalPairing(self string) bool { + v, ok := localPairings.Load(self) + return ok && v.(*atomic.Int32).Load() > 0 +} + +// modelDaemonTarget is the address model management (modelswap.go: the +// model-swap probe in the beat, model:list, model:pull and its probes, +// model:drop) dials for the pairing self, or the reason it dials nothing. +// For a Host with no platform it is the serve lane's own ollama target, under +// the same rule: OLLAMA_ADDR (else loopback) only when every address it names +// is this machine, and the address dialled is one of those checked: the first +// that answers, tried in turn (servePick). A bastion serves only the computer +// it runs on, and managing the models on another computer is serving it. A +// tunnel pairing keeps OLLAMA_ADDR wherever it points, as its forwards do. +func modelDaemonTarget(self string) (string, string) { + t := localOllamaAddr() + if !isLocalPairing(self) { + return t, "" + } + addrs, why := serveVetTarget(t) + if why != "" { + s, _ := serveServiceNamed("ollama") + serveTargetNotice(self, s, t, why) + return "", "refused: the model server set on this machine (OLLAMA_ADDR) is not used for this Host: " + why + } + return servePick(addrs), "" +} + +// serveTargetNoticed is the last refusal said at the machine for each pairing +// and service, so the console carries it once, and again only when it changes. +var serveTargetNoticed sync.Map // "\x00" -> string + +// serveTargetNotice says at this machine, in plain words, that the server set +// here for a service is not used for the Host with no platform, and why. A +// refusal that only the Host hears leaves the person who set the address with +// nothing on their own screen to explain why the model is missing. +func serveTargetNotice(self string, s serveService, target, why string) { + key := self + "\x00" + s.name + // The beat and a model op can refuse the same target at once; only the one + // that records this reason says it. + for { + prev, loaded := serveTargetNoticed.LoadOrStore(key, why) + if !loaded { + break + } + if prev.(string) == why { + return + } + if serveTargetNoticed.CompareAndSwap(key, prev, why) { + break + } + } + fmt.Printf("[%s] %s=%s is not used for the Host on this computer: %s. Set %s to a server on this "+ + "computer, or unset it.\n", self, s.envVar, target, why, s.envVar) +} + +// serveTargetNoticeClear forgets what was said for a service that is fine +// again, so a later refusal is said again. +func serveTargetNoticeClear(self string, s serveService) { + serveTargetNoticed.Delete(self + "\x00" + s.name) +} + +// runServeWorkers keeps n workers parked on the Host's /bastion-serve for the +// pairing self, each serving one job (serveJob). A worker that takes a job +// starts its replacement at once and ends with the job; one whose WebSocket +// dies while parked dials again. pctx is the pairing's context: a revoked +// pairing's workers exit. +func runServeWorkers(pctx context.Context, self, dcnDir, wsURL string, tlsCfg *tls.Config, + dialHost func(ctx context.Context) (net.Conn, error), bearer string, n int) { + dialer := &websocket.Dialer{ + NetDialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { return dialHost(ctx) }, + TLSClientConfig: tlsCfg, + HandshakeTimeout: 20 * time.Second, + } + var spawn func() + spawn = func() { + go func() { + wait := 3 * time.Second + for pctx.Err() == nil { + ctx, cancel := context.WithTimeout(pctx, 25*time.Second) + ws, resp, err := dialer.DialContext(ctx, wsURL, http.Header{}) + cancel() + if err != nil { + if resp != nil { + sleepCtx(pctx, wait) + wait = min(2*wait, serveRetryMax) + } else { + sleepCtx(pctx, 3*time.Second) + } + continue + } + wait = 3 * time.Second + if err := ws.WriteMessage(websocket.TextMessage, []byte("BEARER "+bearer)); err != nil { + _ = ws.Close() + sleepCtx(pctx, time.Second) + continue + } + stopKA := armWSKeepalive(ws) + unpark := context.AfterFunc(pctx, func() { _ = ws.Close() }) + took := serveJob(ws, self, dcnDir, spawn) + unpark() + stopKA() + _ = ws.Close() + if took { + return + } + // Closed while parked (a Host restart), or the job refused: not + // straight back, so a Host that closes or misuses every worker is + // not dialled in a loop. + sleepCtx(pctx, serveRefusedWait) + } + }() + } + for i := 0; i < n; i++ { + spawn() + } +} + +// laneServeReport is what a local pairing's beat says of its serve lane. +type laneServeReport struct { + served []string // lane_serves + models []string // ollama's, filtered to the scope + llmModels []string // the llm server's, filtered to the scope + piiOK bool +} + +// laneServes is what this pairing serves over the lane right now, for the +// beat: each service whose target is set and is this machine, whose server +// answers its probe, and whose family the pairing holds. A scoped pairing +// with no model left in a server's list is not served that server, as the +// tunnel's beat withholds a lane with nothing in it. +func laneServes(self string) laneServeReport { + rep := laneServeReport{served: []string{}, models: []string{}, llmModels: []string{}} + if capsAllow(self, "inference") != "" { + return rep + } + scope := pairingModelScope(self) + for _, s := range serveServices { + t := s.target() + if t == "" { + continue + } + addrs, why := serveVetTarget(t) + if why != "" { + serveTargetNotice(self, s, t, why) + continue + } + serveTargetNoticeClear(self, s) + addr := servePick(addrs) + switch s.name { + case "ollama": + ok, models := ollamaModelsAt([]string{addr}) + models = scope.filterModels(models) + if !ok || (scope.scoped && len(models) == 0) { + continue + } + if models != nil { + rep.models = models + } + case "llm": + ok, models := openAICompatModelsAt(addr) + models = scope.filterModels(models) + if !ok || (scope.scoped && len(models) == 0) { + continue + } + if models != nil { + rep.llmModels = models + } + case "pii": + if !scope.allowsPII() { + continue + } + if _, healthy := piiProbeAt(addr); !healthy { + continue + } + rep.piiOK = true + } + rep.served = append(rep.served, s.name) + } + return rep +} diff --git a/client-go/servelane_dial_linux_test.go b/client-go/servelane_dial_linux_test.go new file mode 100644 index 0000000..682b6b3 --- /dev/null +++ b/client-go/servelane_dial_linux_test.go @@ -0,0 +1,131 @@ +//go:build linux + +// The serve lane dials a target set by name at each address it checked, in +// turn (servelane.go: serveVetTarget, serveDial, servePick). Uses the fixtures +// of servelane_linux_test.go, modelscope_test.go and modelswap_test.go. + +package main + +import ( + "bufio" + "context" + "fmt" + "io" + "net" + "net/http" + "slices" + "strings" + "testing" + "time" +) + +// A target set by name is dialled at every address it was checked for, in +// turn, as a plain dial of the name is. OLLAMA_ADDR=localhost:, a +// resolver that answers ::1 first (as most do) and a server listening on +// 127.0.0.1 only (ollama's default): a job on the lane, the beat and model +// management each reach it. The resolver's two addresses are both this +// machine's and are the only ones there are to dial. +// +// MUTATION: serveVetTarget returns only the first address it checked (as it +// did), or serveJob dials only addrs[0]. +// RESULT: red — the job's request gets no answer (its one dial, to [::1], is +// refused), the beat has no ollama in lane_serves, and model management is +// handed [::1]. +func TestTheServeLaneDialsEachCheckedAddressInTurn(t *testing.T) { + up := newFakeOllama(t) + listening := up.srv.Listener.Addr().String() + if !strings.HasPrefix(listening, "127.0.0.1:") { + t.Fatalf("the stand-in listens on %s, not 127.0.0.1 only, so this case proves nothing", listening) + } + port := portOf(up.srv.Listener) + if c, err := net.DialTimeout("tcp", net.JoinHostPort("::1", port), time.Second); err == nil { + _ = c.Close() + t.Fatalf("something answers on [::1]:%s, so the first address would not fail and this case proves nothing", port) + } + old := serveLookupIP + t.Cleanup(func() { serveLookupIP = old }) + serveLookupIP = func(_ context.Context, host string) ([]net.IPAddr, error) { + if host != "localhost" { + return nil, fmt.Errorf("no such host") + } + return []net.IPAddr{{IP: net.IPv6loopback}, {IP: net.ParseIP("127.0.0.1")}}, nil + } + t.Setenv("OLLAMA_ADDR", "localhost:"+port) + t.Setenv("PRIVATAE_LLM_ADDR", "") + t.Setenv("PRIVATAE_PII_ADDR", "") + resetModelDaemonCache() + h, cfg := serveLaneBox(t, "dual.example.test") + runInBackground(t, cfg) + + if got := strsOf(h.beat(t)["lane_serves"]); !slices.Contains(got, "ollama") { + t.Errorf("the beat does not say ollama is served: lane_serves %q", got) + } + + ws := h.take(t) + if got := ask(t, ws, "ollama"); got != "A" { + t.Fatalf("the lane must accept ollama: %q", got) + } + conn := laneConn(ws) + req, _ := http.NewRequest(http.MethodGet, "http://serve/api/tags", nil) + req.Close = true + if err := req.Write(conn); err != nil { + t.Fatal(err) + } + _ = conn.SetReadDeadline(time.Now().Add(10 * time.Second)) + resp, err := http.ReadResponse(bufio.NewReader(conn), req) + if err != nil { + t.Fatalf("no answer over the lane (only [::1] was dialled?): %v", err) + } + b, _ := io.ReadAll(resp.Body) + if resp.StatusCode != http.StatusOK || !strings.Contains(string(b), "qwen4-embed") { + t.Fatalf("over the lane: %d %s", resp.StatusCode, b) + } + + if addr, why := modelDaemonTarget(cfg.HostName); addr != listening || why != "" { + t.Errorf("model management is handed %q (%q), want the address that answers, %s", addr, why, listening) + } + out, errOut, code, _ := driveModelOp(t, cfg.HostName, "model:list", nil, t.TempDir()) + if code != 0 || !strings.Contains(out, "qwen4-embed") { + t.Errorf("model:list: exit %d, %q %q", code, out, errOut) + } + for _, s := range up.requests() { + if !strings.HasPrefix(s, "GET ") { + t.Errorf("the stand-in was asked %q", s) + } + } +} + +// servePick returns the first checked address that accepts, and the first +// checked when none does; it never returns an address it was not given. +func TestServePickTakesTheFirstCheckedAddressThatAnswers(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer ln.Close() + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + _ = c.Close() + } + }() + port := portOf(ln) + v6 := net.JoinHostPort("::1", port) + if c, err := net.DialTimeout("tcp", v6, time.Second); err == nil { + _ = c.Close() + t.Fatalf("something answers on %s, so this case proves nothing", v6) + } + if got := servePick([]string{v6, ln.Addr().String()}); got != ln.Addr().String() { + t.Errorf("pick %q, want %s", got, ln.Addr()) + } + _ = ln.Close() + if got := servePick([]string{v6, "127.0.0.1:" + port}); got != v6 { + t.Errorf("with nothing answering, pick %q, want the first checked, %s", got, v6) + } + if got := servePick([]string{"127.0.0.1:" + port}); got != "127.0.0.1:"+port { + t.Errorf("one address: %q", got) + } +} diff --git a/client-go/servelane_linux_test.go b/client-go/servelane_linux_test.go new file mode 100644 index 0000000..2874875 --- /dev/null +++ b/client-go/servelane_linux_test.go @@ -0,0 +1,992 @@ +//go:build linux + +// The serve lane (servelane.go) over a tcp pairing: a Host in a VM on this +// machine reaches the model servers set here by service name, through the +// forwards' own filter, within the lane's bounds, and the pairing's beat says +// what it serves. +// +// Uses the fixtures of local_host_linux_test.go and local_host_tcp_linux_test.go +// (newTestCA, tcpPairingFor, laneListener, pairingBox...), so it carries the +// same build tag. + +package main + +import ( + "bufio" + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "fmt" + "io" + "log" + "net" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "slices" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/gorilla/websocket" +) + +// ── fixtures ──────────────────────────────────────────────────────────────── + +// serveHost is a Host's lane port as the serve lane meets it: the lane +// port's mutual TLS, every worker route kept parked, each beat recorded +// whole, and each /bastion-serve worker handed to the test once it has sent +// its hello. +type serveHost struct { + mu sync.Mutex + beats []map[string]any + hellos []string + workers chan *websocket.Conn +} + +func startServeHost(t *testing.T, raw net.Listener, cert tls.Certificate, clientCA *testCA) *serveHost { + t.Helper() + h := &serveHost{workers: make(chan *websocket.Conn, 64)} + pool := x509.NewCertPool() + pool.AddCert(clientCA.cert) + ln := tls.NewListener(raw, &tls.Config{Certificates: []tls.Certificate{cert}, ClientCAs: pool, + ClientAuth: tls.RequireAndVerifyClientCert}) + up := websocket.Upgrader{} + mux := http.NewServeMux() + mux.HandleFunc("/api/network/bastions/heartbeat", func(w http.ResponseWriter, r *http.Request) { + var m map[string]any + _ = json.NewDecoder(r.Body).Decode(&m) + h.mu.Lock() + h.beats = append(h.beats, m) + h.mu.Unlock() + _, _ = w.Write([]byte("{}")) + }) + mux.HandleFunc("/bastion-serve", func(w http.ResponseWriter, r *http.Request) { + c, err := up.Upgrade(w, r, nil) + if err != nil { + return + } + _, hello, err := c.ReadMessage() + if err != nil { + c.Close() + return + } + h.mu.Lock() + h.hellos = append(h.hellos, string(hello)) + h.mu.Unlock() + h.workers <- c // the test owns it from here + }) + park := func(w http.ResponseWriter, r *http.Request) { + c, err := up.Upgrade(w, r, nil) + if err != nil { + return + } + defer c.Close() + for { + if _, _, err := c.ReadMessage(); err != nil { + return + } + } + } + for _, p := range []string{"/bastion-harness", "/bastion-mgmt", "/bastion-egress"} { + mux.HandleFunc(p, park) + } + srv := &http.Server{Handler: mux, ErrorLog: log.New(io.Discard, "", 0)} + go func() { _ = srv.Serve(ln) }() + t.Cleanup(func() { + _ = srv.Close() + for { + select { + case c := <-h.workers: + c.Close() + default: + return + } + } + }) + return h +} + +// take is the next parked serve worker. +func (h *serveHost) take(t *testing.T) *websocket.Conn { + t.Helper() + select { + case c := <-h.workers: + t.Cleanup(func() { c.Close() }) + return c + case <-time.After(10 * time.Second): + t.Fatal("no /bastion-serve worker parked on the Host within 10 s") + } + return nil +} + +// ask sends the 'S' frame for service and returns the bastion's answer: "A", +// or "E" and the reason. +func ask(t *testing.T, c *websocket.Conn, service string) string { + t.Helper() + b, _ := json.Marshal(map[string]string{"service": service}) + return askRaw(t, c, websocket.BinaryMessage, append([]byte{'S'}, b...)) +} + +func askRaw(t *testing.T, c *websocket.Conn, mt int, frame []byte) string { + t.Helper() + if err := c.WriteMessage(mt, frame); err != nil { + t.Fatalf("send the first frame: %v", err) + } + _ = c.SetReadDeadline(time.Now().Add(10 * time.Second)) + _, m, err := c.ReadMessage() + _ = c.SetReadDeadline(time.Time{}) + if err != nil { + t.Fatalf("no answer to the first frame: %v", err) + } + return string(m) +} + +// beat is the first beat the Host received, waiting up to 10 s. +func (h *serveHost) beat(t *testing.T) map[string]any { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for time.Now().Before(deadline) { + h.mu.Lock() + if len(h.beats) > 0 { + b := h.beats[0] + h.mu.Unlock() + return b + } + h.mu.Unlock() + time.Sleep(20 * time.Millisecond) + } + t.Fatal("no heartbeat reached the Host within 10 s") + return nil +} + +// runInBackground runs the pairing until the test ends. +func runInBackground(t *testing.T, cfg *Config) { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- runPairing(ctx, cfg) }() + t.Cleanup(func() { + cancel() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Error("the pairing did not stop") + } + // A stopped (revoked) pairing's streams end with it: each one's worker + // is closed. Waited for here too because a stream's last act is its + // audit line, in the pairing's directory, which the test then removes. + deadline := time.Now().Add(5 * time.Second) + for { + v, ok := serveInFlight.Load(cfg.HostName) + if !ok || v.(*atomic.Int32).Load() == 0 { + return + } + if time.Now().After(deadline) { + t.Errorf("%d streams still open 5 s after the pairing stopped", v.(*atomic.Int32).Load()) + return + } + time.Sleep(10 * time.Millisecond) + } + }) +} + +// serveLaneBox is an unanchored local machine with a tcp pairing to a +// stand-in Host on 127.0.0.1, running. host keeps each test's pairing (and its +// stream count) its own. +func serveLaneBox(t *testing.T, host string) (*serveHost, *Config) { + t.Helper() + pairingBox(t) + laneTestProcess(t) + ca := newTestCA(t, "home host CA") + ln := laneListener(t) + h := startServeHost(t, ln, ca.serverCert(t, tcpServerName), ca) + cfg := tcpPairingFor(t, ca, host, ln.Addr().String()) + if err := cfg.save(); err != nil { + t.Fatal(err) + } + return h, cfg +} + +// laneConn is the Host's end of an accepted job: 'D' frames both ways, 'E' +// for the end, as the Host's relay frames a spliced connection. +func laneConn(c *websocket.Conn) net.Conn { return &wsConn{ws: c} } + +// chatRequest writes one streaming chat request on conn and returns the reader +// its response is read from. +func chatRequest(t *testing.T, conn net.Conn, model string) (*http.Response, *http.Request) { + t.Helper() + body := fmt.Sprintf(`{"model":%q,"stream":true,"messages":[{"role":"user","content":"hello"}]}`, model) + req, _ := http.NewRequest(http.MethodPost, "http://serve/v1/chat/completions", strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + if err := req.Write(conn); err != nil { + t.Fatalf("write the request: %v", err) + } + _ = conn.SetReadDeadline(time.Now().Add(15 * time.Second)) + resp, err := http.ReadResponse(bufio.NewReader(conn), req) + if err != nil { + t.Fatalf("read the response: %v", err) + } + return resp, req +} + +// ── 1. a chat streams ─────────────────────────────────────────────────────── + +// streamingServer is a stand-in OpenAI-compatible server. Its chat answer is +// three chunks; after the first it waits until the test says the Host has +// read it, and records whether it had to give up waiting. +type streamingServer struct { + srv *httptest.Server + sawFirst chan struct{} + gated atomic.Bool + seen atomic.Int32 +} + +func newStreamingServer(t *testing.T) *streamingServer { + t.Helper() + s := &streamingServer{sawFirst: make(chan struct{})} + s.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + s.seen.Add(1) + switch r.URL.Path { + case "/v1/chat/completions": + case "/api/tags": + _, _ = w.Write([]byte(`{"models":[{"name":"test-model:latest"}]}`)) + return + default: + _, _ = w.Write([]byte(`{"ok":true}`)) + return + } + _, _ = io.Copy(io.Discard, r.Body) + w.Header().Set("Content-Type", "text/event-stream") + fl := w.(http.Flusher) + _, _ = io.WriteString(w, "data: {\"chunk\":1}\n\n") + fl.Flush() + select { + case <-s.sawFirst: + s.gated.Store(true) + case <-time.After(10 * time.Second): + } + _, _ = io.WriteString(w, "data: {\"chunk\":2}\n\n") + fl.Flush() + _, _ = io.WriteString(w, "data: {\"chunk\":3}\n\n") + fl.Flush() + _, _ = io.WriteString(w, "data: [DONE]\n\n") + })) + t.Cleanup(s.srv.Close) + return s +} + +// SECURITY (and the lane's purpose): over a tcp pairing, the Host asks for a +// service by name and a chat streams from the server set here. The Host reads +// chunk 1 before the server writes chunk 3, so nothing in the path holds a +// response back: not the lane's framing, not the pass-through an unscoped +// pairing gets, not the filter a pairing without `models` gets. +// +// MUTATION: drop the serve workers from runUnixPairing. +// RESULT: red — no /bastion-serve worker ever parks on the Host. +func TestTheServeLaneStreamsAChatOverATCPPairing(t *testing.T) { + for _, c := range []struct { + name, service, env string + modelsOff bool + }{ + {"llm, passed through", "llm", "PRIVATAE_LLM_ADDR", false}, + {"ollama, through the filter", "ollama", "OLLAMA_ADDR", true}, + } { + t.Run(c.name, func(t *testing.T) { + s := newStreamingServer(t) + t.Setenv(c.env, s.srv.Listener.Addr().String()) + h, cfg := serveLaneBox(t, "stream-"+c.service+".example.test") + if c.modelsOff { + if err := localDenySet(cfg.HostName, "models", true); err != nil { + t.Fatal(err) + } + if !inferenceFilterNeeded(cfg.HostName, laneOllama, pairingModelScope(cfg.HostName)) { + t.Fatal("with models off the ollama lane must be filtered, or this case proves nothing") + } + } + runInBackground(t, cfg) + ws := h.take(t) + if got := ask(t, ws, c.service); got != "A" { + t.Fatalf("the lane must accept %s: %q", c.service, got) + } + conn := laneConn(ws) + resp, _ := chatRequest(t, conn, "test-model") + if resp.StatusCode != http.StatusOK { + t.Fatalf("the chat answered %d", resp.StatusCode) + } + br := bufio.NewReader(resp.Body) + var got []string + first := false + for { + line, err := br.ReadString('\n') + if strings.HasPrefix(line, "data: ") { + got = append(got, strings.TrimSpace(line)) + if !first && strings.Contains(line, `"chunk":1`) { + first = true + close(s.sawFirst) + } + } + if err != nil || strings.Contains(line, "[DONE]") { + break + } + } + if !first { + t.Fatalf("the stream never carried chunk 1: %q", got) + } + if !s.gated.Load() { + t.Fatal("the server wrote chunk 3 before the Host read chunk 1: the lane held the stream back") + } + want := []string{`data: {"chunk":1}`, `data: {"chunk":2}`, `data: {"chunk":3}`, `data: [DONE]`} + if !slices.Equal(got, want) { + t.Fatalf("the stream arrived as %q", got) + } + }) + } +} + +// ── 2. a service, never an address ────────────────────────────────────────── + +// SECURITY: the Host names a service and the target is the one set on this +// machine. A first frame naming an address, a name the lane does not carry, a +// frame that is not 'S', a target set to a computer that is not this one, and +// a service with no target each answer 'E' with a reason, and nothing is +// dialled: a listener at the address named gets no connection. +// +// MUTATION: in serveVetTarget, skip the ownDestination check. +// RESULT: red — ollama with OLLAMA_ADDR on another computer is accepted. +func TestTheServeLaneTakesAServiceNameNeverAnAddress(t *testing.T) { + named, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer named.Close() + var reached atomic.Int32 + go func() { + for { + c, err := named.Accept() + if err != nil { + return + } + reached.Add(1) + c.Close() + } + }() + t.Setenv("OLLAMA_ADDR", "192.0.2.10:11434") // TEST-NET-1: never this machine + t.Setenv("PRIVATAE_LLM_ADDR", "") + t.Setenv("PRIVATAE_PII_ADDR", "198.51.100.7:"+portOf(named)) + h, cfg := serveLaneBox(t, "names.example.test") + runInBackground(t, cfg) + + for _, c := range []struct { + name string + frame []byte + mt int + want []string + }{ + {"an address for a service", sFrame(named.Addr().String()), websocket.BinaryMessage, + []string{"Erefused: ", "never takes an address"}}, + {"a name the lane does not carry", sFrame("nas"), websocket.BinaryMessage, + []string{"Erefused: ", `"nas" is not a service`}}, + {"a URL for a service", sFrame("http://" + named.Addr().String() + "/api/tags"), websocket.BinaryMessage, + []string{"Erefused: ", "is not a service"}}, + {"a frame that is not S", []byte("D" + "GET / HTTP/1.1\r\n\r\n"), websocket.BinaryMessage, + []string{"Erefused: ", "the first frame on this lane is 'S'"}}, + {"a text frame", []byte(`S{"service":"ollama"}`), websocket.TextMessage, + []string{"Erefused: ", "the first frame on this lane is 'S'"}}, + {"an S frame past its bound", append(sFrame("ollama"), make([]byte, serveHelloMax)...), websocket.BinaryMessage, + []string{"Erefused: ", "at most 1024 bytes"}}, + {"ollama set to another computer", sFrame("ollama"), websocket.BinaryMessage, + []string{"Erefused: ", "OLLAMA_ADDR", "192.0.2.10 is not this machine"}}, + {"pii set to another computer", sFrame("pii"), websocket.BinaryMessage, + []string{"Erefused: ", "PRIVATAE_PII_ADDR", "198.51.100.7 is not this machine"}}, + {"llm with nothing set", sFrame("llm"), websocket.BinaryMessage, + []string{"Eunavailable: ", "PRIVATAE_LLM_ADDR"}}, + } { + ws := h.take(t) + got := askRaw(t, ws, c.mt, c.frame) + for _, w := range c.want { + if !strings.Contains(got, w) { + t.Errorf("%s: the answer %q does not carry %q", c.name, got, w) + } + } + if !strings.HasPrefix(got, "E") { + t.Errorf("%s: answered %q, not E", c.name, got) + } + // Refused means closed: nothing after the E. + _ = ws.SetReadDeadline(time.Now().Add(3 * time.Second)) + if _, m, err := ws.ReadMessage(); err == nil { + t.Errorf("%s: the worker carried on after its refusal: %q", c.name, m) + } + } + time.Sleep(200 * time.Millisecond) + if n := reached.Load(); n != 0 { + t.Fatalf("a refused job dialled the address it named (%d connections)", n) + } +} + +func sFrame(service string) []byte { + b, _ := json.Marshal(map[string]string{"service": service}) + return append([]byte{'S'}, b...) +} + +func portOf(ln net.Listener) string { + _, p, _ := net.SplitHostPort(ln.Addr().String()) + return p +} + +// A target set by name is resolved and each address checked; the addresses +// dialled are the ones checked, every one of them, in the resolver's order. +// 0.0.0.0 is a listen address and dials as loopback (once, if loopback is +// there already). This machine's own interface address passes. +func TestServeVetTargetChecksEveryAddressItWouldDial(t *testing.T) { + old := serveLookupIP + t.Cleanup(func() { serveLookupIP = old }) + serveLookupIP = func(_ context.Context, host string) ([]net.IPAddr, error) { + switch host { + case "localhost": + return []net.IPAddr{{IP: net.ParseIP("127.0.0.1")}, {IP: net.ParseIP("::1")}}, nil + case "v6first.example.test": + return []net.IPAddr{{IP: net.ParseIP("::1")}, {IP: net.ParseIP("127.0.0.1")}, {IP: net.ParseIP("0.0.0.0")}}, nil + case "mixed.example.test": + return []net.IPAddr{{IP: net.ParseIP("127.0.0.1")}, {IP: net.ParseIP("192.0.2.10")}}, nil + } + return nil, fmt.Errorf("no such host") + } + for target, want := range map[string][]string{ + "127.0.0.1:11434": {"127.0.0.1:11434"}, + "[::1]:8080": {"[::1]:8080"}, + "0.0.0.0:11434": {"127.0.0.1:11434"}, + "[::]:11434": {"[::1]:11434"}, + "localhost:11434": {"127.0.0.1:11434", "[::1]:11434"}, + "v6first.example.test:80": {"[::1]:80", "127.0.0.1:80"}, + "mixed.example.test:11434": nil, + "nowhere.example.test:80": nil, + "192.0.2.10:11434": nil, + "198.51.100.9:80": nil, + "127.0.0.1": nil, + "127.0.0.1:0": nil, + "127.0.0.1:99999": nil, + } { + got, why := serveVetTarget(target) + if !slices.Equal(got, want) || (want == nil) != (why != "") { + t.Errorf("%q: dial %q (%q), want %q", target, got, why, want) + } + } + ips, err := ownInterfaceIPs() + if err != nil { + t.Fatal(err) + } + for _, ip := range ips { + if ip.IsLoopback() || ip.IsLinkLocalUnicast() { + continue + } + a := net.JoinHostPort(ip.String(), "11434") + if got, why := serveVetTarget(a); !slices.Equal(got, []string{a}) { + t.Errorf("this machine's own address %s is refused: %q", a, why) + } + } +} + +// ── 3. the forwards' walls, unchanged ─────────────────────────────────────── + +// SECURITY: the lane keeps every wall the tunnel forwards keep. With +// `inference` switched off at this machine, a job is refused before any dial. +// With `models` off, /api/pull and /api/delete are refused 403 and never reach +// ollama, while a chat does. On an anchored machine they are refused to the +// anchor itself. A pairing scoped to a model is refused another one with 403, +// which never reaches the server, and is served its own. +func TestTheServeLaneKeepsTheForwardsWalls(t *testing.T) { + t.Run("inference off at the machine", func(t *testing.T) { + // A raw listener, so a dial by the serve job is seen even though the + // Host sends it nothing: a connection that does not open with a beat's + // probe (GET /api/tags, which any pairing's beat in this process may + // send to OLLAMA_ADDR) is the job's. + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { ln.Close() }) + var jobDials atomic.Int32 + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + go func() { + defer c.Close() + _ = c.SetReadDeadline(time.Now().Add(2 * time.Second)) + b := make([]byte, len("GET /api/tags ")) + if _, err := io.ReadFull(c, b); err != nil || string(b) != "GET /api/tags " { + jobDials.Add(1) + } + }() + } + }() + t.Setenv("OLLAMA_ADDR", ln.Addr().String()) + h, cfg := serveLaneBox(t, "walls-off.example.test") + if err := localDenySet(cfg.HostName, "inference", true); err != nil { + t.Fatal(err) + } + runInBackground(t, cfg) + got := ask(t, h.take(t), "ollama") + if !strings.HasPrefix(got, "Erefused: ") { + t.Fatalf("with inference off the job must be refused: %q", got) + } + time.Sleep(2500 * time.Millisecond) + if n := jobDials.Load(); n != 0 { + t.Fatalf("a refused job dialled ollama (%d connections)", n) + } + }) + pullAndDelete := func(t *testing.T, h *serveHost, up *fakeOllama) { + t.Helper() + for _, p := range []string{"/api/pull", "/api/delete"} { + ws := h.take(t) + if got := ask(t, ws, "ollama"); got != "A" { + t.Fatalf("%s: the job must be accepted, and refused by the filter: %q", p, got) + } + conn := laneConn(ws) + req, _ := http.NewRequest(http.MethodPost, "http://serve"+p, strings.NewReader(`{"model":"test-model"}`)) + if err := req.Write(conn); err != nil { + t.Fatal(err) + } + _ = conn.SetReadDeadline(time.Now().Add(10 * time.Second)) + resp, err := http.ReadResponse(bufio.NewReader(conn), req) + if err != nil { + t.Fatalf("%s: %v", p, err) + } + b, _ := io.ReadAll(resp.Body) + if resp.StatusCode != http.StatusForbidden || !strings.Contains(string(b), "models") { + t.Fatalf("%s over the lane: %d %s", p, resp.StatusCode, b) + } + } + for _, s := range up.requests() { + if strings.Contains(s, "/api/pull") || strings.Contains(s, "/api/delete") { + t.Fatalf("a refused store request reached ollama: %q", up.requests()) + } + } + } + t.Run("models off at the machine", func(t *testing.T) { + up := newFakeOllama(t) + t.Setenv("OLLAMA_ADDR", up.srv.Listener.Addr().String()) + h, cfg := serveLaneBox(t, "walls-models.example.test") + if err := localDenySet(cfg.HostName, "models", true); err != nil { + t.Fatal(err) + } + runInBackground(t, cfg) + pullAndDelete(t, h, up) + }) + t.Run("an anchored machine, as its anchor", func(t *testing.T) { + up := newFakeOllama(t) + t.Setenv("OLLAMA_ADDR", up.srv.Listener.Addr().String()) + h, cfg := serveLaneBox(t, "walls-anchor.example.test") + p, _ := modePath() + if err := os.WriteFile(p, []byte(`{"mode":"networked"}`), 0o600); err != nil { + t.Fatal(err) + } + if err := writeAnchor("Acme IT", cfg.HostName); err != nil { + t.Fatal(err) + } + if currentAnchor() == nil || capsAllow(cfg.HostName, "inference") != "" { + t.Fatal("the pairing must be this machine's anchor, holding inference, or this case proves nothing") + } + runInBackground(t, cfg) + pullAndDelete(t, h, up) + }) + t.Run("a pairing scoped to one model", func(t *testing.T) { + priv := setupBox(t) + withMarkerAt(t, filepath.Join(t.TempDir(), "machine.json")) + laneTestProcess(t) + up := newFakeOllama(t) + t.Setenv("PRIVATAE_LLM_ADDR", up.srv.Listener.Addr().String()) + ca := newTestCA(t, "home host CA") + ln := laneListener(t) + h := startServeHost(t, ln, ca.serverCert(t, tcpServerName), ca) + member := tcpPairingFor(t, ca, tMember, ln.Addr().String()) + member.State = pairingPending + if err := member.save(); err != nil { + t.Fatal(err) + } + syncGrant(t, priv, tMember, []string{"inference"}, []string{"qwen4-embed"}) + cfgs, err := loadPairings() + if err != nil { + t.Fatal(err) + } + cfg := pairingFor(cfgs, tMember) + if cfg == nil || !cfg.active() || !cfg.tcpTransport() { + t.Fatalf("the member's tcp pairing must be active after the grant: %+v", cfg) + } + runInBackground(t, cfg) + for model, want := range map[string]int{"llama3:70b": http.StatusForbidden, "qwen4-embed": http.StatusOK} { + ws := h.take(t) + if got := ask(t, ws, "llm"); got != "A" { + t.Fatalf("the member holds inference, so the job is accepted: %q", got) + } + resp, _ := chatRequest(t, laneConn(ws), model) + b, _ := io.ReadAll(resp.Body) + if resp.StatusCode != want { + t.Fatalf("%s over the scoped lane: %d %s", model, resp.StatusCode, b) + } + } + chats := 0 + for _, s := range up.requests() { + if s == "POST /v1/chat/completions" { + chats++ + } + } + if chats != 1 { + t.Fatalf("only the scoped model's request may reach the server: %q", up.requests()) + } + }) +} + +// ── 4. the bounds ─────────────────────────────────────────────────────────── + +// SECURITY: one pairing holds at most 16 streams open here. Sixteen run at +// once; the 17th is answered 'E' busy with the reason and dials nothing; once +// one ends, the next is accepted. +func TestTheServeLaneHoldsSixteenStreamsAndRefusesTheSeventeenth(t *testing.T) { + var held, opened atomic.Int32 + release := make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/chat/completions" { + _, _ = w.Write([]byte(`{}`)) // the beat's probe + return + } + opened.Add(1) + held.Add(1) + defer held.Add(-1) + w.Header().Set("Content-Type", "text/event-stream") + _, _ = io.WriteString(w, "data: {\"chunk\":1}\n\n") + w.(http.Flusher).Flush() + select { + case <-release: + case <-r.Context().Done(): + } + })) + t.Cleanup(func() { close(release); srv.Close() }) + t.Setenv("PRIVATAE_LLM_ADDR", srv.Listener.Addr().String()) + h, cfg := serveLaneBox(t, "sixteen.example.test") + runInBackground(t, cfg) + + var streams []*websocket.Conn + for i := 0; i < serveMaxInFlight; i++ { + ws := h.take(t) + if got := ask(t, ws, "llm"); got != "A" { + t.Fatalf("stream %d of %d must be accepted: %q", i+1, serveMaxInFlight, got) + } + resp, _ := chatRequest(t, laneConn(ws), "test-model") + if resp.StatusCode != http.StatusOK { + t.Fatalf("stream %d: %d", i+1, resp.StatusCode) + } + streams = append(streams, ws) + } + if n := held.Load(); n != serveMaxInFlight { + t.Fatalf("the server holds %d streams, want %d at once", n, serveMaxInFlight) + } + got := ask(t, h.take(t), "llm") + if !strings.HasPrefix(got, "Ebusy: ") || !strings.Contains(got, "16 streams") { + t.Fatalf("the 17th stream must be answered busy with the reason: %q", got) + } + if n := opened.Load(); n != serveMaxInFlight { + t.Fatalf("the refused 17th reached the server (%d requests)", n) + } + // One ends; the next is accepted. + streams[0].Close() + deadline := time.Now().Add(10 * time.Second) + for { + ws := h.take(t) + got := ask(t, ws, "llm") + if got == "A" { + break + } + if !strings.HasPrefix(got, "Ebusy: ") || time.Now().After(deadline) { + t.Fatalf("after a stream ended the next must be accepted: %q", got) + } + time.Sleep(50 * time.Millisecond) + } +} + +// SECURITY: a stream with no byte either way for the idle bound is closed, +// both ends, on a pass-through and through the filter: a server that answers +// headers and then nothing does not hold the stream (or this machine's slot) +// for ever. A stream that keeps sending, for several times the bound, is not +// cut: the bound is on silence, never on a response's length. +func TestTheServeLaneClosesAStreamIdleForTheBound(t *testing.T) { + old := serveIdleTimeout + serveIdleTimeout = 400 * time.Millisecond + t.Cleanup(func() { serveIdleTimeout = old }) + + // silent answers headers and then holds the connection, recording when + // this machine closes it. + silentClosed := make(chan time.Time, 4) + silent, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { silent.Close() }) + go func() { + for { + c, err := silent.Accept() + if err != nil { + return + } + go func(c net.Conn) { + defer c.Close() + br := bufio.NewReader(c) + req, err := http.ReadRequest(br) + if err != nil { + return + } + if req.URL.Path != "/v1/chat/completions" { + // The beat's probes get an ordinary answer. + _, _ = io.WriteString(c, "HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\n{}") + return + } + _, _ = io.WriteString(c, "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nTransfer-Encoding: chunked\r\n\r\n") + _, _ = io.Copy(io.Discard, br) + silentClosed <- time.Now() + }(c) + } + }() + // steady sends a chunk every 100 ms for 1.6 s: four times the bound. + steady := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/chat/completions" { + _, _ = w.Write([]byte(`{}`)) + return + } + w.Header().Set("Content-Type", "text/event-stream") + for i := 1; i <= 16; i++ { + fmt.Fprintf(w, "data: {\"chunk\":%d}\n\n", i) + w.(http.Flusher).Flush() + time.Sleep(100 * time.Millisecond) + } + _, _ = io.WriteString(w, "data: [DONE]\n\n") + })) + t.Cleanup(steady.Close) + + for _, c := range []struct { + name, service, env string + modelsOff bool + }{ + {"a pass-through", "llm", "PRIVATAE_LLM_ADDR", false}, + {"through the filter", "ollama", "OLLAMA_ADDR", true}, + } { + t.Run(c.name, func(t *testing.T) { + t.Setenv(c.env, silent.Addr().String()) + h, cfg := serveLaneBox(t, "idle-"+c.service+".example.test") + if c.modelsOff { + if err := localDenySet(cfg.HostName, "models", true); err != nil { + t.Fatal(err) + } + } + runInBackground(t, cfg) + ws := h.take(t) + if got := ask(t, ws, c.service); got != "A" { + t.Fatalf("accept: %q", got) + } + conn := laneConn(ws) + resp, _ := chatRequest(t, conn, "test-model") + start := time.Now() + _ = conn.SetReadDeadline(time.Now().Add(10 * time.Second)) + _, err := io.Copy(io.Discard, resp.Body) + took := time.Since(start) + if err == nil { + t.Fatal("a silent stream must end in a close, not a complete answer") + } + if took < 300*time.Millisecond || took > 3*time.Second { + t.Fatalf("the silent stream was closed after %v; the bound is %v", took, serveIdleTimeout) + } + select { + case <-silentClosed: + case <-time.After(3 * time.Second): + t.Fatal("the server's end of the silent stream was left open") + } + + t.Setenv(c.env, steady.Listener.Addr().String()) + ws = h.take(t) + if got := ask(t, ws, c.service); got != "A" { + t.Fatalf("accept: %q", got) + } + resp, _ = chatRequest(t, laneConn(ws), "test-model") + b, err := io.ReadAll(resp.Body) + if err != nil || !strings.Contains(string(b), `"chunk":16`) || !strings.Contains(string(b), "[DONE]") { + t.Fatalf("a stream that keeps sending was cut: %v %q", err, b) + } + }) + } +} + +// ── 5. the beat says what is served ───────────────────────────────────────── + +func capsOf(m map[string]any) []string { + var out []string + if l, ok := m["capabilities"].([]any); ok { + for _, c := range l { + out = append(out, fmt.Sprint(c)) + } + } + return out +} + +func strsOf(v any) []string { + out := []string{} + if l, ok := v.([]any); ok { + for _, c := range l { + out = append(out, fmt.Sprint(c)) + } + } + return out +} + +// SECURITY (an honest beat): a tcp pairing's beat lists in lane_serves what +// its serve lane carries, and advertises a model lane only when its service +// is there. With ollama answering here, llm unset and pii set to another +// computer, the beat carries lane_serves ["ollama"] and `embed` with its +// models, and no `llm-openai`, `pii`, pii_port, llm_port, `services` or +// `coders`, none of which runs on a tcp pairing. With all three served, all +// three are listed and advertised, pii_ok with them and pii_port still not. A +// unix pairing's beat lists its lane too and keeps `coders` (its runs have a +// socket); a tunnel pairing's beat has no lane_serves. +// +// MUTATION: have heartbeatPayload call buildHeartbeatBody (the tunnel's beat). +// RESULT: red — no lane_serves, and pii and services advertised with nothing +// serving them. +func TestATCPPairingsBeatAdvertisesOnlyWhatItServes(t *testing.T) { + up := newFakeOllama(t) + t.Setenv("OLLAMA_ADDR", up.srv.Listener.Addr().String()) + t.Setenv("PRIVATAE_LLM_ADDR", "") + t.Setenv("PRIVATAE_PII_ADDR", "192.0.2.10:9") + stubServiceRuntime(t, "docker") + stubCoders(t) + h, cfg := serveLaneBox(t, "beat.example.test") + coderOn(t, 2) + if !servicesCapable(cfg.HostName) || coderHeartbeat(cfg.HostName) == nil { + t.Fatal("the slots and coder runs must be available on this machine, or their absence proves nothing") + } + runInBackground(t, cfg) + beat := h.beat(t) + caps := capsOf(beat) + if got := strsOf(beat["lane_serves"]); !slices.Equal(got, []string{"ollama"}) { + t.Fatalf("lane_serves: %q (the beat: %v)", got, beat) + } + if !slices.Contains(caps, "embed") || !slices.Contains(strsOf(beat["models"]), "qwen4-embed:latest") { + t.Fatalf("ollama is served, so embed and its models are advertised: %q %v", caps, beat["models"]) + } + for _, c := range []string{"llm-openai", "pii", serviceCapability, coderCapability} { + if slices.Contains(caps, c) { + t.Errorf("a tcp pairing advertises %q, which nothing serves on it: %q", c, caps) + } + } + for _, k := range []string{"pii_port", "pii_ok", "services", "coders"} { + if _, ok := beat[k]; ok { + t.Errorf("a tcp pairing's beat carries %q: %v", k, beat[k]) + } + } + if beat["llm_port"] != float64(0) { + t.Errorf("no port listens for the llm server on a tcp pairing: llm_port %v", beat["llm_port"]) + } + + // All three served. + health := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(`{"status":"ok"}`)) + })) + t.Cleanup(health.Close) + t.Setenv("PRIVATAE_LLM_ADDR", up.srv.Listener.Addr().String()) + t.Setenv("PRIVATAE_PII_ADDR", health.Listener.Addr().String()) + var all map[string]any + if err := json.Unmarshal([]byte(heartbeatPayload(cfg, nil, "box", "pub")), &all); err != nil { + t.Fatal(err) + } + if got := strsOf(all["lane_serves"]); !slices.Equal(got, []string{"ollama", "llm", "pii"}) { + t.Fatalf("lane_serves with all three set: %q", got) + } + caps = capsOf(all) + for _, c := range []string{"embed", "llm-openai", "pii"} { + if !slices.Contains(caps, c) { + t.Errorf("%q is served, so it is advertised: %q", c, caps) + } + } + if all["pii_ok"] != true || all["pii_port"] != nil || !slices.Contains(strsOf(all["llm_models"]), "qwen4-embed") { + t.Errorf("pii_ok %v, pii_port %v, llm_models %v", all["pii_ok"], all["pii_port"], all["llm_models"]) + } + + // A unix pairing: its lane, and its coder runs (a socket each). + unix := *cfg + unix.Transport, unix.LaneAddr, unix.LaneSocket, unix.RunsDir = transportUnix, "", "/run/privatae-host/lane.sock", "/run/privatae-host/runs" + var u map[string]any + _ = json.Unmarshal([]byte(heartbeatPayload(&unix, nil, "box", "pub")), &u) + if _, ok := u["lane_serves"]; !ok || slices.Contains(capsOf(u), serviceCapability) || !slices.Contains(capsOf(u), coderCapability) { + t.Errorf("a unix pairing: lane_serves %v, caps %q", u["lane_serves"], capsOf(u)) + } + // A tunnel pairing: the forwards, no lane. + tunnel := *cfg + tunnel.Transport, tunnel.LaneAddr = "", "" + var tb map[string]any + _ = json.Unmarshal([]byte(heartbeatPayload(&tunnel, nil, "box", "pub")), &tb) + if _, ok := tb["lane_serves"]; ok || tb["pii_port"] != float64(piiForwardPort) { + t.Errorf("a tunnel pairing: lane_serves %v, pii_port %v", tb["lane_serves"], tb["pii_port"]) + } +} + +// The lane writes one audit line per job: the service, the address dialled +// and the bytes each way for a served one; the reason for a refused one. +func TestTheServeLaneAuditsEachJob(t *testing.T) { + up := newFakeOllama(t) + t.Setenv("OLLAMA_ADDR", up.srv.Listener.Addr().String()) + h, cfg := serveLaneBox(t, "audit.example.test") + runInBackground(t, cfg) + ws := h.take(t) + if got := ask(t, ws, "ollama"); got != "A" { + t.Fatalf("accept: %q", got) + } + conn := laneConn(ws) + req, _ := http.NewRequest(http.MethodGet, "http://serve/api/tags", nil) + req.Close = true + _ = req.Write(conn) + resp, err := http.ReadResponse(bufio.NewReader(conn), req) + if err != nil { + t.Fatal(err) + } + _, _ = io.ReadAll(resp.Body) + ws.Close() + if got := ask(t, h.take(t), "10.0.0.2:11434"); !strings.HasPrefix(got, "E") { + t.Fatalf("refuse: %q", got) + } + path := filepath.Join(cfg.dcnDir(), "audit.jsonl") + deadline := time.Now().Add(5 * time.Second) + for { + raw, _ := os.ReadFile(path) + var served, refused map[string]any + for _, line := range strings.Split(strings.TrimSpace(string(raw)), "\n") { + var m map[string]any + if json.Unmarshal([]byte(line), &m) != nil || m["op"] != "serve" { + continue + } + switch m["decision"] { + case "served": + served = m + case "refused": + refused = m + } + } + if served != nil && refused != nil { + if served["service"] != "ollama" || served["target"] != up.srv.Listener.Addr().String() || + served["bytes_from_host"].(float64) <= 0 || served["bytes_to_host"].(float64) <= 0 { + t.Fatalf("the served line: %v", served) + } + if !strings.Contains(fmt.Sprint(refused["reason"]), "never takes an address") { + t.Fatalf("the refused line: %v", refused) + } + return + } + if time.Now().After(deadline) { + t.Fatalf("no served and refused lines in %s: %s", path, raw) + } + time.Sleep(50 * time.Millisecond) + } +} diff --git a/client-go/servelane_models_linux_test.go b/client-go/servelane_models_linux_test.go new file mode 100644 index 0000000..fbf4ee6 --- /dev/null +++ b/client-go/servelane_models_linux_test.go @@ -0,0 +1,240 @@ +//go:build linux + +// Model management for a Host with no platform (modelswap.go through +// modelDaemonTarget): the model-swap probe, model:list, model:pull and +// model:drop reach only a model server on this machine, as the serve lane +// does, and the machine's console says why one is not used. Uses the +// fixtures of servelane_linux_test.go and modelswap_test.go. + +package main + +import ( + "bytes" + "os" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/gorilla/websocket" +) + +// captureStdout collects what this process prints until the test ends, and +// returns a func that reads it so far. +func captureStdout(t *testing.T) func() string { + t.Helper() + r, w, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + old := os.Stdout + os.Stdout = w + var mu sync.Mutex + var buf bytes.Buffer + done := make(chan struct{}) + go func() { + defer close(done) + b := make([]byte, 4096) + for { + n, err := r.Read(b) + if n > 0 { + mu.Lock() + buf.Write(b[:n]) + mu.Unlock() + _, _ = old.Write(b[:n]) + } + if err != nil { + return + } + } + }() + t.Cleanup(func() { + os.Stdout = old + _ = w.Close() + <-done + _ = r.Close() + }) + return func() string { + mu.Lock() + defer mu.Unlock() + return buf.String() + } +} + +// SECURITY: a tcp pairing (a Host with no platform) is not given management of +// a model server on another computer. With OLLAMA_ADDR on TEST-NET, model:list, +// an approved model:pull and model:drop of a model this Host placed are each +// refused with the reason, before anything is dialled or downloaded, and the +// machine's console says in plain words that OLLAMA_ADDR is not used and what +// to set. With OLLAMA_ADDR on this machine the same pairing lists its models. +// +// MUTATION: modelDaemonTarget returns localOllamaAddr() for every pairing. +// RESULT: red — model:list fails dialling 192.0.2.10, the approved pull +// starts a download job (exit 0), and no line is printed at the machine. +func TestModelManagementOnALocalPairingReachesOnlyThisMachine(t *testing.T) { + console := captureStdout(t) + t.Setenv("OLLAMA_ADDR", "192.0.2.10:11434") // TEST-NET-1: never this machine + t.Setenv("PRIVATAE_LLM_ADDR", "") + t.Setenv("PRIVATAE_PII_ADDR", "") + resetModelDaemonCache() + h, cfg := serveLaneBox(t, "models.example.test") + runInBackground(t, cfg) + beat := h.beat(t) + for _, c := range capsOf(beat) { + if c == modelSwapCapability { + t.Fatalf("a Host with no platform is offered management of a server on another computer: %q", capsOf(beat)) + } + } + + root := t.TempDir() + start := time.Now() + _, errOut, code, _ := driveModelOp(t, cfg.HostName, "model:list", nil, root) + if code == 0 || !strings.Contains(errOut, "OLLAMA_ADDR") || !strings.Contains(errOut, "192.0.2.10 is not this machine") { + t.Fatalf("model:list: exit %d, %q; want a refusal naming OLLAMA_ADDR and why", code, errOut) + } + if d := time.Since(start); d > 2*time.Second { + t.Errorf("model:list took %v: it waited on a dial instead of refusing first", d) + } + + enrollFactor(t, root) + approval := proposeAndApprove(t, root, "nomic-embed-text", "embed", 1<<30, "") + _, errOut, code, _ = driveModelOp(t, cfg.HostName, "model:pull", + []string{pullArg("nomic-embed-text", "embed", 1<<30, "", approval)}, root) + if code != 126 || !strings.Contains(errOut, "was not downloaded") || !strings.Contains(errOut, "192.0.2.10 is not this machine") { + t.Fatalf("model:pull: exit %d, %q; want 126 and a refusal saying why", code, errOut) + } + modelPullJobsMu.Lock() + for id, j := range modelPullJobs { + if j.Model == "nomic-embed-text" { + t.Errorf("a pull job %s was started for a server on another computer", id) + } + } + modelPullJobsMu.Unlock() + + if err := recordPlaced(root, "placed-here", placedModel{PlacedAt: "now", Role: "embed"}); err != nil { + t.Fatal(err) + } + _, errOut, code, _ = driveModelOp(t, cfg.HostName, "model:drop", []string{`{"model":"placed-here"}`}, root) + if code == 0 || !strings.Contains(errOut, "192.0.2.10 is not this machine") { + t.Fatalf("model:drop: exit %d, %q; want a refusal saying why", code, errOut) + } + if !placedByThisHost(root, "placed-here") { + t.Error("a refused drop changed the ledger") + } + + said := console() + if !strings.Contains(said, "["+cfg.HostName+"] OLLAMA_ADDR=192.0.2.10:11434 is not used for the Host on this computer") || + !strings.Contains(said, "Set OLLAMA_ADDR to a server on this computer, or unset it.") { + t.Errorf("the machine's console does not say why OLLAMA_ADDR is not used:\n%s", said) + } + // Counted for this pairing only: a pairing from an earlier test whose beat + // is still ending can say it for itself. + if n := strings.Count(said, "["+cfg.HostName+"] OLLAMA_ADDR=192.0.2.10:11434 is not used"); n != 1 { + t.Errorf("the console said it %d times; once, until it changes", n) + } + + // The control: a model server on this machine is listed for the same pairing. + newFakeDaemon(t, &fakeDaemon{tags: []string{"here-model"}}) + out, errOut, code, _ := driveModelOp(t, cfg.HostName, "model:list", nil, root) + if code != 0 || !strings.Contains(out, "here-model") { + t.Fatalf("model:list of a server on this machine: exit %d, %q %q", code, out, errOut) + } +} + +// The beat and a model op can refuse the same target at the same moment. The +// machine's console still says it once: the reason is recorded and said by one +// caller, not checked by each and then said by all. +// +// MUTATION: in serveTargetNotice, Load the last reason, then Store and print +// (as it did). +// RESULT: red — more than one line for 32 callers released together (the +// count varies from run to run). +func TestTheConsoleSaysARefusedTargetOnceUnderConcurrentCallers(t *testing.T) { + console := captureStdout(t) + s, _ := serveServiceNamed("ollama") + self := "notice-once.example.test" + t.Cleanup(func() { serveTargetNoticeClear(self, s) }) + start := make(chan struct{}) + var wg sync.WaitGroup + for i := 0; i < 32; i++ { + wg.Add(1) + go func() { + defer wg.Done() + <-start + serveTargetNotice(self, s, "192.0.2.10:11434", "192.0.2.10 is not this machine") + }() + } + close(start) + wg.Wait() + serveTargetNotice(self, s, "192.0.2.10:11434", "another reason") + deadline := time.Now().Add(2 * time.Second) + for !strings.Contains(console(), "another reason") && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + } + said := console() + if n := strings.Count(said, "["+self+"] OLLAMA_ADDR=192.0.2.10:11434 is not used"); n != 2 { + t.Fatalf("the console said it %d times; once for the first reason and once when it changed:\n%s", n, said) + } +} + +// A tunnel pairing keeps OLLAMA_ADDR wherever it points, as its forwards do: +// the rule is for a Host with no platform. +func TestModelDaemonTargetKeepsATunnelPairingsAddress(t *testing.T) { + t.Setenv("OLLAMA_ADDR", "192.0.2.10:11434") + if addr, why := modelDaemonTarget("tunnel.example.test"); addr != "192.0.2.10:11434" || why != "" { + t.Fatalf("a tunnel pairing: %q %q", addr, why) + } + unmark := markLocalPairing("tunnel.example.test") + if addr, why := modelDaemonTarget("tunnel.example.test"); addr != "" || !strings.Contains(why, "192.0.2.10 is not this machine") { + t.Fatalf("the same name run as a local pairing: %q %q", addr, why) + } + unmark() + unmark() // once only + if isLocalPairing("tunnel.example.test") { + t.Fatal("still marked local after its run ended") + } +} + +// SECURITY (this machine's resources): a Host that answers every parked serve +// worker with a job it knows is refused (a malformed frame, a name the lane +// does not carry) cannot make this machine dial, handshake and audit in a +// loop. Each refused worker waits before it parks again, so two workers cost +// at most a few dials a second. +// +// MUTATION: spawn the replacement when the first frame arrives (taken() before +// the hello is read), as the lane first did. +// RESULT: red — hundreds of workers park in the 2 s window. +func TestARefusingHostCannotSpinTheServeWorkers(t *testing.T) { + t.Setenv("OLLAMA_ADDR", "") + h, cfg := serveLaneBox(t, "spin.example.test") + runInBackground(t, cfg) + var answered atomic.Int32 + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + var ws *websocket.Conn + select { + case ws = <-h.workers: + case <-time.After(time.Until(deadline)): + } + if ws == nil { + break + } + frame := sFrame("nas") + if answered.Load()%2 == 1 { + frame = []byte("Dnot a hello") + } + _ = ws.WriteMessage(websocket.BinaryMessage, frame) + _ = ws.SetReadDeadline(time.Now().Add(2 * time.Second)) + _, _, _ = ws.ReadMessage() + ws.Close() + answered.Add(1) + } + // 2 parked at the start, then each worker once a second: about 6 in 2 s. + if n := answered.Load(); n > 10 { + t.Fatalf("%d serve workers parked in 2 s on a Host that refuses every job: the pool re-dials in a loop", n) + } + if answered.Load() < 2 { + t.Fatalf("only %d workers parked: the pool did not run, so the count proves nothing", answered.Load()) + } +} diff --git a/client-go/servelane_netns_linux_test.go b/client-go/servelane_netns_linux_test.go new file mode 100644 index 0000000..0b5f7a0 --- /dev/null +++ b/client-go/servelane_netns_linux_test.go @@ -0,0 +1,274 @@ +//go:build linux + +package main + +import ( + "fmt" + "net" + "os" + "os/exec" + "runtime" + "strings" + "sync/atomic" + "syscall" + "testing" + "time" + + "golang.org/x/sys/unix" +) + +const serveNetnsEnv = "PRIVATAE_TEST_SERVE_NETNS" + +// otherComputer is a computer on this machine's LAN, at 10.0.0.2 in a network +// namespace of its own, joined to this one by a veth that carries this +// machine's default route (so 10.0.0.2 is the LAN, not a container here). It +// listens on 80 and 11434 and counts what reaches it off the wire: IPv4 +// packets addressed to it, TCP SYNs to any address (IPv4 and IPv6), and +// accepted connections. +type otherComputer struct { + toIt atomic.Int64 + syns atomic.Int64 + accepts atomic.Int64 + stop atomic.Bool +} + +// startOtherComputer makes the namespace on one OS thread locked for good +// (the thread ends with its goroutine), so every socket it opens and every +// ip(8) it runs are the other computer's, and this test's are not. +func startOtherComputer(t *testing.T, ipBin string) *otherComputer { + t.Helper() + oc := &otherComputer{} + t.Cleanup(func() { oc.stop.Store(true) }) + errc := make(chan error, 1) + go func() { + runtime.LockOSThread() + // This test's namespace, held open before the thread leaves it: the + // veth's other end goes there. Not /proc//ns/net, which is the + // main thread's, and the main thread may be this one. + here, err := os.Open("/proc/thread-self/ns/net") + if err != nil { + errc <- err + return + } + defer here.Close() + if err := unix.Unshare(unix.CLONE_NEWNET); err != nil { + errc <- fmt.Errorf("unshare: %w", err) + return + } + for _, a := range [][]string{ + {"link", "add", "vB", "type", "veth", "peer", "name", "vA", "netns", "/proc/self/fd/3"}, + {"link", "set", "lo", "up"}, + {"addr", "add", "10.0.0.2/24", "dev", "vB"}, + {"link", "set", "vB", "up"}, + } { + cmd := exec.Command(ipBin, a...) + cmd.ExtraFiles = []*os.File{here} + if out, err := cmd.CombinedOutput(); err != nil { + errc <- fmt.Errorf("ip %s: %v %s", strings.Join(a, " "), err, out) + return + } + } + for _, port := range []string{"80", "11434"} { + ln, err := net.Listen("tcp", "10.0.0.2:"+port) + if err != nil { + errc <- err + return + } + go func() { + for !oc.stop.Load() { + c, err := ln.Accept() + if err != nil { + return + } + oc.accepts.Add(1) + c.Close() + } + }() + t.Cleanup(func() { ln.Close() }) + } + ifi, err := net.InterfaceByName("vB") + if err != nil { + errc <- err + return + } + proto := int(htons16(unix.ETH_P_ALL)) + fd, err := unix.Socket(unix.AF_PACKET, unix.SOCK_RAW, proto) + if err != nil { + errc <- fmt.Errorf("packet socket: %w", err) + return + } + if err := unix.Bind(fd, &unix.SockaddrLinklayer{Protocol: htons16(unix.ETH_P_ALL), Ifindex: ifi.Index}); err != nil { + unix.Close(fd) + errc <- fmt.Errorf("bind the packet socket: %w", err) + return + } + tv := unix.NsecToTimeval((200 * time.Millisecond).Nanoseconds()) + _ = unix.SetsockoptTimeval(fd, unix.SOL_SOCKET, unix.SO_RCVTIMEO, &tv) + go oc.capture(fd) + errc <- nil + }() + if err := <-errc; err != nil { + t.Skipf("the other computer's namespace could not be made: %v", err) + } + return oc +} + +func htons16(v uint16) uint16 { return v<<8 | v>>8 } + +// capture counts the frames arriving on vB (its own replies are left out). +func (oc *otherComputer) capture(fd int) { + defer unix.Close(fd) + buf := make([]byte, 65536) + for !oc.stop.Load() { + n, from, err := unix.Recvfrom(fd, buf, 0) + if err != nil { + continue + } + if ll, ok := from.(*unix.SockaddrLinklayer); ok && ll.Pkttype == unix.PACKET_OUTGOING { + continue + } + f := buf[:n] + if len(f) < 14 { + continue + } + switch uint16(f[12])<<8 | uint16(f[13]) { + case 0x0800: + if len(f) < 34 { + continue + } + ihl := int(f[14]&0x0f) * 4 + if net.IP(f[30:34]).Equal(net.IPv4(10, 0, 0, 2)) { + oc.toIt.Add(1) + } + if f[23] == 6 && len(f) >= 14+ihl+14 && isSYN(f[14+ihl+13]) { + oc.syns.Add(1) + } + case 0x86dd: + if len(f) >= 14+40+14 && f[20] == 6 && isSYN(f[14+40+13]) { + oc.syns.Add(1) + } + } + } +} + +func isSYN(flags byte) bool { return flags&0x02 != 0 && flags&0x10 == 0 } + +// SECURITY, IN A REAL NETWORK NAMESPACE: a job that names an address instead +// of a service, a name the lane does not carry, and each service set on this +// machine to a computer on its LAN (OLLAMA_ADDR, PRIVATAE_LLM_ADDR and +// PRIVATAE_PII_ADDR all at 10.0.0.2) each answer 'E', and the other computer +// sees no packet addressed to it and no SYN at all, from the lane or from the +// beat that probes what the lane serves, the beat's model-management probe, or +// model:list. A direct dial from this machine +// afterwards is seen, so the count is real. Runs the test binary again in a +// private user and network namespace; skipped where one cannot be made. +// +// MUTATION: in serveVetTarget, skip the ownDestination check. +// RESULT: red — the other computer sees the lane's SYN on 11434 and 80. +// MUTATION: modelDaemonTarget returns localOllamaAddr() for every pairing. +// RESULT: red — the beat's model-swap probe and model:list SYN 10.0.0.2:11434. +func TestTheServeLaneSendsNoPacketForAnAddressInARealNamespace(t *testing.T) { + if os.Getenv(serveNetnsEnv) != "1" { + if _, err := exec.LookPath("ip"); err != nil { + if _, err := os.Stat("/usr/sbin/ip"); err != nil { + t.Skip("needs ip(8)") + } + } + cmd := exec.Command(os.Args[0], "-test.run=^TestTheServeLaneSendsNoPacketForAnAddressInARealNamespace$", + "-test.v", "-test.count=1") + cmd.Env = append(os.Environ(), serveNetnsEnv+"=1") + cmd.SysProcAttr = &syscall.SysProcAttr{ + Cloneflags: syscall.CLONE_NEWUSER | syscall.CLONE_NEWNET, + UidMappings: []syscall.SysProcIDMap{{ContainerID: 0, HostID: os.Getuid(), Size: 1}}, + GidMappings: []syscall.SysProcIDMap{{ContainerID: 0, HostID: os.Getgid(), Size: 1}}, + } + out, err := cmd.CombinedOutput() + if cmd.ProcessState == nil { + t.Skipf("no private user and network namespace here: %v", err) + } + t.Logf("in a private network namespace:\n%s", out) + if strings.Contains(string(out), "--- SKIP") { + t.Skip("the namespaced run could not set up its network") + } + if err != nil { + t.Fatalf("the namespaced run failed: %v", err) + } + if !strings.Contains(string(out), "--- PASS: TestTheServeLaneSendsNoPacketForAnAddressInARealNamespace") { + t.Fatal("the namespaced run did not run") + } + return + } + + ipBin, err := exec.LookPath("ip") + if err != nil { + ipBin = "/usr/sbin/ip" + } + if out, err := exec.Command(ipBin, "link", "set", "lo", "up").CombinedOutput(); err != nil { + t.Skipf("ip link set lo up: %v %s", err, out) + } + oc := startOtherComputer(t, ipBin) + for _, a := range [][]string{ + {"addr", "add", "10.0.0.1/24", "dev", "vA"}, + {"link", "set", "vA", "up"}, + // This machine's way out: 10.0.0.2 is its LAN (and its router), not a + // container on a veth of its own (ownaddr.go). + {"route", "add", "default", "via", "10.0.0.2", "dev", "vA"}, + } { + if out, err := exec.Command(ipBin, a...).CombinedOutput(); err != nil { + t.Skipf("ip %s: %v %s", strings.Join(a, " "), err, out) + } + } + t.Setenv("OLLAMA_ADDR", "10.0.0.2:11434") + t.Setenv("PRIVATAE_LLM_ADDR", "10.0.0.2:80") + t.Setenv("PRIVATAE_PII_ADDR", "10.0.0.2:80") + h, cfg := serveLaneBox(t, "netns.example.test") + // `models` stays on: the model-management probe in the beat and model:list + // (modelswap.go, modelDaemonTarget) are held to this machine for a Host + // with no platform too, so they are in the count. + resetModelDaemonCache() + runInBackground(t, cfg) + beat := h.beat(t) + if got := strsOf(beat["lane_serves"]); len(got) != 0 { + t.Fatalf("nothing set here is this machine's, so nothing is served: %q", got) + } + for _, c := range capsOf(beat) { + if c == modelSwapCapability { + t.Fatalf("model management of the other computer is offered: %q", capsOf(beat)) + } + } + if _, errOut, code, _ := driveModelOp(t, cfg.HostName, "model:list", nil, ""); code == 0 || + !strings.Contains(errOut, "10.0.0.2 is not this machine") { + t.Errorf("model:list of the other computer: exit %d, %q", code, errOut) + } + for service, want := range map[string]string{ + "10.0.0.2:80": "never takes an address", + "10.0.0.2:11434": "never takes an address", + "nas": "is not a service", + "ollama": "10.0.0.2 is not this machine", + "llm": "10.0.0.2 is not this machine", + "pii": "10.0.0.2 is not this machine", + } { + got := ask(t, h.take(t), service) + if !strings.HasPrefix(got, "Erefused: ") || !strings.Contains(got, want) { + t.Errorf("%q: answered %q, want E carrying %q", service, got, want) + } + } + time.Sleep(500 * time.Millisecond) + if a, s, c := oc.toIt.Load(), oc.syns.Load(), oc.accepts.Load(); a != 0 || s != 0 || c != 0 { + t.Fatalf("the other computer saw %d packets addressed to it, %d SYNs and %d connections", a, s, c) + } + // The count is real: a dial from this machine is seen. + c, err := net.DialTimeout("tcp", "10.0.0.2:80", 3*time.Second) + if err != nil { + t.Fatalf("the control dial to the other computer failed, so the count proves nothing: %v", err) + } + c.Close() + deadline := time.Now().Add(3 * time.Second) + for oc.toIt.Load() == 0 || oc.syns.Load() == 0 || oc.accepts.Load() == 0 { + if time.Now().After(deadline) { + t.Fatalf("the control dial was not counted: %d packets, %d SYNs, %d connections", + oc.toIt.Load(), oc.syns.Load(), oc.accepts.Load()) + } + time.Sleep(20 * time.Millisecond) + } +} diff --git a/client-go/session.go b/client-go/session.go index b5f94c5..cb4852a 100644 --- a/client-go/session.go +++ b/client-go/session.go @@ -113,13 +113,26 @@ func runSessionWorkers(pctx context.Context, self string, wsURL string, tlsCfg * // session; an H frame is refused with a plain line (harness jobs belong on the // harness workers) so a host that routes wrongly learns it at once. func handleSessionWorkerConn(ws *websocket.Conn, self string, senv sessionEnv) { - mt, data, err := ws.ReadMessage() - if err != nil || mt != websocket.BinaryMessage || len(data) == 0 { + // The spec is read up to sessionSpecMaxBytes (lane_bounds.go) and refused + // past it; a frame of any other tag is not read at all. + tag, body, tooBig, err := readLaneJob(ws, func(t byte) int64 { + if t == 'S' { + return sessionSpecMaxBytes + } + return 0 + }) + if err != nil { return } - switch data[0] { + switch tag { case 'S': - handleSessionConn(ws, data, self, senv) + if tooBig { + sendSession(ws, 'E', []byte(fmt.Sprintf("the session spec is more than the %d KiB a session may ask with; nothing of it was read\n", + sessionSpecMaxBytes>>10))) + sendSessionExit(ws, 1) + return + } + handleSessionConn(ws, append([]byte{'S'}, body...), self, senv) case 'H': sendSession(ws, 'E', []byte("this is a session worker (lane=session); harness jobs go to a harness worker\n")) sendSessionExit(ws, 1) @@ -573,23 +586,29 @@ func handleSessionConn(ws *websocket.Conn, first []byte, self string, env sessio closed := make(chan struct{}) go func() { defer close(closed) + // Each frame is streamed (lane_bounds.go): input goes into the pty as it + // arrives, never held whole, and a frame is at most sessionInputMaxBytes. + ws.SetReadLimit(sessionInputMaxBytes) for { - mt, data, err := ws.ReadMessage() + mt, r, err := ws.NextReader() if err != nil { return } - if mt != websocket.BinaryMessage || len(data) == 0 { - continue + if mt != websocket.BinaryMessage { + continue // NextReader drops what is left of it } - switch data[0] { + var tag [1]byte + if _, err := io.ReadFull(r, tag[:]); err != nil { + continue // an empty frame + } + switch tag[0] { case 'I': lastInput.Store(time.Now().UnixNano()) - if len(data) > 1 { - _, _ = proc.Write(data[1:]) - } + _, _ = io.Copy(proc, r) case 'R': var rz sessionResize - if json.Unmarshal(data[1:], &rz) == nil && rz.Cols > 0 && rz.Rows > 0 { + b, _ := io.ReadAll(io.LimitReader(r, sessionResizeMaxBytes)) + if json.Unmarshal(b, &rz) == nil && rz.Cols > 0 && rz.Rows > 0 { _ = proc.Resize(rz.Cols, rz.Rows) } } diff --git a/deploy/VERSION b/deploy/VERSION index 9dd7871..2f4c74e 100644 --- a/deploy/VERSION +++ b/deploy/VERSION @@ -1 +1 @@ -0.55.3 +0.56.0 diff --git a/deploy/fuzz.sh b/deploy/fuzz.sh index d86c126..b477642 100755 --- a/deploy/fuzz.sh +++ b/deploy/fuzz.sh @@ -12,14 +12,102 @@ # An input that breaks a target is written under client-go/testdata/fuzz// # and the run stops there. Commit that file with the fix: from then on it is one # of the seeds every `go test` runs. +# +# The one failure this script does not stop on is Go's own timer, and only on +# the Go releases that have the race behind it (go_has_fuzztime_race below). +# When -fuzztime runs out there is a moment in which the fuzzer's parent +# context has the deadline error and its child context does not yet. The check +# in Go's fuzz coordinator that drops a cancellation error compares the error +# only with the child's (internal/fuzz/fuzz.go, `err == fuzzCtx.Err()`), so the +# deadline is reported as a failure instead of a clean stop. Go's fix is in +# 1.27 and is due in 1.26.11. The run then ends with exactly +# +# --- FAIL: (N.NNs) +# context deadline exceeded +# +# and nothing else, whatever the target does. A run is counted as a pass only +# when ALL of these hold; any other failure is a failure: +# - the go in PATH is a release with the race. On any other, including one +# whose version cannot be read, the timer report is a failure like any +# other, and deploy/test_fuzz.sh fails until this workaround is removed; +# - the output carries none of the lines Go prints for a failing input +# ("Failing input written to", "To re-run:", "fuzz: minimizing", +# "failure while testing seed corpus entry"), wherever they appear; +# - the output, less the fuzzer's progress lines and go test's closing +# FAIL lines, is those two lines and no more (a crash, a panic, a t.Fatal +# message, a seed that fails: each adds a line, so none of them can look +# like the timer); +# - no file under any testdata/fuzz directory of the module was added or +# changed during the run. A crash still being minimized when the race hits +# is written there by Go with only the deadline error as its message, so +# the directories are checked, not just the text. Go writes a failing +# input nowhere else: its build-cache corpus only ever receives inputs +# that widened coverage, which most passing runs add, so it is not a sign +# of a failure. +# deploy/test_fuzz.sh checks each of these, with planted failing inputs. set -euo pipefail HERE="$(cd "$(dirname "$0")/.." && pwd)" SECS="${1:-20}" cd "$HERE/client-go" + +log="$(mktemp)" +trap 'rm -f "$log"' EXIT + +# go_has_fuzztime_race: the go in PATH is a release that reports its own +# -fuzztime deadline as a failure. The module needs Go 1.26 or later +# (client-go/go.mod), and the fix is due in 1.26.11, so only 1.26.0 to 1.26.10 +# qualify. When the image moves past them, remove go_has_fuzztime_race, its +# use below and the stand-in cases in deploy/test_fuzz.sh, which fails on such +# a Go to say so. If a later Go still shows the race, add it here instead. +go_has_fuzztime_race() { + case "$(go env GOVERSION 2>/dev/null || true)" in + go1.26.[0-9] | go1.26.10) return 0 ;; + esac + return 1 +} + +# Every file under any testdata/fuzz directory of the module, with its +# checksum, in a fixed order. +corpus_state() { + find . -path '*/testdata/fuzz/*' -type f -exec cksum {} + | LC_ALL=C sort -k3 +} + +# crash_reported: $log carries a line Go prints only for a failing input. +crash_reported() { + grep -q -i -E -e 'failing input written to' -e 'to re-run:' \ + -e 'fuzz: minimizing' -e 'failure while testing seed corpus entry' "$log" +} + +# deadline_only : the output in $log is Go's timer report and nothing +# else. The fuzzer's progress lines and go test's closing "FAIL", "exit status" +# and "FAIL