1210 lines
50 KiB
Go
1210 lines
50 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"crypto/tls"
|
|
"crypto/x509"
|
|
"encoding/base64"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net"
|
|
"net/http"
|
|
"net/netip"
|
|
"os"
|
|
"os/signal"
|
|
"runtime"
|
|
"strings"
|
|
"sync"
|
|
"syscall"
|
|
"time"
|
|
|
|
"golang.org/x/crypto/curve25519"
|
|
"golang.zx2c4.com/wireguard/conn"
|
|
"golang.zx2c4.com/wireguard/device"
|
|
"golang.zx2c4.com/wireguard/tun/netstack"
|
|
)
|
|
|
|
// clientVersion is stamped at build time via -ldflags "-X main.clientVersion=<ver>" (see
|
|
// deploy/publish.sh + deploy/VERSION). The "dev" default flags an unstamped local build.
|
|
var clientVersion = "dev"
|
|
|
|
func b64ToHex(b64 string) (string, error) {
|
|
raw, err := base64.StdEncoding.DecodeString(b64)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return hex.EncodeToString(raw), nil
|
|
}
|
|
|
|
// cmdRun serves EVERY pairing from one process (multi-homing, 0.10.0). The old
|
|
// single-config daemon meant pairing to a new host silently unpaired the previous
|
|
// one — re-pairing a desktop to a second host killed the first host's embed lane. Now
|
|
// each pairing gets its own tunnel + local listener; the first fatal error (or a
|
|
// signal) takes the whole process down so the supervisor restarts all lanes
|
|
// together instead of limping half-homed.
|
|
func cmdRun() error {
|
|
// THE machine-wide exclusion lock, taken before any pairing starts and never
|
|
// inside runPairing. Holding it means primary; blocking on it means standby,
|
|
// and the standby simply waits in the kernel until the primary dies.
|
|
//
|
|
// It replaces an ACCIDENTAL singleton that was already here and failed badly:
|
|
// two daemons both allocate 127.0.0.1:7878, the loser dies on the bind and
|
|
// restarts every two seconds forever, flapping its WireGuard handshake each
|
|
// time, and which one wins is decided by boot order. A lock is atomic (no
|
|
// check-then-act window), immune to a stale file, kernel-released on death
|
|
// including SIGKILL, and — the property a socket probe cannot give — a HUNG
|
|
// primary keeps it, so a standby never takes over from a process that is merely
|
|
// unresponsive and leaves two daemons holding the same WireGuard keys.
|
|
// Windows first, because acquireSingletonLock below is a no-op there
|
|
// (singletonSupported() is linux-only) and a second process would otherwise
|
|
// run all the way to a loopback port bind and die naming a port number.
|
|
if err := guardSingleRun(); err != nil {
|
|
return err
|
|
}
|
|
lock, err := acquireSingletonLock()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// Thaw anything a previous run froze for a kill it never finished
|
|
// (killtree.go): the primary does it once, before any pairing can order
|
|
// another.
|
|
recoverFrozenTrees()
|
|
defer lock.release()
|
|
|
|
// Write the machine marker BEFORE the not-paired return below. A fresh box is
|
|
// `enable --now` at install and crash-loops here until it is paired — and that
|
|
// is exactly the window in which a `sudo`-without-`-H` pair happens. A marker
|
|
// written after this return does not exist when it is needed.
|
|
_ = maybeWriteMarker()
|
|
|
|
cfgs, err := loadPairings()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(cfgs) == 0 {
|
|
return fmt.Errorf("not paired — run `privatae-bastion pair <token>` first")
|
|
}
|
|
|
|
// Clear any leftover from a prior self-update swap (the superseded binary can't
|
|
// delete itself while running, so it's removed on the next start).
|
|
cleanupStaleBinaries()
|
|
// An op run the last process died in the middle of: its tmpfs (with an
|
|
// action run's opened credentials), its work folder, its unit.
|
|
opStartupSweep()
|
|
// A root command's connection watch the last process died in the middle
|
|
// of: its nftables table (rootexec_watch_linux.go).
|
|
rootExecWatchSweep()
|
|
|
|
// Pairings from before the git lane (< 0.13.0) have no git proxy port: allocate
|
|
// one now and persist it, so `git-setup` and `run` agree.
|
|
for _, cfg := range cfgs {
|
|
if cfg.GitProxyAddr == "" {
|
|
cfg.GitProxyAddr = nextGitProxyAddr(cfgs)
|
|
if err := cfg.updateOnDisk(func(d *Config) { d.GitProxyAddr = cfg.GitProxyAddr }); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
errc := make(chan error, 64)
|
|
supervisor.Lock()
|
|
supervisor.ctx, supervisor.errc = ctx, errc
|
|
// Installed here, not at package init: the mgmt handler table refers to the
|
|
// grant handler, which starts pairings, which run the enforcing report,
|
|
// which reads the handler table — a static reference would be an
|
|
// initialization cycle. A runtime hook is the same call without the loop.
|
|
supervisor.start = startPairing
|
|
supervisor.Unlock()
|
|
fmt.Printf("mode: %s\n", boxMode())
|
|
if a := currentAnchor(); a != nil {
|
|
fmt.Printf("anchored to %s (owner %q)\n", a.AnchorHost, a.Owner)
|
|
}
|
|
started := 0
|
|
for _, cfg := range cfgs {
|
|
if startPairing(cfg) {
|
|
started++
|
|
}
|
|
}
|
|
if started == 0 {
|
|
return fmt.Errorf("no active pairing — every pairing is pending or revoked; the anchor host must approve one")
|
|
}
|
|
go superviseLoop(ctx)
|
|
// The local UI (ui.go): ONE per machine, not one per pairing, because it
|
|
// shows the whole box. Started last so it renders a daemon that is already
|
|
// up, and non-fatal by construction — a machine that cannot bind the UI port
|
|
// must still serve its hosts.
|
|
startUIServer(ctx)
|
|
// The tray (tray_windows.go): Windows only, a no-op elsewhere. It is the
|
|
// only part of this UI that answers "is it running" without the user
|
|
// opening anything, which is the question that started all of it.
|
|
startTray(ctx)
|
|
|
|
sig := make(chan os.Signal, 1)
|
|
signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM)
|
|
select {
|
|
case <-sig:
|
|
fmt.Println("\nshutting down")
|
|
cancel() // closes every pairing's server + WG device via runPairing's exits
|
|
return nil
|
|
case <-shutdownCh:
|
|
// "Stop the bastion" in the tray menu. Deliberately the SAME path as a
|
|
// signal rather than an os.Exit: a machine that stops serving must tear
|
|
// its tunnels down, not vanish and leave the host waiting for a beat.
|
|
fmt.Println("\nstopping — asked from the tray")
|
|
cancel()
|
|
return nil
|
|
case err := <-errc:
|
|
cancel()
|
|
return err
|
|
}
|
|
}
|
|
|
|
// The pairing supervisor: which pairings are running and how to
|
|
// stop one. Before anchoring, the set of running pairings was fixed at start;
|
|
// now a pairing grant opens a pending pairing's lanes while the daemon runs,
|
|
// and a revoke closes them, so each pairing gets its own child context here.
|
|
var supervisor struct {
|
|
sync.Mutex
|
|
ctx context.Context // cmdRun's root context; nil when the daemon is not running
|
|
errc chan error // fatal pairing errors, forwarded to cmdRun
|
|
runs map[string]*pairingRun
|
|
// start is startPairing while the daemon runs; nil otherwise. The mgmt
|
|
// grant handler calls through it (supervisorStart) so pairing_ops.go never
|
|
// references runPairing directly — see cmdRun for why.
|
|
start func(*Config) bool
|
|
}
|
|
|
|
// supervisorStart opens a pairing's lanes if the daemon is running. False from
|
|
// a CLI process (nothing to start; the next `run` picks the pairing up).
|
|
func supervisorStart(cfg *Config) bool {
|
|
supervisor.Lock()
|
|
start := supervisor.start
|
|
supervisor.Unlock()
|
|
if start == nil {
|
|
return false
|
|
}
|
|
return start(cfg)
|
|
}
|
|
|
|
type pairingRun struct {
|
|
cancel context.CancelFunc
|
|
done chan struct{}
|
|
// cfg is the copy the run was started with, and the one its tunnel, lanes
|
|
// and heartbeat hold.
|
|
cfg *Config
|
|
}
|
|
|
|
// startPairing spawns runPairing for cfg under the supervisor. Returns false
|
|
// when the pairing is not active (nothing is started for a pending or revoked
|
|
// pairing — it is a file), when it is already running, or when the daemon is
|
|
// not running (a CLI process handling a grant has nothing to start).
|
|
func startPairing(cfg *Config) bool {
|
|
if !cfg.active() {
|
|
fmt.Printf("[%s] %s — not started (no tunnel, no lanes, no dirs) until the anchor host approves it\n", cfg.HostName, cfg.stateName())
|
|
return false
|
|
}
|
|
supervisor.Lock()
|
|
defer supervisor.Unlock()
|
|
if supervisor.ctx == nil {
|
|
return false
|
|
}
|
|
if supervisor.runs == nil {
|
|
supervisor.runs = map[string]*pairingRun{}
|
|
}
|
|
if _, running := supervisor.runs[cfg.HostName]; running {
|
|
return false
|
|
}
|
|
pctx, pcancel := context.WithCancel(supervisor.ctx)
|
|
run := &pairingRun{cancel: pcancel, done: make(chan struct{}), cfg: cfg}
|
|
supervisor.runs[cfg.HostName] = run
|
|
errc := supervisor.errc
|
|
go func() {
|
|
defer close(run.done)
|
|
err := runPairing(pctx, cfg)
|
|
// Retired BEFORE it leaves the registry: leaving is what lets the
|
|
// pairing start again, and this run's heartbeat may still be running.
|
|
retireRun(cfg)
|
|
supervisor.Lock()
|
|
if supervisor.runs[cfg.HostName] == run {
|
|
delete(supervisor.runs, cfg.HostName)
|
|
}
|
|
supervisor.Unlock()
|
|
// nil is a clean exit (ctx cancelled: shutdown or a revoke) and must not
|
|
// take the process down. Anything else is fatal for the whole daemon, as
|
|
// before: the supervisor restarts all lanes together, never half-homed.
|
|
if err != nil && errc != nil {
|
|
errc <- fmt.Errorf("[%s] %w", cfg.HostName, err)
|
|
}
|
|
}()
|
|
return true
|
|
}
|
|
|
|
// superviseLoop makes the daemon follow the pairing files on disk. Every
|
|
// supervisePeriod: a pairing whose file says active but which is not running
|
|
// is started; one that is running but whose file now says pending/revoked, or
|
|
// whose file is gone, is stopped; one whose file holds another WireGuard key
|
|
// than the run (paired again) is stopped and started from the new file. This
|
|
// is how a change made by ANOTHER process — claim-anchor --rotate demoting the
|
|
// old anchor, `unpair` deleting a file, `pair --replace` writing new
|
|
// credentials, a grant handled before a restart — reaches the running daemon
|
|
// without a restart. The in-process handlers still act at once; this is the
|
|
// catch-up.
|
|
const supervisePeriod = 15 * time.Second
|
|
|
|
func superviseLoop(ctx context.Context) {
|
|
t := time.NewTicker(supervisePeriod)
|
|
defer t.Stop()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-t.C:
|
|
superviseOnce()
|
|
}
|
|
}
|
|
}
|
|
|
|
func superviseOnce() {
|
|
cfgs, err := loadPairings()
|
|
if err != nil {
|
|
return // cannot read the directory: change nothing on a read error
|
|
}
|
|
onDisk := map[string]*Config{}
|
|
for _, c := range cfgs {
|
|
onDisk[c.HostName] = c
|
|
}
|
|
supervisor.Lock()
|
|
running := make(map[string]*pairingRun, len(supervisor.runs))
|
|
for h, r := range supervisor.runs {
|
|
running[h] = r
|
|
}
|
|
supervisor.Unlock()
|
|
for h, r := range running {
|
|
c, ok := onDisk[h]
|
|
switch {
|
|
case !ok || !c.active():
|
|
fmt.Printf("[%s] no longer active on disk — stopping its tunnel and lanes\n", h)
|
|
stopPairing(h)
|
|
case r.cfg != nil && c.WGPrivateKey != r.cfg.WGPrivateKey:
|
|
// Only `pair` writes the key, and always a fresh one, so this
|
|
// is a pairing made again (`pair --replace` in another
|
|
// process, or an unpair and a pair within one period). The run
|
|
// still holds the credentials that pair rotated out; the start
|
|
// below brings the new ones up, once the old run is gone.
|
|
fmt.Printf("[%s] paired again on disk — restarting its tunnel and lanes with the new pairing\n", h)
|
|
stopPairing(h)
|
|
}
|
|
}
|
|
for _, c := range cfgs {
|
|
if c.active() {
|
|
startPairing(c) // no-op when already running
|
|
}
|
|
}
|
|
}
|
|
|
|
// stopPairing cancels one running pairing and waits (bounded) for its tunnel,
|
|
// listener and lanes to exit. Returns whether anything was running.
|
|
func stopPairing(host string) bool {
|
|
supervisor.Lock()
|
|
run, ok := supervisor.runs[host]
|
|
supervisor.Unlock()
|
|
if !ok {
|
|
return false
|
|
}
|
|
retireRun(run.cfg)
|
|
run.cancel()
|
|
select {
|
|
case <-run.done:
|
|
case <-time.After(5 * time.Second):
|
|
}
|
|
return true
|
|
}
|
|
|
|
// retireRun marks the copy a run holds as stopped. Once it returns, nothing is
|
|
// written through that copy (updateOnDiskIf, errRunStopped): the run's
|
|
// heartbeat goroutine is not waited for, and a beat stuck on a dead tunnel can
|
|
// come back after the run is gone — even after the same pairing, same key, was
|
|
// started again — and save its copy's older entry and Hub key over the new
|
|
// run's. Set under saveMu, the lock every write of a pairing file holds, so a
|
|
// write already under way finishes first and none starts after. Called before
|
|
// a run is cancelled and again before it leaves the registry, so it is always
|
|
// done before the pairing can start again.
|
|
func retireRun(cfg *Config) {
|
|
if cfg == nil {
|
|
return
|
|
}
|
|
saveMu.Lock()
|
|
cfg.retired = true
|
|
saveMu.Unlock()
|
|
}
|
|
|
|
// errPairingNotActive is runPairing's answer to a pending or revoked pairing:
|
|
// it returns before creating anything. A guard, not the normal path —
|
|
// startPairing never calls it for one.
|
|
var errPairingNotActive = errors.New("pairing is not active — nothing started")
|
|
|
|
// runPairing is the entire single-host daemon for ONE pairing — tunnel, worker
|
|
// lanes, local control listener, heartbeat — exactly the pre-0.10.0 body, per cfg.
|
|
// It returns nil when ctx is cancelled (clean shutdown) or the pairing's fatal
|
|
// error. Log lines are prefixed with the host so interleaved pairings read apart.
|
|
func runPairing(ctx context.Context, cfg *Config) error {
|
|
logf := func(format string, a ...any) { fmt.Printf("["+cfg.HostName+"] "+format, a...) }
|
|
|
|
// A pending or revoked pairing (anchor.go) is a FILE and nothing more: no
|
|
// WireGuard device, no worker lanes, no listener, and — because contextRoot()
|
|
// and dcnDir() create on read — not even its directories. This guard sits
|
|
// above every one of those so the order of the code below can never leak a
|
|
// side effect for a host the anchor has not approved.
|
|
if !cfg.active() {
|
|
return errPairingNotActive
|
|
}
|
|
// A Host with no platform: the same lanes over its socket or its lane
|
|
// port, no tunnel.
|
|
if cfg.localTransport() {
|
|
return runUnixPairing(ctx, cfg)
|
|
}
|
|
|
|
// 1. Userspace WireGuard (gVisor netstack): no TUN device, no root.
|
|
gwIP, err := netip.ParseAddr(cfg.BastionTunnelIP)
|
|
if err != nil {
|
|
return fmt.Errorf("bad gateway tunnel ip %q: %w", cfg.BastionTunnelIP, err)
|
|
}
|
|
tunDev, tnet, err := netstack.CreateNetTUN([]netip.Addr{gwIP}, []netip.Addr{}, 1420)
|
|
if err != nil {
|
|
return fmt.Errorf("create userspace tun: %w", err)
|
|
}
|
|
dev := device.NewDevice(tunDev, conn.NewDefaultBind(), device.NewLogger(device.LogLevelError, "wg: "))
|
|
if err := configureTunnelPeer(dev, cfg); err != nil {
|
|
return err
|
|
}
|
|
if err := dev.Up(); err != nil {
|
|
return fmt.Errorf("bring up wireguard: %w", err)
|
|
}
|
|
defer dev.Close()
|
|
|
|
// 1b. Expose the bastion's LOCAL ollama to the host over the tunnel so the
|
|
// embed lane works end to end. The host dials <bastion tunnel IP>:11434;
|
|
// we accept on the netstack and proxy to ollama on loopback. Without this
|
|
// the heartbeat can advertise 'embed' but the host could never reach the
|
|
// embedder — the netstack tunnel otherwise only dials OUT (lanes/heartbeat).
|
|
// ollama just runs on its default loopback; no WG adapter / firewall dance.
|
|
// All three forwards are the `inference` caps family (anchor.go): on an
|
|
// anchored box a member pairing outside it is refused at accept.
|
|
go serveOllamaForward(ctx, cfg.HostName, tnet, gwIP)
|
|
// 1c. Same, for an OpenAI-compatible server (llama.cpp/vLLM) when
|
|
// PRIVATAE_LLM_ADDR is set: host dials <bastion tunnel IP>:18080.
|
|
go serveLLMForward(ctx, cfg.HostName, tnet, gwIP)
|
|
// 1d. Same, for the PII sterilization service when PRIVATAE_PII_ADDR is
|
|
// set: host dials <bastion tunnel IP>:18081. Unset = lane absent.
|
|
go servePIIForward(ctx, cfg.HostName, tnet, gwIP)
|
|
// 1d'. The service slots (service.go): one forward per server a host built
|
|
// on this machine, host dials <bastion tunnel IP>:<tunnel_port>. The
|
|
// "services" family is checked at every accept.
|
|
go serveServiceForwards(ctx, cfg.HostName, cfg.dcnDir(), tnet, gwIP)
|
|
// 1d''. Coder runs (coders.go): one forward per run this host started
|
|
// here, host dials <bastion tunnel IP>:<tunnel_port> (18200-18299)
|
|
// and is spliced to the run's control socket. The box-wide coder
|
|
// loop (egress proxies, reaper) starts once per daemon.
|
|
coderInit(context.Background())
|
|
go serveCoderForwards(ctx, cfg.HostName, cfg.HostTunnelIP, tnet, gwIP)
|
|
// 1e. Managed local agents (freeagent_managed.go): bring up an instance for
|
|
// every synced store grant carrying 'search', so a daemon restart never
|
|
// loses a running local agent.
|
|
go faReconcileLoop(ctx, cfg.dcnDir())
|
|
|
|
// 2. mTLS client that dials the host's :8443 lanes THROUGH the tunnel
|
|
// (heartbeat, git-credential, DCN propose, the reverse-relay WebSockets).
|
|
// The bastion↔host session is end-to-end encrypted; the concentrator only
|
|
// ever sees ciphertext.
|
|
caPool := x509.NewCertPool()
|
|
if !caPool.AppendCertsFromPEM([]byte(cfg.CACert)) {
|
|
return fmt.Errorf("bad CA cert in config")
|
|
}
|
|
cert, err := tls.X509KeyPair([]byte(cfg.OperatorCert), []byte(cfg.OperatorKey))
|
|
if err != nil {
|
|
return fmt.Errorf("operator cert/key: %w", err)
|
|
}
|
|
hostAddr := net.JoinHostPort(cfg.HostTunnelIP, "8443")
|
|
transport := &http.Transport{
|
|
// Ignore the requested host; always dial the host's tunnel alias over WG.
|
|
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
|
|
return tnet.DialContext(ctx, "tcp", hostAddr)
|
|
},
|
|
TLSClientConfig: &tls.Config{
|
|
RootCAs: caPool,
|
|
Certificates: []tls.Certificate{cert},
|
|
ServerName: cfg.HostName, // SAN check against <handle>.host.privatae.ai
|
|
},
|
|
}
|
|
upstream := &http.Client{Transport: transport}
|
|
|
|
// 2a. The extraction PULL path (contextpull.go). ctx_extract used to
|
|
// take the document inline in the management frame, which caps a source
|
|
// at 8 MiB — fine for the text PDFs that shipped, not for the scanned
|
|
// ones OCR is for. The fetch reuses THIS client deliberately: its
|
|
// DialContext ignores the address in a URL and dials the host's tunnel
|
|
// endpoint, and its TLS config pins ServerName to the host name. That
|
|
// pinning is what lets contextpull.go build its own URL and carry no
|
|
// SSRF wall — unlike backup_fetch, the host never names a destination.
|
|
// Set BEFORE the mgmt workers come up, so a job cannot arrive to find
|
|
// the lane unconfigured.
|
|
setCtxSourcePuller(newHostSourcePuller(upstream, cfg.HostName, cfg.Bearer))
|
|
|
|
// Egress (reverse relay): the host can't reach the bastion's tunnel IP (the entity
|
|
// pod has no WG iface), so we open worker WebSockets TO the host over the same
|
|
// tunnel -> :8443 mTLS lane and serve egress over them — the host dispatches each
|
|
// allowed request to a worker, and we dial it out the USER's own connection.
|
|
go func() {
|
|
realDialer := &net.Dialer{Timeout: egressDialTimeout}
|
|
wsURL := fmt.Sprintf("wss://%s:8443/bastion-egress", cfg.HostName)
|
|
wsTLS := &tls.Config{RootCAs: caPool, Certificates: []tls.Certificate{cert}, ServerName: cfg.HostName}
|
|
dialHost := func(ctx context.Context) (net.Conn, error) { return tnet.DialContext(ctx, "tcp", hostAddr) }
|
|
logf("egress workers up — outbound exits THIS machine's internet via %s\n", wsURL)
|
|
runEgressWorkers(ctx, cfg.HostName, wsURL, wsTLS, dialHost, cfg.Bearer, realDialer.DialContext, egressWorkerCount)
|
|
}()
|
|
|
|
// Management (reverse relay): the same trick, a different job type — the host dispatches
|
|
// a "run this op on the server" job onto a worker; the bastion runs it locally against its
|
|
// OWN allowlist (the authoritative wall) and relays stdout/stderr/exit back. Default-DENY +
|
|
// always-audited; even a compromised pod can't make this host run an ungranted op.
|
|
go func() {
|
|
wsURL := fmt.Sprintf("wss://%s:8443/bastion-mgmt", cfg.HostName)
|
|
wsTLS := &tls.Config{RootCAs: caPool, Certificates: []tls.Certificate{cert}, ServerName: cfg.HostName}
|
|
dialHost := func(ctx context.Context) (net.Conn, error) { return tnet.DialContext(ctx, "tcp", hostAddr) }
|
|
// A process that died mid-extraction left one of the user's documents in
|
|
// the context root. Nothing else would ever notice it, so sweep before
|
|
// the lane that creates them comes up (contextextract.go).
|
|
if n := sweepStaleScratch(cfg.contextRoot()); n > 0 {
|
|
logf("context-prep: removed %d scratch file(s) left by a previous run\n", n)
|
|
}
|
|
logf("mgmt workers up — TACACS-style command lane via %s\n", wsURL)
|
|
// The backup folder is passed as a path, not created here: it can be
|
|
// a network mount that stopped answering, and the lane must come up
|
|
// regardless. Each backup op checks and creates it, with a bound
|
|
// (mgmt.go, backupRootReady).
|
|
runMgmtWorkers(ctx, cfg.HostName, wsURL, wsTLS, dialHost, cfg.Bearer, cfg.contextRoot(), cfg.dcnDir(),
|
|
cfg.backupRootPath(), mgmtWorkerCount)
|
|
}()
|
|
|
|
// Harness (harness-dispatch): the host runs its OWN agent on THIS box via the existing harness
|
|
// container, dispatched over the same reverse relay — same image, different location. Echo mode
|
|
// proves the lane; the real container lifecycle follows. NOT a homemade harness — the bastion runs
|
|
// the platform's image + relays; the mgmt allowlist governs it.
|
|
go func() {
|
|
wsURL := fmt.Sprintf("wss://%s:8443/bastion-harness", cfg.HostName)
|
|
wsTLS := &tls.Config{RootCAs: caPool, Certificates: []tls.Certificate{cert}, ServerName: cfg.HostName}
|
|
dialHost := func(ctx context.Context) (net.Conn, error) { return tnet.DialContext(ctx, "tcp", hostAddr) }
|
|
logf("harness workers up — host-dispatched agents land HERE via %s\n", wsURL)
|
|
runHarnessWorkers(ctx, cfg.HostName, cfg.dcnDir(), wsURL, wsTLS, dialHost, cfg.Bearer, harnessWorkerCount)
|
|
}()
|
|
|
|
// Sessions (session.go): terminals on THIS box for the Terminal app. Same endpoint,
|
|
// their OWN worker pool (hello `BEARER <bearer> lane=session`) so an open shell never holds
|
|
// one of the harness workers for hours.
|
|
go func() {
|
|
wsURL := fmt.Sprintf("wss://%s:8443/bastion-harness", cfg.HostName)
|
|
wsTLS := &tls.Config{RootCAs: caPool, Certificates: []tls.Certificate{cert}, ServerName: cfg.HostName}
|
|
dialHost := func(ctx context.Context) (net.Conn, error) { return tnet.DialContext(ctx, "tcp", hostAddr) }
|
|
srvName, _ := os.Hostname()
|
|
logf("session workers up — terminals on this machine via %s (lane=session)\n", wsURL)
|
|
runSessionWorkers(ctx, cfg.HostName, wsURL, wsTLS, dialHost, cfg.Bearer, sessionWorkerCount,
|
|
sessionEnv{CtxRoot: cfg.contextRoot(), DcnDir: cfg.dcnDir(), HostName: cfg.HostName, Server: srvName})
|
|
}()
|
|
|
|
// 2d. Git lane: loopback HTTP proxy for git.privatae.ai only.
|
|
// Each local connection = one WebSocket to the host, relayed out the PLATFORM
|
|
// path there (the forge answers platform/DMZ addresses only). No credential
|
|
// rides this lane; git presents its own via the git-credential helper below.
|
|
if cfg.GitProxyAddr != "" {
|
|
if gitLn, err := net.Listen("tcp", cfg.GitProxyAddr); err != nil {
|
|
// Optional lane: a port collision must not take the tunnel, lanes and
|
|
// heartbeat down with it. Log and run without git.
|
|
logf("git proxy NOT started — %s: %v (tunnel/lanes unaffected; free the port and restart)\n", cfg.GitProxyAddr, err)
|
|
} else {
|
|
// A revoked pairing must give its port back: closing the listener is
|
|
// what makes runGitLane's Accept loop return.
|
|
stopGit := context.AfterFunc(ctx, func() { _ = gitLn.Close() })
|
|
defer stopGit()
|
|
go func() {
|
|
wsURL := fmt.Sprintf("wss://%s:8443/bastion-git", cfg.HostName)
|
|
wsTLS := &tls.Config{RootCAs: caPool, Certificates: []tls.Certificate{cert}, ServerName: cfg.HostName}
|
|
dialHost := func(ctx context.Context) (net.Conn, error) { return tnet.DialContext(ctx, "tcp", hostAddr) }
|
|
logf("git proxy up — http.proxy for git.privatae.ai is http://%s (privatae-bastion git-setup)\n", cfg.GitProxyAddr)
|
|
runGitLane(gitLn, wsURL, wsTLS, dialHost, cfg.Bearer)
|
|
}()
|
|
}
|
|
}
|
|
|
|
// 3. Local control endpoint: the loopback listener the CLI verbs talk to
|
|
// (add-folder → /dcn/propose, git-credential → /git-credential). Guarded by
|
|
// a local secret so other processes on the box can't ride the tunnel.
|
|
// There is NO MCP relay here any more (removed deliberately): the
|
|
// bastion never forwards an operator client's requests to the host's
|
|
// /mcp. The host reaches this machine only through the lanes this box
|
|
// dials out and gates itself; nothing on this box drives the host.
|
|
mux := http.NewServeMux()
|
|
mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("ok\n")) })
|
|
// git-credential helper backend: relays the host's answer (the user's platform-
|
|
// minted forge token) to `privatae-bastion git-credential`. Local secret, then the
|
|
// bastion bearer over the tunnel — nothing is stored on this machine.
|
|
mux.HandleFunc("/git-credential", func(w http.ResponseWriter, r *http.Request) {
|
|
if !uiSecretMatches(strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer "), cfg.OperatorSecret) {
|
|
http.Error(w, "unauthorized", http.StatusUnauthorized)
|
|
return
|
|
}
|
|
req, err := http.NewRequest(http.MethodGet,
|
|
fmt.Sprintf("https://%s:8443/api/network/bastions/git-credential", cfg.HostName), nil)
|
|
if err != nil {
|
|
http.Error(w, "bad request", http.StatusBadRequest)
|
|
return
|
|
}
|
|
req.Header.Set("Authorization", "Bearer "+cfg.Bearer)
|
|
resp, err := upstream.Do(req)
|
|
if err != nil {
|
|
http.Error(w, "upstream: "+err.Error(), http.StatusBadGateway)
|
|
return
|
|
}
|
|
defer resp.Body.Close()
|
|
w.WriteHeader(resp.StatusCode)
|
|
_, _ = io.Copy(w, resp.Body)
|
|
})
|
|
// DCN store onboarding: `privatae-bastion add-folder` lands here; the
|
|
// draft rides the tunnel with the bearer and stays inert until the user approves
|
|
// it in their host with their step-up code.
|
|
mux.HandleFunc("/dcn/propose", dcnProposeLocal(upstream, cfg.HostName, cfg.Bearer, cfg.OperatorSecret))
|
|
|
|
// 4. Liveness heartbeat — POST to the host over the tunnel (same mTLS client) so
|
|
// the Network app shows this bastion online and self-reports device_health
|
|
// (posture). The device fingerprint is NOT sent here: it is bound at
|
|
// pairing from the operator CSR; the reverse lanes are admitted on the bearer's
|
|
// binding to that cert, which the host enforces. Authed by the bearer.
|
|
go func() {
|
|
hbURL := fmt.Sprintf("https://%s:8443/api/network/bastions/heartbeat", cfg.HostName)
|
|
hostname, _ := os.Hostname()
|
|
wgPub := ""
|
|
if raw, e := base64.StdEncoding.DecodeString(cfg.WGPrivateKey); e == nil && len(raw) == 32 {
|
|
if pub, e2 := curve25519.X25519(raw, curve25519.Basepoint); e2 == nil {
|
|
wgPub = base64.StdEncoding.EncodeToString(pub)
|
|
}
|
|
}
|
|
beat := func() {
|
|
payload := heartbeatPayload(cfg, dev, hostname, wgPub)
|
|
req, err := http.NewRequest(http.MethodPost, hbURL, strings.NewReader(payload))
|
|
if err != nil {
|
|
return
|
|
}
|
|
req.Header.Set("Content-Type", "application/json")
|
|
req.Header.Set("Authorization", "Bearer "+cfg.Bearer)
|
|
if resp, err := upstream.Do(req); err == nil {
|
|
body, _ := io.ReadAll(resp.Body)
|
|
resp.Body.Close()
|
|
if resp.StatusCode < 300 {
|
|
// last_ok per pairing (anchor.go): the anchor rotation's
|
|
// silence path reads this — 14 days with no 2xx from the
|
|
// anchor host lets root re-anchor without its consent.
|
|
recordPairingOK(cfg.HostName)
|
|
}
|
|
// Box-wide directives (update, contain, mirror) are anchor-only
|
|
// on an anchored box — each takes this pairing's identity and
|
|
// refuses a member host's directive (anchor.go anchorOnly).
|
|
maybeSelfUpdate(cfg.HostName, cfg.dcnDir(), body) // host-directed, signed, sha256-verified, approved unless newer (update.go); no-op unless an update is pending
|
|
maybeContain(cfg.HostName, cfg.dcnDir(), body) // host-directed agent termination, approved; no-op unless a contain is pending
|
|
maybeContainFindings(cfg.HostName, cfg.dcnDir(), body) // host-directed stop of a finding by fingerprint (containfinding.go)
|
|
maybeReconcileGrants(body, cfg.dcnDir()) // drop store grants the host no longer lists (reconcile.go)
|
|
maybeReconcileReach(body, cfg.dcnDir()) // and reach grants (service_reach.go)
|
|
maybeReconcileCoderGrants(body, cfg.dcnDir()) // and coder grants (coder_grant.go)
|
|
coderHeartbeatReply(cfg.HostName, body) // coder_runs + the coder reaper (coders.go)
|
|
maybeRotateEndpoint(body, cfg, dev) // host-directed WG endpoint rotation (concentrator→relay); live swap
|
|
// Memory mirror: starts the standby + its forwarder the
|
|
// first time the host says this box holds the mirror. No-op on
|
|
// every other beat, and on every bastion that is not the mirror.
|
|
maybeMirror(body, cfg, mirrorWSURL(cfg.HostName),
|
|
&tls.Config{RootCAs: caPool, Certificates: []tls.Certificate{cert}, ServerName: cfg.HostName},
|
|
func(ctx context.Context) (net.Conn, error) { return tnet.DialContext(ctx, "tcp", hostAddr) })
|
|
}
|
|
}
|
|
startedAt := time.Now()
|
|
// Host posture refreshes on its own goroutine (observe_ops.go): a beat
|
|
// reads the last value and never waits on a probe, because a late
|
|
// beat is what flips this box stale on the pod.
|
|
go runPostureRefresh(ctx)
|
|
// The Alertmanager summary (observe_pull.go): the same shape for
|
|
// the same reason — one HTTP GET a minute against the box's own
|
|
// neighbourhood, published for the beat to read. No-op on every box
|
|
// whose endpoints file names no alertmanager.
|
|
go runAlertsRefresh(ctx)
|
|
// Second-bastion detection (otherbastions.go): same reason for the same
|
|
// shape — the scan reads other users' home directories and the beat may
|
|
// never wait on disk.
|
|
go runOtherBastionRefresh(ctx)
|
|
// The delta against the accounting (deltarun.go): the same
|
|
// shape again, and here the reason is measured — the first pass on this
|
|
// box costs 2.9 s because it hashes every executable once, and no beat
|
|
// may ever wait on that.
|
|
go runDeltaRefresh(ctx)
|
|
beat() // first beat: flip online + bind fp now, don't wait a full interval
|
|
t := time.NewTicker(60 * time.Second)
|
|
defer t.Stop()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-t.C:
|
|
// Path failover BEFORE the beat: if the current relay entry has gone dead
|
|
// (no handshake for RejectAfterTime) the beat could not reach the host
|
|
// anyway; move to the next candidate first, no platform round trip.
|
|
maybeFailover(cfg, dev, startedAt)
|
|
beat()
|
|
}
|
|
}
|
|
}()
|
|
|
|
srv := &http.Server{Addr: cfg.ListenAddr, Handler: mux}
|
|
errc := make(chan error, 1)
|
|
go func() {
|
|
logf("privatae-bastion up — tunnel to %s, local control on http://%s\n", cfg.HostName, cfg.ListenAddr)
|
|
logf("add folders to your network (files stay here; Privatae Host keeps only the index):\n")
|
|
logf(" privatae-bastion add-folder ~/Documents/some-folder\n")
|
|
errc <- srv.ListenAndServe()
|
|
}()
|
|
|
|
select {
|
|
case <-ctx.Done():
|
|
_ = srv.Close()
|
|
return nil
|
|
case err := <-errc:
|
|
return err
|
|
}
|
|
}
|
|
|
|
// unixLaneDialer is the transport of a pairing with a Host on this machine
|
|
// (pair_unix.go): a dial of the Host's lane socket, and the mutual TLS every
|
|
// lane runs inside it, exactly as over the tunnel. The Host's CA is the only
|
|
// root, its certificate must carry the pairing's host name, and this
|
|
// bastion presents its operator certificate. A pairing with no bearer gets
|
|
// no lane at all.
|
|
func unixLaneDialer(cfg *Config) (func(ctx context.Context) (net.Conn, error), *tls.Config, error) {
|
|
if cfg.Bearer == "" {
|
|
return nil, nil, errors.New("this pairing holds no bearer, and its lane never opens without one")
|
|
}
|
|
if !cleanAbsSocketPath(cfg.LaneSocket) {
|
|
return nil, nil, fmt.Errorf("lane socket %q is not an absolute, clean path", cfg.LaneSocket)
|
|
}
|
|
caPool := x509.NewCertPool()
|
|
if !caPool.AppendCertsFromPEM([]byte(cfg.CACert)) {
|
|
return nil, nil, fmt.Errorf("bad CA cert in config")
|
|
}
|
|
cert, err := tls.X509KeyPair([]byte(cfg.OperatorCert), []byte(cfg.OperatorKey))
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("operator cert/key: %w", err)
|
|
}
|
|
sock := cfg.LaneSocket
|
|
dial := func(ctx context.Context) (net.Conn, error) {
|
|
return (&net.Dialer{Timeout: egressDialTimeout}).DialContext(ctx, "unix", sock)
|
|
}
|
|
return dial, &tls.Config{RootCAs: caPool, Certificates: []tls.Certificate{cert}, ServerName: cfg.HostName,
|
|
MinVersion: tls.VersionTLS12}, nil
|
|
}
|
|
|
|
// runUnixPairing is runPairing for a pairing with a Host that has no
|
|
// platform: on this machine (unix), or in a VM on it (tcp, pair_tcp.go). It
|
|
// has no WireGuard device. Every lane that rides the tunnel to the host's
|
|
// :8443 rides the Host's lane socket or lane port instead, with the same
|
|
// URLs, the same mutual TLS and the same bearer: egress, management, the two
|
|
// parked harness workers, sessions, git, the heartbeat and what its reply
|
|
// directs. Absent, because they listen on the tunnel's stack or move the
|
|
// tunnel: the ollama, LLM and PII forwards, the service slots, and endpoint
|
|
// rotation and failover. A unix pairing's coder run has its control plane on
|
|
// a socket in RunsDir (coder_socket.go), not a tunnel port; a tcp pairing has
|
|
// no coder runs.
|
|
func runUnixPairing(ctx context.Context, cfg *Config) error {
|
|
logf := func(format string, a ...any) { fmt.Printf("["+cfg.HostName+"] "+format, a...) }
|
|
if !cfg.active() {
|
|
return errPairingNotActive
|
|
}
|
|
dialHost, laneTLS, err := localLaneDialer(cfg)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
lane := cfg.laneWhere()
|
|
upstream := &http.Client{Transport: &http.Transport{
|
|
DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { return dialHost(ctx) },
|
|
TLSClientConfig: laneTLS.Clone(),
|
|
}}
|
|
|
|
// Coder runs: the box-wide loop, then a unix pairing's run sockets, which
|
|
// are registered before the harness workers come up so a run-up that
|
|
// arrives at once still gets its socket.
|
|
coderInit(context.Background())
|
|
if cfg.unixTransport() {
|
|
setCoderSockLane(cfg.HostName, cfg.LaneSocket, dialHost, laneTLS)
|
|
startCoderSockets(ctx, cfg.HostName, cfg.RunsDir)
|
|
}
|
|
go faReconcileLoop(ctx, cfg.dcnDir())
|
|
setCtxSourcePuller(newHostSourcePuller(upstream, cfg.HostName, cfg.Bearer))
|
|
|
|
go func() {
|
|
realDialer := &net.Dialer{Timeout: egressDialTimeout}
|
|
wsURL := fmt.Sprintf("wss://%s:8443/bastion-egress", cfg.HostName)
|
|
logf("egress workers up — outbound exits THIS machine's internet via %s on %s\n", wsURL, lane)
|
|
runEgressWorkers(ctx, cfg.HostName, wsURL, laneTLS.Clone(), dialHost, cfg.Bearer, realDialer.DialContext, egressWorkerCount)
|
|
}()
|
|
go func() {
|
|
wsURL := fmt.Sprintf("wss://%s:8443/bastion-mgmt", cfg.HostName)
|
|
if n := sweepStaleScratch(cfg.contextRoot()); n > 0 {
|
|
logf("context-prep: removed %d scratch file(s) left by a previous run\n", n)
|
|
}
|
|
logf("mgmt workers up via %s on %s\n", wsURL, lane)
|
|
runMgmtWorkers(ctx, cfg.HostName, wsURL, laneTLS.Clone(), dialHost, cfg.Bearer, cfg.contextRoot(), cfg.dcnDir(),
|
|
cfg.backupRootPath(), mgmtWorkerCount)
|
|
}()
|
|
go func() {
|
|
wsURL := fmt.Sprintf("wss://%s:8443/bastion-harness", cfg.HostName)
|
|
logf("harness workers up via %s on %s\n", wsURL, lane)
|
|
runHarnessWorkers(ctx, cfg.HostName, cfg.dcnDir(), wsURL, laneTLS.Clone(), dialHost, cfg.Bearer, harnessWorkerCount)
|
|
}()
|
|
go func() {
|
|
wsURL := fmt.Sprintf("wss://%s:8443/bastion-harness", cfg.HostName)
|
|
srvName, _ := os.Hostname()
|
|
logf("session workers up via %s on %s (lane=session)\n", wsURL, lane)
|
|
runSessionWorkers(ctx, cfg.HostName, wsURL, laneTLS.Clone(), dialHost, cfg.Bearer, sessionWorkerCount,
|
|
sessionEnv{CtxRoot: cfg.contextRoot(), DcnDir: cfg.dcnDir(), HostName: cfg.HostName, Server: srvName})
|
|
}()
|
|
if cfg.GitProxyAddr != "" {
|
|
if gitLn, err := net.Listen("tcp", cfg.GitProxyAddr); err != nil {
|
|
logf("git proxy NOT started — %s: %v (lanes unaffected; free the port and restart)\n", cfg.GitProxyAddr, err)
|
|
} else {
|
|
stopGit := context.AfterFunc(ctx, func() { _ = gitLn.Close() })
|
|
defer stopGit()
|
|
go runGitLane(gitLn, fmt.Sprintf("wss://%s:8443/bastion-git", cfg.HostName), laneTLS.Clone(), dialHost, cfg.Bearer)
|
|
}
|
|
}
|
|
|
|
mux := http.NewServeMux()
|
|
mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("ok\n")) })
|
|
mux.HandleFunc("/git-credential", func(w http.ResponseWriter, r *http.Request) {
|
|
if !uiSecretMatches(strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer "), cfg.OperatorSecret) {
|
|
http.Error(w, "unauthorized", http.StatusUnauthorized)
|
|
return
|
|
}
|
|
req, err := http.NewRequest(http.MethodGet,
|
|
fmt.Sprintf("https://%s:8443/api/network/bastions/git-credential", cfg.HostName), nil)
|
|
if err != nil {
|
|
http.Error(w, "bad request", http.StatusBadRequest)
|
|
return
|
|
}
|
|
req.Header.Set("Authorization", "Bearer "+cfg.Bearer)
|
|
resp, err := upstream.Do(req)
|
|
if err != nil {
|
|
http.Error(w, "upstream: "+err.Error(), http.StatusBadGateway)
|
|
return
|
|
}
|
|
defer resp.Body.Close()
|
|
w.WriteHeader(resp.StatusCode)
|
|
_, _ = io.Copy(w, resp.Body)
|
|
})
|
|
mux.HandleFunc("/dcn/propose", dcnProposeLocal(upstream, cfg.HostName, cfg.Bearer, cfg.OperatorSecret))
|
|
|
|
// The heartbeat: the same body (no WireGuard device, so no allowed IPs
|
|
// and no endpoint) and every directive its reply carries except the two
|
|
// that move a tunnel.
|
|
go func() {
|
|
hbURL := fmt.Sprintf("https://%s:8443/api/network/bastions/heartbeat", cfg.HostName)
|
|
hostname, _ := os.Hostname()
|
|
wgPub := ""
|
|
if raw, e := base64.StdEncoding.DecodeString(cfg.WGPrivateKey); e == nil && len(raw) == 32 {
|
|
if pub, e2 := curve25519.X25519(raw, curve25519.Basepoint); e2 == nil {
|
|
wgPub = base64.StdEncoding.EncodeToString(pub)
|
|
}
|
|
}
|
|
beat := func() {
|
|
req, err := http.NewRequest(http.MethodPost, hbURL, strings.NewReader(heartbeatPayload(cfg, nil, hostname, wgPub)))
|
|
if err != nil {
|
|
return
|
|
}
|
|
req.Header.Set("Content-Type", "application/json")
|
|
req.Header.Set("Authorization", "Bearer "+cfg.Bearer)
|
|
resp, err := upstream.Do(req)
|
|
if err != nil {
|
|
return
|
|
}
|
|
body, _ := io.ReadAll(resp.Body)
|
|
resp.Body.Close()
|
|
if resp.StatusCode < 300 {
|
|
recordPairingOK(cfg.HostName)
|
|
}
|
|
maybeSelfUpdate(cfg.HostName, cfg.dcnDir(), body)
|
|
maybeContain(cfg.HostName, cfg.dcnDir(), body)
|
|
maybeContainFindings(cfg.HostName, cfg.dcnDir(), body)
|
|
maybeReconcileGrants(body, cfg.dcnDir())
|
|
maybeReconcileReach(body, cfg.dcnDir())
|
|
maybeReconcileCoderGrants(body, cfg.dcnDir())
|
|
coderHeartbeatReply(cfg.HostName, body)
|
|
maybeMirror(body, cfg, mirrorWSURL(cfg.HostName), laneTLS.Clone(), dialHost)
|
|
}
|
|
go runPostureRefresh(ctx)
|
|
go runAlertsRefresh(ctx)
|
|
go runOtherBastionRefresh(ctx)
|
|
go runDeltaRefresh(ctx)
|
|
beat()
|
|
t := time.NewTicker(60 * time.Second)
|
|
defer t.Stop()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-t.C:
|
|
beat()
|
|
}
|
|
}
|
|
}()
|
|
|
|
srv := &http.Server{Addr: cfg.ListenAddr, Handler: mux}
|
|
errc := make(chan error, 1)
|
|
go func() {
|
|
logf("privatae-bastion up — lanes to %s over %s, local control on http://%s\n", cfg.HostName, lane, cfg.ListenAddr)
|
|
errc <- srv.ListenAndServe()
|
|
}()
|
|
select {
|
|
case <-ctx.Done():
|
|
_ = srv.Close()
|
|
return nil
|
|
case err := <-errc:
|
|
return err
|
|
}
|
|
}
|
|
|
|
// heartbeatPayload is the body of one beat for the running pairing cfg; dev is
|
|
// that pairing's WireGuard device. A function of its own so a test can read
|
|
// what a beat would carry for a given running copy.
|
|
func heartbeatPayload(cfg *Config, dev interface{ IpcGet() (string, error) }, hostname, wgPub string) string {
|
|
// The enforcing report gains this pairing's state and caps ceiling
|
|
// (anchor.go), so the host sees what the box holds THIS host to.
|
|
enforcing := buildEnforcingReport(cfg.dcnDir(), wgAllowedIPs(dev))
|
|
for k, v := range pairingEnforcing(cfg.HostName) {
|
|
enforcing[k] = v
|
|
}
|
|
return buildHeartbeatBody(cfg.HostName, hostname, runtime.GOOS, clientVersion, wgPub, cfg.Fingerprint, cfg.BastionTunnelIP, cfg.Endpoint,
|
|
rotation.isUnsaved(cfg), currentMirrorReport(cfg.HostName), enforcing,
|
|
// Read fresh on every beat: a host paired a minute ago must
|
|
// appear as a peer on the NEXT beat, not after a restart.
|
|
peersFor(cfg.HostName))
|
|
}
|
|
|
|
// configureTunnelPeer gives the WireGuard device its one peer: the host's
|
|
// endpoint, with the host's tunnel alias as the only allowed address. A
|
|
// function of its own so a test can read back what a device holds after
|
|
// exactly the configuration runPairing applies.
|
|
func configureTunnelPeer(dev interface{ IpcSet(string) error }, cfg *Config) error {
|
|
privHex, err := b64ToHex(cfg.WGPrivateKey)
|
|
if err != nil {
|
|
return fmt.Errorf("wg private key: %w", err)
|
|
}
|
|
endpointPubHex, err := b64ToHex(cfg.EndpointPubKey)
|
|
if err != nil {
|
|
return fmt.Errorf("endpoint pubkey: %w", err)
|
|
}
|
|
hostCIDR, err := hostTunnelAllowedIP(cfg.HostTunnelIP)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// AllowedIPs is the host's tunnel alias ONLY — the tunnel can reach nothing else.
|
|
uapi := fmt.Sprintf("private_key=%s\npublic_key=%s\nendpoint=%s\npersistent_keepalive_interval=25\nallowed_ip=%s\n",
|
|
privHex, endpointPubHex, cfg.Endpoint, hostCIDR)
|
|
if err := dev.IpcSet(uapi); err != nil {
|
|
return fmt.Errorf("configure wireguard: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// hostTunnelAllowedIP is the one allowed address the tunnel's peer gets: the
|
|
// host's tunnel alias as a single-address prefix (/128, or /32 for IPv4). The
|
|
// value arrives in the pairing response, and the platform sends one bare
|
|
// address there. Anything else — empty, a prefix, a zone, an address that is
|
|
// not one host's — is refused rather than narrowed: an allowed range wider
|
|
// than one address would let the tunnel carry traffic for every address in
|
|
// it, and a pairing response that asks for that is not one to act on.
|
|
func hostTunnelAllowedIP(s string) (string, error) {
|
|
a, err := netip.ParseAddr(s)
|
|
if err != nil || a.Zone() != "" || a.IsUnspecified() || a.IsMulticast() || a.Is4In6() {
|
|
return "", fmt.Errorf("the host's tunnel address %q is not a single host address — refusing to configure the tunnel to it", s)
|
|
}
|
|
return netip.PrefixFrom(a, a.BitLen()).String(), nil
|
|
}
|
|
|
|
// bastionListening reports whether something is already serving a pairing's
|
|
// loopback control port — i.e. whether a bastion is RUNNING on this machine.
|
|
//
|
|
// A plain TCP connect, deliberately, because it is the one probe that works the
|
|
// same on every platform and needs no lock semantics, no pidfile and no
|
|
// privileges. It answers the question the operator actually asks.
|
|
func bastionListening(addr string) bool {
|
|
if addr == "" {
|
|
return false
|
|
}
|
|
c, err := net.DialTimeout("tcp", addr, 400*time.Millisecond)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
_ = c.Close()
|
|
return true
|
|
}
|
|
|
|
func cmdStatus() error {
|
|
cfgs, err := loadPairings()
|
|
if err != nil || len(cfgs) == 0 {
|
|
fmt.Println("not paired — run `privatae-bastion pair <token>`")
|
|
return nil
|
|
}
|
|
// IS IT RUNNING, AND WHERE — the first thing status should answer and the
|
|
// one thing it did not.
|
|
//
|
|
// A person with no terminal open has no way to tell where it is running.
|
|
// On Windows that is a fair question with an uncomfortable answer:
|
|
// relaunchAfterUpdate starts the replacement with DETACHED_PROCESS, so after
|
|
// a self-update the bastion has NO console at all — it is serving, and there
|
|
// is nothing on screen that says so. status printed pairings and mode and
|
|
// never once said whether a daemon was alive.
|
|
running := 0
|
|
for _, cfg := range cfgs {
|
|
if bastionListening(cfg.ListenAddr) {
|
|
running++
|
|
}
|
|
}
|
|
switch {
|
|
case running == len(cfgs):
|
|
fmt.Printf("RUNNING — serving %d host(s) on this machine.\n", len(cfgs))
|
|
case running > 0:
|
|
fmt.Printf("PARTLY RUNNING — %d of %d hosts are being served; the rest are not answering.\n",
|
|
running, len(cfgs))
|
|
default:
|
|
fmt.Printf("NOT RUNNING — nothing is serving these pairings. Start it with `%s`.\n", runCommandHint())
|
|
}
|
|
if running > 0 && runtime.GOOS == "windows" {
|
|
fmt.Println(" (No window? After a self-update it restarts itself detached, so it")
|
|
fmt.Println(" has no console. Task Manager → privatae-bastion.exe is where it lives.)")
|
|
}
|
|
fmt.Println()
|
|
for _, cfg := range cfgs {
|
|
live := "not answering"
|
|
if bastionListening(cfg.ListenAddr) {
|
|
live = "answering"
|
|
}
|
|
if cfg.unixTransport() {
|
|
fmt.Printf("paired with: %s [%s]\n lanes: %s (a Host on this machine)\n runs: %s\n local: http://%s (%s)\n",
|
|
cfg.HostName, cfg.stateName(), cfg.LaneSocket, cfg.RunsDir, cfg.ListenAddr, live)
|
|
continue
|
|
}
|
|
if cfg.tcpTransport() {
|
|
fmt.Printf("paired with: %s [%s]\n lanes: %s (a Host over TCP, name %s)\n runs: none over TCP\n local: http://%s (%s)\n",
|
|
cfg.HostName, cfg.stateName(), cfg.LaneAddr, tcpServerName, cfg.ListenAddr, live)
|
|
continue
|
|
}
|
|
fmt.Printf("paired with: %s [%s]\n tunnel: %s -> %s via %s\n local: http://%s (%s)\n",
|
|
cfg.HostName, cfg.stateName(), cfg.BastionTunnelIP, cfg.HostTunnelIP, cfg.Endpoint,
|
|
cfg.ListenAddr, live)
|
|
}
|
|
fmt.Printf("mode: %s\n", boxMode())
|
|
if a := currentAnchor(); a != nil {
|
|
fmt.Printf("anchor: %s (owner %q, since %s)\n", a.AnchorHost, a.Owner, a.ClaimedAt)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// ollamaModels reports whether a local ollama with the embeddings API answers
|
|
// AND which models it serves, so the bastion can advertise both the "embed"
|
|
// capability and its model list in the heartbeat — the host stores the list
|
|
// and offers it in the embedding pickers,
|
|
// so the user selects a model the box actually runs instead of guessing.
|
|
// Tries OLLAMA_ADDR, then the bastion's own tunnel IP (where the user may bind
|
|
// ollama for host reach over the WG tunnel), then loopback. Cheap (3s), per
|
|
// heartbeat. IPv6 tunnel IPs are bracketed (JoinHostPort) — the old raw
|
|
// concatenation produced an invalid authority for fd00::-style addresses.
|
|
func ollamaModels(tunnelIP string) (bool, []string) {
|
|
addrs := []string{}
|
|
if v := os.Getenv("OLLAMA_ADDR"); v != "" {
|
|
addrs = append(addrs, v)
|
|
}
|
|
if tunnelIP != "" {
|
|
addrs = append(addrs, net.JoinHostPort(tunnelIP, "11434"))
|
|
}
|
|
addrs = append(addrs, "127.0.0.1:11434")
|
|
cl := &http.Client{Timeout: 3 * time.Second}
|
|
for _, a := range addrs {
|
|
resp, err := cl.Get("http://" + a + "/api/tags")
|
|
if err != nil {
|
|
continue
|
|
}
|
|
if resp.StatusCode != http.StatusOK {
|
|
resp.Body.Close()
|
|
continue
|
|
}
|
|
var tags struct {
|
|
Models []struct {
|
|
Name string `json:"name"`
|
|
Model string `json:"model"`
|
|
} `json:"models"`
|
|
}
|
|
err = json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(&tags)
|
|
resp.Body.Close()
|
|
names := []string{}
|
|
if err == nil {
|
|
for _, m := range tags.Models {
|
|
n := m.Name
|
|
if n == "" {
|
|
n = m.Model
|
|
}
|
|
if n != "" {
|
|
names = append(names, n)
|
|
}
|
|
}
|
|
}
|
|
// A 200 from /api/tags is a live ollama even if the parse failed or
|
|
// the box has no models pulled yet.
|
|
return true, names
|
|
}
|
|
return false, nil
|
|
}
|
|
|
|
// llmForwardPort is the tunnel-side port the host dials to reach this box's
|
|
// OpenAI-compatible model server. Reported in the heartbeat (agents.go
|
|
// "llm_port") so the host never has to assume it — ONE definition, here.
|
|
const llmForwardPort = 18080
|
|
|
|
// openAICompatModels detects a non-ollama LLM server (llama.cpp llama-server,
|
|
// vLLM, anything OpenAI-compatible) via GET /v1/models. Only PRIVATAE_LLM_ADDR is
|
|
// probed — unlike ollama there is no conventional default port worth guessing,
|
|
// and the operator setting the env IS the opt-in. Same 200-is-alive rule as
|
|
// /api/tags: a live server with an unparseable list still counts.
|
|
func openAICompatModels() (bool, []string) {
|
|
a := llmTarget()
|
|
if a == "" {
|
|
return false, nil
|
|
}
|
|
cl := &http.Client{Timeout: 3 * time.Second}
|
|
resp, err := cl.Get("http://" + a + "/v1/models")
|
|
if err != nil {
|
|
return false, nil
|
|
}
|
|
defer resp.Body.Close()
|
|
if resp.StatusCode != http.StatusOK {
|
|
return false, nil
|
|
}
|
|
var list struct {
|
|
Data []struct {
|
|
ID string `json:"id"`
|
|
} `json:"data"`
|
|
}
|
|
names := []string{}
|
|
if err := json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(&list); err == nil {
|
|
for _, m := range list.Data {
|
|
if m.ID != "" {
|
|
names = append(names, m.ID)
|
|
}
|
|
}
|
|
}
|
|
return true, names
|
|
}
|
|
|
|
// localOllamaAddr is where ollama actually listens ON THIS BOX — the proxy
|
|
// target. OLLAMA_ADDR if set, else loopback. Distinct from the tunnel IP the
|
|
// host reaches us at (that's the netstack listener below). Deliberately NOT
|
|
// PRIVATAE_LLM_ADDR-aware: a box can run ollama (embed) and an OpenAI-compat
|
|
// server (chat) side by side — :11434 stays ollama's, the OpenAI server gets
|
|
// its own tunnel-side listener (serveLLMForward, :18080).
|
|
func localOllamaAddr() string {
|
|
if v := os.Getenv("OLLAMA_ADDR"); v != "" {
|
|
return v
|
|
}
|
|
return "127.0.0.1:11434"
|
|
}
|
|
|
|
// serveLLMForward is serveOllamaForward's twin for the OpenAI-compatible
|
|
// server: host dials <bastion tunnel IP>:18080, we proxy raw TCP to
|
|
// PRIVATAE_LLM_ADDR. Only runs when the env is set (the operator's opt-in).
|
|
// The "llm-openai" heartbeat capability is the host's signal that this
|
|
// listener exists and speaks the /v1 dialect.
|
|
func serveLLMForward(ctx context.Context, self string, tnet *netstack.Net, gwIP netip.Addr) {
|
|
target := llmTarget()
|
|
if target == "" {
|
|
return
|
|
}
|
|
ln, err := tnet.ListenTCP(&net.TCPAddr{IP: gwIP.AsSlice(), Port: llmForwardPort})
|
|
if err != nil {
|
|
return
|
|
}
|
|
defer context.AfterFunc(ctx, func() { _ = ln.Close() })()
|
|
refused := inferenceRefuser(self, "llm")
|
|
for {
|
|
in, err := ln.Accept()
|
|
if err != nil {
|
|
return
|
|
}
|
|
if refused(in) {
|
|
continue
|
|
}
|
|
// A pairing whose grant NAMES models gets an HTTP-aware proxy
|
|
// that checks the model on each request; one with a free hand gets the
|
|
// raw splice this lane has always been (inferencefilter.go).
|
|
go proxyInference(in, target, laneLLM, self)
|
|
}
|
|
}
|
|
|
|
// inferenceRefuser is the accept-time gate the three inbound forwards share.
|
|
// The `inference` caps family is checked on EVERY accepted connection rather
|
|
// than once at start, because a pairing grant is re-signed while the daemon
|
|
// runs (a narrowed re-approval, a revoke) and the listener must follow it
|
|
// without a restart. capsAllow reads the grant file each call; a forward
|
|
// carries a handful of connections a minute, so that is cheap. Logged once per
|
|
// listener per refusal reason, not per connection.
|
|
func inferenceRefuser(self, lane string) func(in net.Conn) bool {
|
|
lastWhy := ""
|
|
return func(in net.Conn) bool {
|
|
why := capsAllow(self, "inference")
|
|
if why == "" {
|
|
return false
|
|
}
|
|
_ = in.Close()
|
|
if why != lastWhy {
|
|
fmt.Printf("[%s] %s forward: %s\n", self, lane, why)
|
|
lastWhy = why
|
|
}
|
|
return true
|
|
}
|
|
}
|
|
|
|
// serveOllamaForward accepts host connections to <tunnel IP>:11434 over the
|
|
// netstack and proxies them to the local ollama, so the host's embed lane can
|
|
// reach the bastion's embedder. The Go client's tunnel is in-process (no OS WG
|
|
// adapter), so ollama can't bind the tunnel IP itself — this bridges it. ollama
|
|
// runs on its default loopback; nothing else to configure. Best-effort: if
|
|
// ollama is down, an accepted conn just fails to dial (host sees a refusal).
|
|
func serveOllamaForward(ctx context.Context, self string, tnet *netstack.Net, gwIP netip.Addr) {
|
|
ln, err := tnet.ListenTCP(&net.TCPAddr{IP: gwIP.AsSlice(), Port: 11434})
|
|
if err != nil {
|
|
return
|
|
}
|
|
// A revoked pairing's netstack goes away with dev.Close(); closing the
|
|
// listener too is what makes this Accept loop return instead of hanging.
|
|
defer context.AfterFunc(ctx, func() { _ = ln.Close() })()
|
|
refused := inferenceRefuser(self, "ollama")
|
|
for {
|
|
in, err := ln.Accept()
|
|
if err != nil {
|
|
return
|
|
}
|
|
if refused(in) {
|
|
continue
|
|
}
|
|
// The model scope, and the store-mutating half of ollama's API
|
|
// (/api/pull, /api/delete — the `models` family's own powers, reachable
|
|
// through a raw splice until now). inferencefilter.go carries both.
|
|
go proxyInference(in, localOllamaAddr(), laneOllama, self)
|
|
}
|
|
}
|