package main import ( "archive/tar" "bytes" "context" "crypto/sha256" "crypto/tls" "encoding/base64" "encoding/hex" "encoding/json" "errors" "fmt" "io" "io/fs" "net" "net/http" "net/url" "os" "os/exec" "path/filepath" "runtime" "sort" "strconv" "strings" "time" "github.com/gorilla/websocket" ) const harnessWorkerCount = 2 // harnessSpec is the H-frame job: run a harness container locally + dispatch the prompt to it. // - mode "echo": no container — prove the lane end-to-end. // - mode "container": the lifecycle engine — pull/start the image, wait for its HTTP port, // stop + remove it, report metered wall-time. Image-generic; the host passes image + port. // - mode "worktree" / "git" add clone-in -> POST /run -> collect-out (the real harness contract), keyed server>worktree>Gitea. type harnessSpec struct { HarnessID string `json:"harness_id"` Prompt string `json:"prompt"` Image string `json:"image"` Mode string `json:"mode"` Worktree string `json:"worktree"` WorktreeB64 string `json:"worktree_b64"` // mode=worktree: the worktree tar in (clone-in) RepoURL string `json:"repo_url"` // mode=git: clone URL carrying the per-run minted token Branch string `json:"branch"` // mode=git: push target branch Command []string `json:"command"` // the stand-in agent command (legacy); the image's /run is driven instead AgentCreds string `json:"agent_creds"` // the agent's OWN credential (claudeAiOauth JSON, pulled from the entity vault), delivered for THIS run only — mounted as the harness creds file, deleted after, never baked into the image Port int `json:"port"` MaxMinutes int `json:"max_minutes"` // Repo is the repository a mode=git run clones, as "forge-host/owner/name"; // RepoURL above carries a per-run minted token and so differs on every // run. The approval binds this, forge included, so the person taps once // for a run the host may have to re-mint the URL for — and a URL on any // other server does not pass as the same repository. Repo string `json:"repo,omitempty"` // Approval: on an ANCHORED box a run is a write — a container, a clone, a // push — and spends the person's approval (writes.go), bound to the // harness id, mode, image, repo and branch of THIS run. Approval string `json:"approval,omitempty"` // mode=lab (the lab harness): test tooling built only with -tags lab // (lab.go). A release build refuses it with exit 126 (lab_off.go). Hosts int `json:"hosts"` // how many lab hosts to build Envs map[string]string `json:"envs"` // app settings written to every lab host (.env) Tests []string `json:"tests"` // the runbook: ordered steps, one governed /run each Engine string `json:"engine"` // which coding harness drives the tests (PRIVATAE_APP_ID) Gate string `json:"gate"` // "auto" (typed deliberately) | "ask" (the App lane; not built yet) App string `json:"app"` // project name the worktree lands under in each host // mode=service (service.go): a long-running server in this machine's slot. // Reuses Image, Port and Envs above. ServiceOp string `json:"service_op"` // up | down | status ServiceName string `json:"service_name"` // the slot's name on this machine DataPath string `json:"data_path"` // where the slot's data dir is mounted in the container DataUID int `json:"data_uid"` // uid the data dir is handed to (a rootless image's user); 0 = leave Purge bool `json:"purge"` // down: also delete the slot's data (default keeps it) Socket string `json:"socket"` // up: the server's unix socket file in its data dir (it has no network) // mirror (0.38.0): the host's credential for ITS OWN server, used for the // push into it and never stored, logged or bound into an approval. PushToken string `json:"push_token"` // mode=coder (coders.go): one sealed coding run per container. Reuses // Engine, Image and MaxMinutes above. CoderOp string `json:"coder_op"` // run-up | run-down | run-status RunID string `json:"run_id"` // the kernel's run id, also the container's name UID int `json:"uid"` // the engine's uid (10001-10099) CredsDir string `json:"creds_dir"` // where the engine reads its credential, /home/privatae/. Creds map[string]string `json:"creds"` // file name -> contents, this run only, never logged ControlCap string `json:"control_cap"` // the run's control capability, handed to the harness as its cap Pull *coderPull `json:"pull"` // the per-run registry credential // owner is the pairing this job arrived on, set by handleHarnessConn from // the lane itself. Unexported so no JSON from the host can set it. owner string // runImage is what a coder run's container starts from: Image, or the // local image ID it resolved to (coderLocalImage). Never from JSON. runImage string // dcnDir is that pairing's grant directory (reach grants live there). dcnDir string } // runHarnessWorkers opens a pool of worker WebSockets to the host's /bastion-harness lane and serves // one harness job per worker, reconnecting after each. Mirrors runMgmtWorkers: the host can't reach // the bastion's tunnel IP, so the bastion dials out to the host and parks until a harness job arrives. // // pctx is the pairing's context (run.go): a revoked pairing's workers exit // instead of re-dialing a closed tunnel for the life of the process. func runHarnessWorkers(pctx context.Context, self, dcnDir string, wsURL string, tlsCfg *tls.Config, dialHost func(ctx context.Context) (net.Conn, error), bearer string, n int) { dialer := laneWSDialer(tlsCfg, dialHost) for i := 0; i < n; i++ { go func() { hold := laneHoldFor(self) for pctx.Err() == nil { if !hold.waitClear(pctx) { return } ctx, cancel := context.WithTimeout(pctx, 25*time.Second) ws, _, err := dialer.DialContext(ctx, wsURL, http.Header{}) cancel() if err != nil { sleepCtx(pctx, 3*time.Second) continue } if err := ws.WriteMessage(websocket.TextMessage, []byte("BEARER "+bearer)); err != nil { _ = ws.Close() sleepCtx(pctx, time.Second) continue } held := watchHeldClose(ws) stopKA := armWSKeepalive(ws) // ping/read-deadline: a dead tunnel unblocks the park unpark := context.AfterFunc(pctx, func() { _ = ws.Close() }) handleHarnessConn(ws, self, dcnDir) // blocks until a job, serves it, then returns unpark() stopKA() _ = ws.Close() if held() { hold.refused(self) // the Host holds this machine (lanehold.go) } } }() } } // harnessArgs is what the approval for one run binds — EVERYTHING the run // acts on, in the order the pod derives it (the host's // harness_approval_args): the harness id, mode, image, repo NAME and branch; // the prompt; the command; a digest of the worktree tar; the port and the // minute cap; and the lab fields. A field left out of this list is a field // the pod could change under an approval the person gave for another run. func harnessArgs(spec harnessSpec) []string { envs := make([]string, 0, len(spec.Envs)) for k, v := range spec.Envs { envs = append(envs, k+"="+v) } sort.Strings(envs) wt := "" if spec.WorktreeB64 != "" { sum := sha256.Sum256([]byte(spec.WorktreeB64)) wt = hex.EncodeToString(sum[:]) } return []string{ spec.HarnessID, spec.Mode, spec.Image, spec.Repo, spec.Branch, spec.Prompt, strings.Join(spec.Command, "\x00"), wt, strconv.Itoa(spec.Port), strconv.Itoa(spec.MaxMinutes), strconv.Itoa(spec.Hosts), strings.Join(envs, "\n"), strings.Join(spec.Tests, "\n"), spec.Engine, spec.Gate, spec.App, } } // repoNameOf is "host/owner/name" from a clone URL — the forge and the path, // without the userinfo (the per-run minted token), the scheme or ".git". The // forge is part of it: a name alone would let a URL on any server pass as // the repository the person approved, and the run clones from and pushes to // that server with the agent's own credentials mounted. func repoNameOf(repoURL string) string { u, err := url.Parse(strings.TrimSpace(repoURL)) if err != nil || u.Scheme == "" || u.Host == "" || u.Path == "" { return "" } return strings.ToLower(u.Host) + "/" + strings.TrimSuffix(strings.Trim(u.Path, "/"), ".git") } // harnessGate: "" to run, else the refusal. Anchored boxes only. The repo // the run would clone and push is checked against the name the approval // bound, so an approval for owner/site cannot run against another remote. // A quarantined pairing spends nothing here, and a refusal on this lane // counts as a strike the way one on the management lane does. func harnessGate(self, dcnDir string, spec harnessSpec) string { if currentAnchor() == nil { return "" } // Coder runs (coders.go) never spend a per-run approval: the person // approved a standing coder grant once (coder_grant.go). if spec.Mode == "coder" { return coderGate(self, dcnDir, spec) } why := "" op := "harness:run" if spec.Mode == "service" { op = serviceGateOp(spec.ServiceOp) // `status` is a READ: it names the slots and changes nothing, so it // spends no approval and, like every read, still answers a pairing // whose writes are quarantined (quarantine.go: "Reads still answer"). if spec.ServiceOp == "status" { return "" } // `mirror` is STANDING WORK under the mirror list the person approved in // the reach grant (service_mirror.go): no per-run approval, but it // changes the server, so a quarantined pairing's writes stay closed. if spec.ServiceOp == "mirror" { if until := quarantinedUntil(self, time.Now()); !until.IsZero() { return quarantineRefusal(until) } return "" } } if until := quarantinedUntil(self, time.Now()); !until.IsZero() { why = quarantineRefusal(until) } else if spec.Mode == "service" { // The service slot (service.go). `up` and `down` change this machine // and spend an approval bound to exactly what they would do // (serviceArgs): which server, which image, which ports, where its // data is mounted, what it is told, and whether its data is deleted. hn, _ := os.Hostname() if _, err := boxConsume(dcnDir, spec.Approval, mgmtOpPurpose(op, hn, serviceArgs(spec))); err != nil { why = "refused: " + op + " changes this machine and " + err.Error() } } else if spec.Mode == "git" && (spec.Repo == "" || repoNameOf(spec.RepoURL) == "" || repoNameOf(spec.RepoURL) != spec.Repo) { // A git run names a repository on a forge — never blank, never a local // path or a file: URL (repoNameOf is "" for those), and the URL it // clones must be that repository. A blank on both sides compared // equal once, and a local path passed as the approved repository. why = "refused: harness:run — the repository the run would clone (" + orUnknown(repoNameOf(spec.RepoURL)) + ") is not the one the approval names (" + orUnknown(spec.Repo) + ")" } else { hn, _ := os.Hostname() if _, err := boxConsume(dcnDir, spec.Approval, mgmtOpPurpose("harness:run", hn, harnessArgs(spec))); err != nil { why = "refused: harness:run changes this machine and " + err.Error() } } if why != "" { noteRefusal(self, op, why, refusalCounts(op, why), time.Now()) } return why } // handleHarnessConn serves ONE harness job: read the H frame, run it, reply J(result) + X(exit). // self is the pairing this lane belongs to: on an anchored box a member's // harness jobs are refused unless its pairing grant carries the "harness" cap // (anchor.go capsAllow) — the lane runs containers on this machine, and the // mgmt op table never sees it, so the gate has to sit here. // // An S frame (a terminal session) belongs on a SESSION worker (session.go, hello line // `BEARER lane=session`); a harness worker refuses it so a shell can never hold // an agent worker for hours. func handleHarnessConn(ws *websocket.Conn, self, dcnDir string) { // The spec is read up to harnessSpecMaxBytes (lane_bounds.go) and refused // past it; a frame of any other tag is not read at all. tag, body, tooBig, err := readLaneJob(ws, func(t byte) int64 { if t == 'H' { return harnessSpecMaxBytes } return 0 }) if err != nil { return } if tag == 'S' { sendHarness(ws, 'E', []byte("this is a harness worker; sessions go to a worker that said lane=session\n")) sendHarnessExit(ws, 1) return } if tag != 'H' { return } if tooBig { // Gated like a spec that does not parse (below): a pairing the caps // refuse learns nothing more from a frame that was too large. if why := capsAllow(self, "harness:run"); why != "" { b, _ := json.Marshal(map[string]any{"ok": false, "error": why}) sendHarness(ws, 'J', b) sendHarnessExit(ws, 126) return } b, _ := json.Marshal(map[string]any{"ok": false, "error": fmt.Sprintf( "the harness spec is more than the %d MiB a job on this lane may be; nothing of it was read as a job", harnessSpecMaxBytes>>20)}) sendHarness(ws, 'J', b) sendHarnessExit(ws, 1) return } // The gate depends on WHAT is asked: a service-slot job (service.go) is the // "services" family, every other mode is "harness". So the spec is parsed // first — but a spec that does not parse is still gated as harness:run // before its parse error is reported, exactly as it always was, so an // unauthorized pairing learns nothing from a malformed frame. var spec harnessSpec perr := json.Unmarshal(body, &spec) gateOp := "harness:run" if perr == nil && spec.Mode == "service" { gateOp = serviceGateOp(spec.ServiceOp) } if perr == nil && spec.Mode == "coder" { gateOp = coderGateOp } if why := capsAllow(self, gateOp); why != "" { b, _ := json.Marshal(map[string]any{"ok": false, "error": why}) sendHarness(ws, 'J', b) sendHarnessExit(ws, 126) return } if perr != nil { sendHarness(ws, 'J', []byte(`{"ok":false,"error":"bad harness spec"}`)) sendHarnessExit(ws, 126) return } // A service request the box would refuse anyway is refused BEFORE the // write gate below spends the person's approval on it. if spec.Mode == "service" { if why := serviceSpecRefusal(spec); why != "" { b, _ := json.Marshal(map[string]any{"ok": false, "mode": "service", "error": why}) sendHarness(ws, 'J', b) sendHarnessExit(ws, 1) return } } if spec.Mode == "coder" { if why := coderSpecRefusal(spec); why != "" { b, _ := json.Marshal(map[string]any{"ok": false, "mode": "coder", "coder_op": spec.CoderOp, "error": why}) sendHarness(ws, 'J', b) sendHarnessExit(ws, 1) return } } // THE WRITE GATE for this lane (writes.go): the caps ceiling above says the // pairing may be asked; on an anchored box the run itself spends the // person's approval, bound to what this run would do. A local box keeps // the pairing as its authorization. if why := harnessGate(self, dcnDir, spec); why != "" { b, _ := json.Marshal(map[string]any{"ok": false, "error": why}) sendHarness(ws, 'J', b) sendHarnessExit(ws, 126) return } spec.owner = self spec.dcnDir = dcnDir // Run the (possibly slow) job in a goroutine and keep the lane warm: a read-pump auto-pongs the // host's keepalive pings + notices a close, and an O-frame ticker keeps traffic flowing — so // neither the WS ping-timeout nor the tunnel's TCP idle-timeout drops a long container/agent run. // (One reader + one writer = gorilla-safe: the pump only reads, this loop only writes.) type harnessRes struct { body []byte code int } fmt.Printf("harness: job %s mode=%s image=%s\n", spec.HarnessID, spec.Mode, spec.Image) resCh := make(chan harnessRes, 1) go func() { b, c := runHarness(spec); resCh <- harnessRes{b, c} }() closed := make(chan struct{}) go func() { // Nothing the host sends during a run is read as anything: each frame // is dropped as it arrives (lane_bounds.go), never held whole. _ = drainLane(ws) // returns on a host close or drop close(closed) }() ticker := time.NewTicker(8 * time.Second) defer ticker.Stop() for { select { case r := <-resCh: body, code := fitLane(r.body, r.code) fmt.Printf("harness: job %s done exit=%d\n", spec.HarnessID, code) sendHarness(ws, 'J', body) sendHarnessExit(ws, code) return case <-ticker.C: sendHarness(ws, 'O', []byte("running\n")) // keepalive for long ops (worktree/git/lab runs) case <-closed: return // peer/tunnel gone; runHarness's deferred cleanup still removes the container } } } func runHarness(spec harnessSpec) ([]byte, int) { switch spec.Mode { case "echo", "": b, _ := json.Marshal(map[string]interface{}{ "ok": true, "harness": spec.HarnessID, "mode": "echo", "echo": spec.Prompt, "ran_on": orUnknown(hostnameSafe()), }) return b, 0 case "container": return runContainer(spec) case "worktree": return runWorktree(spec) case "git": return runGit(spec) case "lab": return runLab(spec) case "service": return runService(spec) case "coder": return runCoder(spec) default: b, _ := json.Marshal(map[string]interface{}{"ok": false, "error": "mode '" + spec.Mode + "' not implemented"}) return b, 1 } } // runContainer is the lifecycle engine: detect a runtime, pull + start the image, wait for its // HTTP port to answer, then ALWAYS stop + remove it (lifecycle-clean) and report metered wall-time. // The image runs on THIS box's OCI boundary — the bastion manages the lifecycle, it is not the agent. func runContainer(spec harnessSpec) ([]byte, int) { res := map[string]interface{}{"image": spec.Image, "ran_on": orUnknown(hostnameSafe()), "mode": "container"} rt := detectRuntime() res["runtime"] = rt if rt == "" { res["ok"], res["error"] = false, "no container runtime (docker/podman/nerdctl) on this host" b, _ := json.Marshal(res) return b, 1 } if spec.Image == "" { res["ok"], res["error"] = false, "image required for mode=container" b, _ := json.Marshal(res) return b, 1 } port := spec.Port if port == 0 { port = 80 } start := time.Now() if out, err := runCmd(150*time.Second, rt, "pull", spec.Image); err != nil { res["ok"], res["error"] = false, "pull failed: "+truncate(out, 220) b, _ := json.Marshal(res) return b, 1 } res["pulled_s"] = round2(time.Since(start).Seconds()) // No -p (port publish): publishing adds per-container DNAT that mutates the host's iptables and // drops the bastion's own WG/Cilium relay tunnel. Reach the container on its bridge IP instead — // same result, no host-network mutation, and portable across any docker/podman bridge. out, err := runCmd(30*time.Second, rt, "run", "-d", spec.Image) if err != nil { res["ok"], res["error"] = false, "run failed: "+truncate(out, 220) b, _ := json.Marshal(res) return b, 1 } cid := strings.TrimSpace(out) res["container"] = shortID(cid) defer func() { _, _ = runCmd(20*time.Second, rt, "rm", "-f", cid) }() // lifecycle-clean: always remove reachable, code := false, 0 if ip := containerIP(rt, cid); ip != "" { res["ip"] = ip reachable, code = waitReachable("http://"+ip+":"+strconv.Itoa(port)+"/", 30*time.Second) } res["reachable"], res["http_code"] = reachable, code res["meter_s"] = round2(time.Since(start).Seconds()) res["ok"] = reachable b, _ := json.Marshal(res) if reachable { return b, 0 } return b, 1 } // runWorktree (clone-in/collect-out): receive the worktree tar over the lane, run a container // with it mounted at /work and drive the image's real harness agent against it, // then send the modified worktree back. Only a bind mount — no -p, no port — so the WG relay is // untouched. runGit (server>worktree>Gitea) clones the worktree from Gitea instead. func runWorktree(spec harnessSpec) ([]byte, int) { res := map[string]interface{}{"mode": "worktree", "ran_on": orUnknown(hostnameSafe())} rt := detectRuntime() res["runtime"] = rt if rt == "" { res["ok"], res["error"] = false, "no container runtime" b, _ := json.Marshal(res) return b, 1 } if spec.Image == "" { res["ok"], res["error"] = false, "image required for mode=worktree" b, _ := json.Marshal(res) return b, 1 } tarIn, err := base64.StdEncoding.DecodeString(spec.WorktreeB64) if err != nil { res["ok"], res["error"] = false, "bad worktree_b64: "+err.Error() b, _ := json.Marshal(res) return b, 1 } dir, err := os.MkdirTemp("", "harness-wt-") if err != nil { res["ok"], res["error"] = false, "tempdir: "+err.Error() b, _ := json.Marshal(res) return b, 1 } defer os.RemoveAll(dir) // collect-out is over the lane; the box keeps nothing // The tree is a folder inside dir, as runGit's is: dir stays 0700, so the // tree opened for the agent is out of every other local user's reach. work := filepath.Join(dir, "work") if err := os.Mkdir(work, 0o700); err != nil { res["ok"], res["error"] = false, "tempdir: "+err.Error() b, _ := json.Marshal(res) return b, 1 } if err := extractTar(tarIn, work); err != nil { res["ok"], res["error"] = false, "extract: "+err.Error() b, _ := json.Marshal(res) return b, 1 } start := time.Now() if out, err := runCmd(150*time.Second, rt, "pull", spec.Image); err != nil { res["ok"], res["error"] = false, "pull failed: "+truncate(out, 220) b, _ := json.Marshal(res) return b, 1 } maxMin := spec.MaxMinutes if maxMin <= 0 { maxMin = 10 } // Drive the harness image's REAL agent (its /run sidecar) with the prompt. The agent works // cwd in the bind-mounted /work, so the worktree is modified in place and collected below // (tar-out). agentRes, agentErr := driveHarnessAgent(rt, work, spec, maxMin) if agentErr != nil { res["ok"], res["error"] = false, "agent run: "+agentErr.Error() b, _ := json.Marshal(res) return b, 1 } res["run_ok"] = agentRes["ok"] == true res["transcript"] = agentRes["transcript"] res["agent_diff"] = agentRes["diff"] res["usage"] = agentRes["usage"] tarOut, skips, err := packWorktreeSkips(work) if errors.Is(err, errTreeTooLarge) { // No tree at all rather than part of one: a shorter tar would read as // the agent's whole work. res["ok"], res["worktree_too_large"] = false, true res["error"] = "the tree the agent left is not returned: " + err.Error() + ", the most this lane carries back" b, _ := json.Marshal(res) return b, 1 } if err != nil { res["ok"], res["error"] = false, "repack: "+err.Error() b, _ := json.Marshal(res) return b, 1 } res["worktree_b64"] = base64.StdEncoding.EncodeToString(tarOut) skips.report(res) res["meter_s"] = round2(time.Since(start).Seconds()) res["ok"] = true b, _ := json.Marshal(res) return b, 0 } // runGit (server>worktree>Gitea): clone the repo from Gitea into a worktree, drive the image's // real harness agent against it, then commit + push to a // branch. The diff rides git — lane-light, no large tar. spec.RepoURL carries the per-run minted // token (the host mints it, dispatches, and revokes it after the run). // // The agent's tree, its .git included, is the agent's to write, and this // process may be root: a hook, core.fsmonitor, a filter or a credential // helper set in that .git would run here, and a changed remote would take the // push and its token elsewhere. So git never runs in that tree after the // agent has had it. The commit is made in a second clone, in a folder only // this process can enter and that is mounted nowhere, which gets the agent's // plain files and folders (mirrorTree) and nothing of its .git. The branch // gets one commit holding the agent's final files. func runGit(spec harnessSpec) ([]byte, int) { res := map[string]interface{}{"mode": "git", "ran_on": orUnknown(hostnameSafe())} fail := func(msg string) ([]byte, int) { res["ok"], res["error"] = false, msg b, _ := json.Marshal(res) return b, 1 } rt := detectRuntime() res["runtime"] = rt if rt == "" { return fail("no container runtime") } if spec.Image == "" || spec.RepoURL == "" { return fail("image and repo_url required for mode=git") } dir, err := os.MkdirTemp("", "harness-git-") if err != nil { return fail("tempdir: " + err.Error()) } defer os.RemoveAll(dir) // The push side: the clean clone and the empty HOME every git call runs // with. MkdirTemp makes it 0700, and it is never mounted. own, err := os.MkdirTemp("", "harness-push-") if err != nil { return fail("tempdir: " + err.Error()) } defer os.RemoveAll(own) home := filepath.Join(own, "home") if err := os.Mkdir(home, 0o700); err != nil { return fail("tempdir: " + err.Error()) } // Every git call names the repository by its URL without the token and // gets the token from its environment, so neither clone's config holds it // and no process argument shows it. bareURL, auth, err := repoGitAuth(spec.RepoURL) if err != nil { return fail("repo_url: " + err.Error()) } git := func(timeout time.Duration, in string, args ...string) (string, error) { return harnessGitAuth(timeout, home, auth, in, args...) } repoDir := filepath.Join(dir, "repo") start := time.Now() if out, err := git(120*time.Second, "", "clone", "--", bareURL, repoDir); err != nil { return fail("clone: " + truncate(scrubURL(out, spec.RepoURL), 220)) } // The agent gets no push access of its own: the run's commit is made from // the clean clone below. Its clone was made with no token in its URL, and // is refused if the token is in any file of its .git all the same. if err := dropRepoCredential(git, repoDir, spec.RepoURL); err != nil { return fail("clone: " + truncate(scrubURL(err.Error(), spec.RepoURL), 220)) } branch := spec.Branch if branch == "" { branch = "harness/" + spec.HarnessID } // The commit the agent starts from, read before it has the tree: the // clean clone commits on top of the same one. An empty repository has // none, and the run's commit is its first. base := "" if out, err := git(10*time.Second, repoDir, "rev-parse", "--verify", "-q", "HEAD^{commit}"); err == nil { base = strings.TrimSpace(out) } _, _ = git(15*time.Second, repoDir, "checkout", "-b", branch) _, _ = git(15*time.Second, repoDir, "config", "user.email", harnessGitEmail) _, _ = git(15*time.Second, repoDir, "config", "user.name", harnessGitName) maxMin := spec.MaxMinutes if maxMin <= 0 { maxMin = 10 } // Drive the harness image's REAL agent against the cloned worktree (its /run sidecar); // the agent modifies /work in place, copied into the clean clone below. agentRes, agentErr := driveHarnessAgent(rt, repoDir, spec, maxMin) if agentErr != nil { return fail("agent run: " + agentErr.Error()) } res["run_ok"] = agentRes["ok"] == true res["transcript"] = agentRes["transcript"] clean := filepath.Join(own, "repo") if out, err := git(120*time.Second, "", "clone", "--no-checkout", "--", bareURL, clean); err != nil { return fail("clone for the push: " + truncate(scrubURL(out, spec.RepoURL), 220)) } checkout := []string{"checkout", "-b", branch} if base != "" { checkout = append(checkout, base) } if out, err := git(30*time.Second, clean, checkout...); err != nil { return fail("clone for the push: could not check out the commit the run started from (" + orUnknown(base) + "): " + truncate(scrubURL(out, spec.RepoURL), 160)) } _, _ = git(15*time.Second, clean, "config", "user.email", harnessGitEmail) _, _ = git(15*time.Second, clean, "config", "user.name", harnessGitName) skips, err := mirrorTree(repoDir, clean, maxCopiedTreeBytes) if errors.Is(err, errTreeTooLarge) { res["tree_too_large"] = true return fail("nothing was pushed: " + err.Error() + ", the most a run copies (each file and folder counts 512 bytes and its name besides its content)") } if err != nil { return fail("copy the agent's tree: " + err.Error()) } skips.report(res) if out, err := git(60*time.Second, clean, "add", "-A"); err != nil { return fail("nothing was pushed: add: " + truncate(scrubURL(out, spec.RepoURL), 220)) } // A run that changed no file has nothing to commit and pushes nothing: // ok, with changed false and no branch, since none was made. dout, derr := git(60*time.Second, clean, "diff", "--cached", "--quiet") var exitErr *exec.ExitError switch { case derr == nil: res["changed"] = false res["note"] = "the agent changed no file; nothing was committed or pushed" res["meter_s"] = round2(time.Since(start).Seconds()) res["ok"] = true b, _ := json.Marshal(res) return b, 0 case !errors.As(derr, &exitErr) || exitErr.ExitCode() != 1: return fail("nothing was pushed: diff: " + truncate(scrubURL(dout, spec.RepoURL), 220)) } commitOut, err := git(20*time.Second, clean, "commit", "-m", "harness run: "+spec.HarnessID) res["commit_out"] = truncate(scrubURL(commitOut, spec.RepoURL), 200) if err != nil { return fail("nothing was pushed: commit: " + truncate(scrubURL(commitOut, spec.RepoURL), 220)) } res["changed"] = true // Hooks never run, so git-lfs's pre-push hook does not either: the large // files the commit points to are sent first, to the repository's own LFS // address, whatever a .lfsconfig in the agent's tree names. if harnessLFSConfig() != nil { args := append(lfsEndpointPins(bareURL), "lfs", "push", "origin", branch) if out, err := git(10*time.Minute, clean, args...); err != nil { return fail("nothing was pushed: lfs push: " + truncate(scrubURL(out, spec.RepoURL), 220)) } } if out, err := git(60*time.Second, clean, "push", "origin", branch); err != nil { return fail("push: " + truncate(scrubURL(out, spec.RepoURL), 220)) } sha, _ := git(10*time.Second, clean, "rev-parse", "HEAD") res["branch"], res["commit"] = branch, strings.TrimSpace(sha) res["meter_s"] = round2(time.Since(start).Seconds()) res["ok"] = true b, _ := json.Marshal(res) return b, 0 } // harnessGitSafe goes on every git the harness runs: no hook, no file // system monitor and no ext:: transport, whatever a repository's config says. var harnessGitSafe = []string{"-c", "core.hooksPath=" + os.DevNull, "-c", "core.fsmonitor=false", "-c", "protocol.ext.allow=never"} // harnessGitAuth runs git (in the folder in, if not "") with harnessGitSafe and // an environment that brings no config of this machine's: no system file, no // global file, HOME and XDG_CONFIG_HOME at home (an empty folder), and no // GIT_ variable of this process's. auth (repoGitAuth's environment) is added // to the environment, so a credential is never in a process argument, where // any local user can read it. Output as runCmd gives it. func harnessGitAuth(timeout time.Duration, home string, auth []string, in string, args ...string) (string, error) { a := append(append([]string{}, harnessGitSafe...), harnessLFSConfig()...) if in != "" { a = append(a, "-C", in) } a = append(a, args...) ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() cmd := exec.CommandContext(ctx, "git", a...) hideConsole(cmd) cmd.Env = append(harnessGitEnv(home), auth...) var out, errb bytes.Buffer cmd.Stdout, cmd.Stderr = &out, &errb if err := cmd.Run(); err != nil { return strings.TrimSpace(out.String() + " " + errb.String()), err } return out.String(), nil } func harnessGitEnv(home string) []string { env := []string{} for _, kv := range os.Environ() { k, _, _ := strings.Cut(kv, "=") switch k = strings.ToUpper(k); { case strings.HasPrefix(k, "GIT_"), k == "HOME", k == "XDG_CONFIG_HOME": continue } env = append(env, kv) } // The harness identity on every call, so git never makes one from this // machine's user and host name: that costs a host name lookup per call // (seconds where there is no resolver) and writes the name into the // reflog of the clone the agent is handed. return append(env, "HOME="+home, "XDG_CONFIG_HOME="+home, "GIT_CONFIG_NOSYSTEM=1", "GIT_CONFIG_GLOBAL="+os.DevNull, "GIT_TERMINAL_PROMPT=0", "GIT_AUTHOR_NAME="+harnessGitName, "GIT_AUTHOR_EMAIL="+harnessGitEmail, "GIT_COMMITTER_NAME="+harnessGitName, "GIT_COMMITTER_EMAIL="+harnessGitEmail) } // The name and address a harness run's commits carry. const ( harnessGitName = "harness" harnessGitEmail = "harness@privatae.ai" ) // harnessLFSConfig is the git-lfs filter, as `git lfs install` writes it, // when git-lfs is installed here. No global config is read, so a repository // that keeps files in LFS would otherwise be cloned as pointer files and its // changed large files committed whole. It comes from this process only: a // filter the repository or the agent names under any other name has no // command, so it never runs. func harnessLFSConfig() []string { if _, err := exec.LookPath("git-lfs"); err != nil { return nil } return []string{"-c", "filter.lfs.clean=git-lfs clean -- %f", "-c", "filter.lfs.smudge=git-lfs smudge -- %f", "-c", "filter.lfs.process=git-lfs filter-process", "-c", "filter.lfs.required=true"} } // lfsEndpointPins sets every LFS address git-lfs reads to the one it derives // from repoURL itself, so a .lfsconfig in the agent's tree cannot send the // push anywhere else. git config wins over .lfsconfig. repoURL is the URL // without its credential (repoGitAuth sends that, to this repository only). // Only an http(s) address is pinned; for any other, git-lfs is left to derive // the address as it does. func lfsEndpointPins(repoURL string) []string { u, err := url.Parse(repoURL) if err != nil || (u.Scheme != "https" && u.Scheme != "http") { return nil } ep := strings.TrimSuffix(repoURL, "/") if !strings.HasSuffix(ep, ".git") { ep += ".git" } ep += "/info/lfs" var a []string for _, k := range []string{"lfs.url", "lfs.pushurl", "remote.origin.lfsurl", "remote.origin.lfspushurl"} { a = append(a, "-c", k+"="+ep) } return a } // repoURLParts is repoURL without its user and password, and the secret it // carried (the password, or a user name standing alone, which is how some // forges take a token). A URL with no user part comes back as it is. func repoURLParts(repoURL string) (bare, secret string, err error) { u, perr := url.Parse(repoURL) if perr != nil { if strings.Contains(repoURL, "://") { return "", "", errors.New("the repository URL does not parse") } return repoURL, "", nil // a local path or user@host:path, which holds no token } if u.User == nil { return repoURL, "", nil } secret = u.User.Username() if p, ok := u.User.Password(); ok && p != "" { secret = p } u.User = nil return u.String(), secret, nil } // repoGitAuth is how git reaches repoURL with no credential in a process // argument or a config file: the URL without its user and password, and the // environment that sends them as an Authorization header to that URL and the // addresses under it only (http..extraHeader, which git and git-lfs both // read, set through GIT_CONFIG_COUNT as the mirror's push sets its own). A // URL with no user part needs no environment. func repoGitAuth(repoURL string) (bare string, env []string, err error) { bare, secret, err := repoURLParts(repoURL) if err != nil || secret == "" { return bare, nil, err } u, err := url.Parse(repoURL) if err != nil { return "", nil, err } pass, _ := u.User.Password() cred := base64.StdEncoding.EncodeToString([]byte(u.User.Username() + ":" + pass)) return bare, []string{"GIT_CONFIG_COUNT=1", "GIT_CONFIG_KEY_0=http." + bare + ".extraHeader", "GIT_CONFIG_VALUE_0=Authorization: Basic " + cred}, nil } // dropRepoCredential makes the clone at repoDir name its origin without the // token repoURL carries, then refuses it if the token is still in any file of // its .git (objects aside, which hold only the repository's content). func dropRepoCredential(git func(time.Duration, string, ...string) (string, error), repoDir, repoURL string) error { bare, secret, err := repoURLParts(repoURL) if err != nil || secret == "" { return err } if out, err := git(15*time.Second, repoDir, "remote", "set-url", "origin", bare); err != nil { return fmt.Errorf("remove the token from the agent's clone: %s", out) } if len(secret) < 8 { return nil // too short to be a token, and would match ordinary text } gitDir := filepath.Join(repoDir, ".git") return filepath.WalkDir(gitDir, func(p string, d fs.DirEntry, werr error) error { if werr != nil { return werr } if d.IsDir() { if p == filepath.Join(gitDir, "objects") || p == filepath.Join(gitDir, "lfs") { return filepath.SkipDir } return nil } if !d.Type().IsRegular() { return nil } b, err := os.ReadFile(p) if err != nil { return err } if bytes.Contains(b, []byte(secret)) { rel, _ := filepath.Rel(repoDir, p) return fmt.Errorf("the token is still in the agent's clone, in %s", filepath.ToSlash(rel)) } return nil }) } // driveHarnessAgent runs the harness IMAGE as a sidecar server (PORT=8844, /work bind-mounted) // and drives its REAL agent via POST /run with the prompt. The // agent works cwd in /work, so the worktree is modified in place (the caller collects it). The agent's // OWN per-run credential (mint->use->revoke, never baked into the image) is delivered via // spec.AgentCreds; absent => the image's baked-in auth (the test path). Always rm's the container. // Returns the harness /run JSON ({ok, exit_code, transcript, diff, usage, ...}). func driveHarnessAgent(rt, dir string, spec harnessSpec, maxMin int) (agentRes map[string]interface{}, runErr error) { const port = 8844 // Run the SAME image the harness-pool Deployment runs, with the SAME bind env it // passes — not invented flags. The sidecar's bind defaults to 127.0.0.1 // (the same-pod loopback model); the pool Deployment sets // PRIVATAE_SIDECAR_BIND=:: + PRIVATAE_APP_ID (both spellings below). On the box we reach the container by its // BRIDGE IP (containerIP below), so without the bind override the server listens on // loopback and waitReachable never connects ("not reachable on :8844"). `::` // serves the IPv4 bridge addr too (the sidecar leaves V6ONLY off) — verified on the box. // NOTE: the pool also sets HTTP_PROXY to the cluster's egress proxy because cluster pods // have NO direct egress; the box owns its own internet, so the box run egresses directly // (not via a platform proxy) — the one intentional difference from the pool env. // // The non-root harness (uid 10005) must READ AND WRITE the whole bind-mounted /work // tree — its cwd is /work and it edits files in place. Its contents are 0644/0755 // and this process's own, so the harness could not even enter it: POST /run dies // "Permission denied: /work" in seconds. So the ephemeral per-run tree is opened to // every user (it's removed after), and it must sit in a folder no other user can // enter: that folder is what keeps the rest of this machine out of the run's tree. if err := harnessTreeEnclosed(dir); err != nil { return nil, err } openHarnessTree(dir) // Both spellings of the sidecar's two variables: the harness image is // published on its own schedule, so a box run must work against one that // reads either name. The older pair comes out once none does. args := []string{"run", "-d", "-v", dir + ":/work", "-e", "PORT=" + strconv.Itoa(port), "-e", "PRIVATAE_SIDECAR_BIND=::", "-e", "PRIVATAE_APP_ID=" + spec.HarnessID, "-e", "SKEMA_SIDECAR_BIND=::", "-e", "SKEMA_APP_ID=" + spec.HarnessID} // Deliver the agent's OWN credential as the FILE the harness sidecar reads // (/home/privatae/.claude/.credentials.json — the claudeAiOauth JSON) — NOT an env var (the // sidecar reads + self-refreshes the FILE, ignoring CLAUDE_CODE_OAUTH_TOKEN). Mount a temp // .claude dir holding it, then DELETE the dir after the run so nothing durable is left on the // box. The host pulls these creds from the entity vault per run. if spec.AgentCreds != "" { credDir, credRemove, err := writeHarnessCreds(spec.AgentCreds) if credRemove != "" { defer os.RemoveAll(credRemove) } if err != nil { return nil, fmt.Errorf("agent credential: %w", err) } args = append(args, "-v", credDir+":/home/privatae/.claude") } args = append(args, spec.Image) out, err := runCmd(60*time.Second, rt, args...) if err != nil { return nil, fmt.Errorf("harness server start: %s", truncate(out, 200)) } cid := strings.TrimSpace(out) // The caller reads the tree once this returns, so the agent must be gone // by then: one still running could go on writing it while it is read. A // container whose removal is not confirmed fails the run, and nothing of // its tree is used. defer func() { if out, err := runCmd(20*time.Second, rt, "rm", "-f", cid); err != nil { agentRes = nil runErr = fmt.Errorf("the agent's container %s could not be removed, so nothing of its tree was used: %s", shortID(cid), truncate(strings.TrimSpace(out), 160)) } }() ip := containerIP(rt, cid) if ip == "" { return nil, fmt.Errorf("harness server: no container IP") } base := fmt.Sprintf("http://%s:%d", ip, port) if ok, _ := waitReachable(base+"/", 45*time.Second); !ok { return nil, fmt.Errorf("harness sidecar not reachable on :%d", port) } reqBody, _ := json.Marshal(map[string]interface{}{ "prompt": spec.Prompt, "project_dir": "/work", "max_minutes": maxMin, }) client := &http.Client{Timeout: time.Duration(maxMin+2) * time.Minute} resp, err := client.Post(base+"/run", "application/json", bytes.NewReader(reqBody)) if err != nil { return nil, fmt.Errorf("harness /run: %v", err) } defer resp.Body.Close() rb, err := readHarnessRunReply(resp.Body) if err != nil { return nil, err } var res map[string]interface{} if err := json.Unmarshal(rb, &res); err != nil { return nil, fmt.Errorf("harness /run bad json (HTTP %d): %s", resp.StatusCode, truncate(string(rb), 200)) } return res, nil } // readHarnessRunReply reads the agent container's answer to /run. The // container writes it, and it goes back to the host in one lane message, so it // is read to the most that message carries (uvicornWSMaxSize) and no further: // a reply that never ends must not grow in this machine's memory. func readHarnessRunReply(body io.Reader) ([]byte, error) { rb, err := io.ReadAll(io.LimitReader(body, uvicornWSMaxSize+1)) if err != nil { return nil, fmt.Errorf("harness /run: reading the reply: %v", err) } if len(rb) > uvicornWSMaxSize { return nil, fmt.Errorf("harness /run: the reply is more than the %d MiB the lane to the host carries", uvicornWSMaxSize>>20) } return rb, nil } // harnessAgentUID is the user the harness image runs its agent as, the owner // of its home /home/privatae, where the credential folder is mounted. const harnessAgentUID = 10005 // writeHarnessCreds puts the agent's credential in a new folder for one run, // and returns the folder to mount and the folder to remove after the run (the // second even with an error). The agent must read the file and write beside // it (its Bash tool makes session-env there, and a token refresh rewrites the // file). As root the folder (0700) and the file (0600) are handed to the // agent's user. A process that is not root cannot do that, so it opens them // to every user (0777, 0666) as it opens the run's tree; either way the // folder sits in one only this process can enter, so no other local user can // reach the credential. func writeHarnessCreds(creds string) (mount, remove string, err error) { remove, err = os.MkdirTemp("", "harness-claude-*") if err != nil { return "", "", err } d := filepath.Join(remove, "claude") f := filepath.Join(d, ".credentials.json") dirMode, fileMode := os.FileMode(0o777), os.FileMode(0o666) if os.Geteuid() == 0 { dirMode, fileMode = 0o700, 0o600 } if err := os.Mkdir(d, 0o700); err != nil { return "", remove, err } if err := os.WriteFile(f, []byte(creds), 0o600); err != nil { return "", remove, err } if os.Geteuid() == 0 { for _, p := range []string{f, d} { if err := os.Lchown(p, harnessAgentUID, harnessAgentUID); err != nil { return "", remove, err } } } if err := os.Chmod(f, fileMode); err != nil { return "", remove, err } if err := os.Chmod(d, dirMode); err != nil { return "", remove, err } return d, remove, nil } // harnessTreeEnclosed refuses a run tree whose folder another local user // could enter: that folder must be a plain folder (never a link) and, where // the system has file modes, have no access for group or others and, where // it has owners, be this process's own. The tree itself is opened to every // user for the agent (openHarnessTree), so this folder is all that keeps the // rest of the machine out of it. func harnessTreeEnclosed(dir string) error { parent := filepath.Dir(filepath.Clean(dir)) fi, err := os.Lstat(parent) if err != nil { return fmt.Errorf("run tree: %w", err) } if !fi.IsDir() { return fmt.Errorf("run tree: %s is not a folder (a link is never followed)", parent) } if runtime.GOOS == "windows" { return nil } if perm := fi.Mode().Perm(); perm&0o077 != 0 { return fmt.Errorf("run tree: %s is open to other users (%04o); the tree is opened to every user, so it must sit in a folder only this process can enter", parent, perm) } if uid, ok := coderFileOwner(fi); ok && uid != os.Geteuid() { return fmt.Errorf("run tree: %s belongs to uid %d, not to this process", parent, uid) } return nil } // openHarnessTree gives every user read and write on the run tree, for the // agent's uid: folders 0777, files 0666, or 0777 for a file that was // executable, so a run commits no mode change of its own. A link is left as // it is: chmod follows it, and the tree comes from the repository or the // host, so a link in it may name any file on this machine. func openHarnessTree(dir string) { _ = filepath.WalkDir(dir, func(p string, d fs.DirEntry, werr error) error { if werr != nil { return nil } mode := os.FileMode(0o777) switch { case d.IsDir(): case d.Type().IsRegular(): if fi, err := d.Info(); err != nil || fi.Mode().Perm()&0o111 == 0 { mode = 0o666 } default: return nil // a link, a socket, a device: not opened, never followed } _ = os.Chmod(p, mode) return nil }) } // scrubURL removes a credential-bearing URL (and its token) from output, so a clone/push error // never echoes the per-run token back over the lane. func scrubURL(s, url string) string { if url == "" { return s } s = strings.ReplaceAll(s, url, "") if at := strings.LastIndex(url, "@"); at > 0 { if cut := strings.LastIndex(url[:at], ":"); cut > 0 { if tok := url[cut+1 : at]; len(tok) > 6 { s = strings.ReplaceAll(s, tok, "") } } } return s } // extractTar unpacks a tar byte stream into dir (paths confined to dir; escapes skipped). func extractTar(data []byte, dir string) error { tr := tar.NewReader(bytes.NewReader(data)) for { hdr, err := tr.Next() if err == io.EOF { return nil } if err != nil { return err } target := filepath.Join(dir, hdr.Name) if target != filepath.Clean(dir) && !strings.HasPrefix(target, filepath.Clean(dir)+string(os.PathSeparator)) { continue // path escapes dir — skip (defense against a crafted tar) } switch hdr.Typeflag { case tar.TypeDir: if err := os.MkdirAll(target, 0o755); err != nil { return err } case tar.TypeReg: if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { return err } f, err := os.OpenFile(target, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644) if err != nil { return err } _, cerr := io.Copy(f, tr) f.Close() if cerr != nil { return cerr } } } } // makeTar packs dir (recursively; plain files and folders, skipping .git) into // a tar byte stream. packWorktree also says what it left out. func makeTar(dir string) ([]byte, error) { b, _, err := packWorktree(dir) return b, err } // uvicornWSMaxSize is the largest WebSocket message the host takes in: the // uvicorn default, which the host's server does not override. A harness // result is one such message, so this is the most a run can return. const uvicornWSMaxSize = 16 << 20 // maxReturnedTreeBytes is the largest tar a run sends back. It rides base64 // in the result (4/3 of its size), so 9 MiB is 12 MiB on the lane and leaves // 4 MiB of the 16 MiB message for the transcript, the diff and the rest. var maxReturnedTreeBytes int64 = 9 << 20 // maxCopiedTreeBytes is the most a git run copies from the agent's tree into // the clone it pushes. That push goes over git, not the lane, and the copy // carries the whole tree, so the lane's limit does not fit it; it is set at // what a large forge takes in one push (GitHub refuses a push over 2 GB), // which also bounds the disk this process writes for a run. var maxCopiedTreeBytes int64 = 2 << 30 // errTreeTooLarge is a tree with more in it than the run may carry. var errTreeTooLarge = errors.New("too large") // treeTooLarge says which entry took the tree over its limit. func treeTooLarge(rel string, limit int64) error { return fmt.Errorf("%s takes the tree over %d MiB: %w", rel, limit>>20, errTreeTooLarge) } // packWorktree packs dir into a tar byte stream and lists, by their path in // the tree, the entries it left out. // // A run's tree is written by the agent, which may still be writing it while // it is packed, and this process may be root: a name seen as a plain file can // be a link to any file on this machine by the time it is opened, and a folder // seen as a folder can be a link to any folder. So the tree is walked by open // folder (packTree, per system): each name is opened relative to the folder // already open above it, a link is never followed, and what goes in is what // was opened. Only a plain file or a folder goes in; a link, a pipe, a device, // a socket, or a name that changed between the look and the open is left out // and listed. // // Files are streamed from the opened file, never read whole, and the tar // stops at maxReturnedTreeBytes: a tree larger than that is an error, never // a shorter tar. func packWorktree(dir string) ([]byte, []string, error) { b, skips, err := packWorktreeSkips(dir) return b, skips.names, err } // packWorktreeSkips is packWorktree with the count of left-out entries. func packWorktreeSkips(dir string) ([]byte, treeSkips, error) { var buf bytes.Buffer cw := &treeCapWriter{w: &buf, limit: maxReturnedTreeBytes} sink := &tarSink{tw: tar.NewWriter(cw), cw: cw} if err := packTree(sink, filepath.Clean(dir)); err != nil { return nil, treeSkips{}, err } if err := sink.tw.Close(); err != nil { return nil, treeSkips{}, err } return buf.Bytes(), sink.treeSkips, nil } // treeSink takes what packTree opened, a folder before what is in it. rel is // the entry's path in the tree with / between names. listed hears each name // as the walk reads it from its folder, in batches of treeListBatch, and // refuses it once the tree holds more entries than the sink takes; folder // hears all the names in a folder, rel, once they are read and before any of // them is put; skip hears each entry the walk leaves out. type treeSink interface { listed(rel string) error put(f *os.File, fi os.FileInfo, rel string) error folder(rel string, names []string) error skip(rel string) } // treeListBatch is how many names the walk reads from a folder at a time, so // a folder of any size is refused after at most this many names past the // limit, never read whole first. const treeListBatch = 512 // readTreeNames reads the names in the open folder d (rel in the tree, with / // between names) treeListBatch at a time, handing each to sink.listed as it // is read, and returns them sorted. func readTreeNames(sink treeSink, d *os.File, rel string) ([]string, error) { var names []string for { batch, err := d.Readdirnames(treeListBatch) for _, n := range batch { p := n if rel != "" { p = rel + "/" + n } if lerr := sink.listed(p); lerr != nil { return nil, lerr } } names = append(names, batch...) if err == io.EOF || (err == nil && len(batch) == 0) { break } if err != nil { return nil, fmt.Errorf("read folder %q: %w", rel, err) } } sort.Strings(names) return names, nil } // treeSkips counts the entries a walk left out and keeps the first // maxSkippedReported of their paths, so a tree of any number of links holds // no more than that in memory. type treeSkips struct { names []string count int } func (s *treeSkips) skip(rel string) { s.count++ if len(s.names) < maxSkippedReported { s.names = append(s.names, rel) } } // report puts the left-out entries in a run's result: the count in full and // the names kept, sorted. func (s treeSkips) report(res map[string]interface{}) { if s.count == 0 { return } names := append([]string(nil), s.names...) sort.Strings(names) res["worktree_skipped_count"], res["worktree_skipped"] = s.count, names } // treeEntryCost is what an entry costs a copy beside its content: a tar // header block, as each entry costs the tar a worktree run returns, and its // name. It bounds how many entries a tree may hold, empty files included. func treeEntryCost(rel string) int64 { return 512 + int64(len(rel)) } // treeCapWriter passes writes on until limit bytes have gone through, then // refuses them with errTreeTooLarge (contextextract.go's capWriter drops // them instead). type treeCapWriter struct { w io.Writer n int64 limit int64 } func (c *treeCapWriter) Write(p []byte) (int, error) { if c.n+int64(len(p)) > c.limit { return 0, errTreeTooLarge } n, err := c.w.Write(p) c.n += int64(n) return n, err } // tarSink writes each entry into a tar. Every entry pays its header in the // tar itself; listed also charges each name read against the tar's limit, // so the names the walk holds stay within it. type tarSink struct { tw *tar.Writer cw *treeCapWriter listCost int64 treeSkips } func (s *tarSink) listed(rel string) error { if s.listCost += treeEntryCost(rel); s.listCost > s.cw.limit { return treeTooLarge(rel, s.cw.limit) } return nil } func (s *tarSink) folder(string, []string) error { return nil } // put writes the header of an entry packTree opened and, for a file, its // bytes, copied from the opened file. The size in the header is the opened // file's size; a file over what is left of the limit is refused before a // byte of it is read, and one that gets shorter while it is copied is an // error, never a short entry. func (s *tarSink) put(f *os.File, fi os.FileInfo, rel string) error { hdr, err := tar.FileInfoHeader(fi, "") if err != nil { return err } hdr.Name = rel if fi.IsDir() { if err := s.tw.WriteHeader(hdr); err != nil { return s.sized(rel, err) } return nil } // A header block, the content padded to a block, and the two blocks that // end the tar. if size := fi.Size(); size > s.cw.limit-s.cw.n-3*512 { return treeTooLarge(rel, s.cw.limit) } if err := s.tw.WriteHeader(hdr); err != nil { return s.sized(rel, err) } n, err := io.CopyN(s.tw, f, hdr.Size) if err == io.EOF { return fmt.Errorf("%s got shorter while it was packed (%d of %d bytes)", rel, n, hdr.Size) } if err != nil { return s.sized(rel, fmt.Errorf("read %s: %w", rel, err)) } return nil } func (s *tarSink) sized(rel string, err error) error { if errors.Is(err, errTreeTooLarge) { return treeTooLarge(rel, s.cw.limit) } return err } // copySink copies each entry into the folder root, the clone a git run // pushes. It writes only plain files and folders it makes itself: whatever is // at a name in root before (a file, a folder, a link the repository holds) is // removed first and never written through. A name .git in any letter case, // at any depth, is left out, so nothing the agent wrote can stand in for the // clone's own repository or a nested one. // // Every entry is charged treeEntryCost besides its bytes, and nothing is kept // per entry, so neither the disk a copy writes nor the memory it holds grows // past the limit however many files the tree has. type copySink struct { root string limit int64 total int64 treeSkips } // inDotGit says whether any name in rel is one dotGitName refuses. func inDotGit(rel string) bool { for _, name := range strings.Split(rel, "/") { if dotGitName(name) { return true } } return false } // charge takes size bytes of rel's content from what is left of the limit. func (s *copySink) charge(rel string, size int64) error { if size > s.limit-s.total { return treeTooLarge(rel, s.limit) } s.total += size return nil } // listed charges each name the walk reads its treeEntryCost, whether or not // it is then copied. func (s *copySink) listed(rel string) error { return s.charge(rel, treeEntryCost(rel)) } // folder removes from the clone's folder rel every name the agent's folder // does not hold, except the clone's own .git. A name the agent's folder holds // but the copy leaves out (a link, a special file, a folder too deep) keeps // what the clone has, the repository's version or nothing. It runs before any // entry of the folder is copied, so a name that differs only in letter case // on a system that ignores case is removed before the new one is written. func (s *copySink) folder(rel string, names []string) error { if rel != "" && inDotGit(rel) { return nil } dir := filepath.Join(s.root, filepath.FromSlash(rel)) ents, err := os.ReadDir(dir) if err != nil { return err } in := make(map[string]bool, len(names)) for _, n := range names { in[n] = true } for _, e := range ents { if in[e.Name()] || (rel == "" && e.Name() == ".git") { continue } if err := os.RemoveAll(filepath.Join(dir, e.Name())); err != nil { return err } } return nil } func (s *copySink) put(f *os.File, fi os.FileInfo, rel string) error { if inDotGit(rel) { s.skip(rel) return nil } dst := filepath.Join(s.root, filepath.FromSlash(rel)) if fi.IsDir() { if st, err := os.Lstat(dst); err == nil && !st.IsDir() { if err := os.RemoveAll(dst); err != nil { return err } } if err := os.Mkdir(dst, 0o755); err != nil && !errors.Is(err, fs.ErrExist) { return err } return nil } size := fi.Size() if err := s.charge(rel, size); err != nil { return err } if err := os.RemoveAll(dst); err != nil { return err } mode := os.FileMode(0o644) if fi.Mode().Perm()&0o111 != 0 { mode = 0o755 } // O_EXCL: the name was just emptied, so this makes a new file and opens // no link. out, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_EXCL, mode) if err != nil { return err } n, err := io.CopyN(out, f, size) if cerr := out.Close(); err == nil { err = cerr } if err == io.EOF { return fmt.Errorf("%s got shorter while it was copied (%d of %d bytes)", rel, n, size) } if err != nil { return fmt.Errorf("copy %s: %w", rel, err) } return os.Chmod(dst, mode) // the mode as asked, whatever the umask } // dotGitName says whether a file system may open name as .git: any letter // case; on Windows also with dots or spaces after it, or a stream after a // colon, or as its short name GIT~1; on HFS+ also with the code points it // ignores in a name. These are the names git itself refuses in a tree // (is_ntfs_dotgit, is_hfs_dotgit), checked on every system, since a name the // agent made on one is copied by a bastion that may run on another. func dotGitName(name string) bool { name = strings.Map(func(r rune) rune { switch { case r >= 0x200c && r <= 0x200f, r >= 0x202a && r <= 0x202e, r >= 0x206a && r <= 0x206f, r == 0xfeff: return -1 } return r }, name) if i := strings.IndexByte(name, ':'); i >= 0 { name = name[:i] } name = strings.TrimRight(name, ". ") return strings.EqualFold(name, ".git") || strings.EqualFold(name, "git~1") } // mirrorTree makes the clone at dst hold the agent's tree at src: every // plain file and folder in src is copied in, and whatever the clone has that // src does not is deleted, except the clone's own .git. An entry the copy // leaves out (a link, a special file, a .git) keeps what the clone has at // that name, the repository's own version or nothing, so a link the agent // made is never committed and one the repository holds is not deleted by a // run that could not copy it (copySink.folder). Returns the entries left out. func mirrorTree(src, dst string, limit int64) (treeSkips, error) { sink := ©Sink{root: dst, limit: limit} err := packTree(sink, filepath.Clean(src)) return sink.treeSkips, err } // packTreeBeforeOpen, when set, runs between packTree's look at an entry and // its open: the gap a concurrent writer aims for. Only tests set it. var packTreeBeforeOpen func(rel string) // maxTreeDepth bounds how deep packTree goes, since every level holds one // folder open. A folder deeper than this is left out and listed. const maxTreeDepth = 128 // maxSkippedReported bounds the left-out names a run reports; the count is // always reported in full. const maxSkippedReported = 100 // fitLane is the result a harness job sends: body, or, when body is more than // one message on the lane can carry (uvicornWSMaxSize, less the frame's tag), // a refusal that says so. The host would drop an oversized message and the // run would read as lost. func fitLane(body []byte, code int) ([]byte, int) { if len(body) < uvicornWSMaxSize { return body, code } var head struct { Mode string `json:"mode"` } _ = json.Unmarshal(body, &head) b, _ := json.Marshal(map[string]any{"ok": false, "mode": head.Mode, "result_too_large": true, "error": fmt.Sprintf("the run's result is %d MiB, more than the %d MiB the lane to the host carries; nothing from it was returned", len(body)>>20, uvicornWSMaxSize>>20)}) if code == 0 { code = 1 } return b, code } // detectRuntime returns the first working container runtime, or "" if none. func detectRuntime() string { for _, rt := range []string{"docker", "podman", "nerdctl"} { if _, err := runCmd(5*time.Second, rt, "ps", "-q"); err == nil { return rt } } return "" } // runCmd runs a command with a timeout; returns clean stdout on success, stdout+stderr on error. func runCmd(timeout time.Duration, name string, args ...string) (string, error) { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() cmd := exec.CommandContext(ctx, name, args...) hideConsole(cmd) // console_windows.go var out, errb bytes.Buffer cmd.Stdout, cmd.Stderr = &out, &errb if err := cmd.Run(); err != nil { return strings.TrimSpace(out.String() + " " + errb.String()), err } return out.String(), nil } // containerIP returns the container's bridge IP — reachable from the host with no published port. func containerIP(rt, cid string) string { out, err := runCmd(10*time.Second, rt, "inspect", "-f", "{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}", cid) if err != nil { return "" } return strings.TrimSpace(out) } // waitReachable polls url until it answers (any HTTP status = the server is up) or the deadline. func waitReachable(url string, within time.Duration) (bool, int) { deadline := time.Now().Add(within) client := &http.Client{Timeout: 3 * time.Second} for time.Now().Before(deadline) { if resp, err := client.Get(url); err == nil { code := resp.StatusCode resp.Body.Close() return true, code } time.Sleep(500 * time.Millisecond) } return false, 0 } func hostnameSafe() string { h, _ := os.Hostname(); return h } func orUnknown(s string) string { if strings.TrimSpace(s) == "" { return "unknown" } return s } func shortID(s string) string { s = strings.TrimSpace(s) if len(s) > 12 { return s[:12] } return s } func round2(f float64) float64 { return float64(int(f*100+0.5)) / 100 } func sendHarness(ws *websocket.Conn, tag byte, payload []byte) { _ = ws.WriteMessage(websocket.BinaryMessage, append([]byte{tag}, payload...)) } func sendHarnessExit(ws *websocket.Conn, code int) { sendHarness(ws, 'X', []byte(strconv.Itoa(code))) }