3 Commits

Author SHA1 Message Date
Privatae 4012da0b75 bastion 0.56.0
ci / personal-data (push) Successful in 10s
ci / test (push) Successful in 5m30s
ci / fuzz (push) Successful in 3m16s
2026-10-28 02:24:07 +00:00
Privatae 8822d5df85 bastion 0.55.3
ci / personal-data (push) Successful in 5s
ci / test (push) Successful in 3m51s
ci / fuzz (push) Successful in 2m55s
2026-10-25 23:16:45 +00:00
Privatae 77b7853ae1 bastion 0.54.1
ci / personal-data (push) Successful in 6s
ci / fuzz (push) Successful in 2m49s
ci / test (push) Successful in 3m7s
2026-10-21 23:39:13 +00:00
185 changed files with 16484 additions and 3195 deletions
+3
View File
@@ -0,0 +1,3 @@
# Test tooling for one test lab, built only with -tags lab; it is not part of
# any release build, so it is left out of the published source.
client-go/lab.go export-ignore
+25 -5
View File
@@ -16,7 +16,7 @@ on:
branches: [main] branches: [main]
env: env:
GO_IMAGE: golang@sha256:1ae0735f00daffa3aaf1363a5184c0d2dc55c78e3db4ec70241cdac97bf84b59 GO_IMAGE: golang@sha256:27f2d3b83cfc5803c8078f8e730af0f7c431bf067afaec8f60a36fb763187cba
# The pull request's own head, or the pushed commit. # The pull request's own head, or the pushed commit.
WANT: ${{ gitea.event.pull_request.head.sha || gitea.sha }} WANT: ${{ gitea.event.pull_request.head.sha || gitea.sha }}
@@ -58,7 +58,10 @@ jobs:
echo "== gofmt"; out="$(gofmt -l .)"; [ -z "$out" ] || { echo "$out"; exit 1; } echo "== gofmt"; out="$(gofmt -l .)"; [ -z "$out" ] || { echo "$out"; exit 1; }
for os in linux darwin windows; do echo "== go vet ($os)"; GOOS=$os go vet ./...; done for os in linux darwin windows; do echo "== go vet ($os)"; GOOS=$os go vet ./...; done
echo "== go build"; go build -o /tmp/privatae-bastion . echo "== go build"; go build -o /tmp/privatae-bastion .
echo "== go test"; go test -count=1 ./...' if [ -f lab.go ]; then echo "== go vet -tags lab"; go vet -tags lab ./...; fi
echo "== go test"; go test -count=1 ./...
echo "== staticcheck"; sh ../deploy/staticcheck.sh
echo "== the GitHub release step, against a fake API"; python3 -I ../deploy/test_github_release.py'
- name: the same for deploy/release-sign - name: the same for deploy/release-sign
run: | run: |
docker run --rm --cpus 6 -e GOMAXPROCS=6 -v "$PWD":/src -w /src/deploy/release-sign \ docker run --rm --cpus 6 -e GOMAXPROCS=6 -v "$PWD":/src -w /src/deploy/release-sign \
@@ -80,9 +83,20 @@ jobs:
test "$(git rev-parse HEAD)" = "$WANT" test "$(git rev-parse HEAD)" = "$WANT"
- name: no person's data in the tree - name: no person's data in the tree
# Reads files, runs nothing from the tree. The term check needs the # Reads files, runs nothing from the tree. The term check needs the
# maintainers' key, which a CI job never holds, so here it is skipped and # maintainers' key and the entity-id check the reserved ids, which a CI
# the run says so; the entity-id, provenance, email and phone checks run. # job never holds, so the job names files that do not exist: both
run: python3 scripts/test_no_personal_data.py # classes are skipped and the run prints a NOTE for each; the
# provenance, email and phone checks run. Left at their defaults, a
# path that exists but cannot be read fails the run instead.
run: |
none="$PWD/.ci-holds-none/none"
NO_PERSONAL_DATA_KEY_FILE="$none" RESERVED_ENTITY_IDS_FILE="$none" python3 scripts/test_no_personal_data.py
- name: every test the claims table cites exists
# docs/claims.md names, for each security claim, the test that fails
# if the claim stops being true. A renamed or deleted test would leave
# the claim looking covered; this fails the run instead. Reads files
# and greps; runs nothing from the tree.
run: bash deploy/check-claims.sh
- name: no Host source file names in the tree - name: no Host source file names in the tree
# The member repository publishes this tree as a release was built, # The member repository publishes this tree as a release was built,
# and its release step refuses a path into the Host's Python # and its release step refuses a path into the Host's Python
@@ -117,8 +131,14 @@ jobs:
- name: every fuzz target, 5 seconds each - name: every fuzz target, 5 seconds each
# A short pass on every commit. Run deploy/fuzz.sh with its default of # A short pass on every commit. Run deploy/fuzz.sh with its default of
# 20 seconds before changing a wall a host's input goes through. # 20 seconds before changing a wall a host's input goes through.
# fuzz.sh does not fail on Go's own -fuzztime report, on the Go
# releases whose fuzzer reports its deadline as a failure, and fails on
# anything else; test_fuzz.sh checks both, with planted failing
# inputs, before the targets run, and fails once the image's Go no
# longer needs the workaround.
run: | run: |
docker run --rm --cpus 6 -e GOMAXPROCS=6 -v "$PWD":/src -w /src \ docker run --rm --cpus 6 -e GOMAXPROCS=6 -v "$PWD":/src -w /src \
-e GOTOOLCHAIN=local -e CGO_ENABLED=0 "$GO_IMAGE" sh -euc ' -e GOTOOLCHAIN=local -e CGO_ENABLED=0 "$GO_IMAGE" sh -euc '
apk add -q --no-cache bash apk add -q --no-cache bash
bash deploy/test_fuzz.sh
deploy/fuzz.sh 5' deploy/fuzz.sh 5'
+248
View File
@@ -0,0 +1,248 @@
# The checks of .gitea/workflows/ci.yml, for the copy of this repository on
# GitHub. Gitea reads .gitea/workflows when it exists and then ignores this
# directory, so each forge runs one of the two files, never both.
#
# The jobs, image and commands are the same as there. Every job runs in the Go
# toolchain image deploy/build.sh builds releases with, pinned by the same
# digest, so CI and the release use the same compiler. GitHub does not let a
# job's container name come from env, so the digest is written out in each
# job; it must match GO_IMAGE in deploy/build.sh and .gitea/workflows/ci.yml.
# No action from the marketplace is used: the commit is fetched with git from
# this repository's public URL, so nothing outside this file and the pinned
# image runs. The job token can only read.
name: ci
on:
push:
branches: [main]
pull_request:
branches: [main]
permissions:
contents: read
env:
# The pull request's own head, or the pushed commit.
WANT: ${{ github.event.pull_request.head.sha || github.sha }}
REPO_URL: ${{ github.server_url }}/${{ github.repository }}.git
GOTOOLCHAIN: local
CGO_ENABLED: "0"
defaults:
run:
shell: sh -eu {0}
jobs:
test:
runs-on: ubuntu-24.04
timeout-minutes: 30
container:
image: golang@sha256:27f2d3b83cfc5803c8078f8e730af0f7c431bf067afaec8f60a36fb763187cba
env:
PRIVATAE_REQUIRE_AGE: "1"
steps:
- name: tools
# pdftotext, python3, setpriv and git are what the extract and
# peer-identity tests call; without them those tests fail rather than
# skip. Alpine's git has no http-backend (it is in git-daemon), which
# the forge-token test serves its repository with.
run: apk add -q --no-cache poppler-utils python3 setpriv curl git git-daemon
- name: fetch the commit
# The checkout belongs to the runner's user and the job runs as root,
# so git is told the directory is safe; go build reads it to stamp the
# binary with the commit.
run: |
set -o pipefail
git config --global --add safe.directory "$PWD"
git init -q .
git fetch -q --depth=1 "$REPO_URL" "$WANT"
git checkout -q --detach FETCH_HEAD
test "$(git rev-parse HEAD)" = "$WANT"
- name: age
# The backup tests check this code against the age command itself: a
# pinned age release, its archive checked against the SHA-256 below,
# and PRIVATAE_REQUIRE_AGE makes those tests fail rather than skip if
# it is missing.
run: |
case "$(uname -m)" in
x86_64) a=amd64; sum=cbe24006683f8eb669266162894b9a522a1af52f2665fbc63a4bb032ed26ac10 ;;
aarch64) a=arm64; sum=6b8dc4333c53a5a57c9e5834e3a48f92605d7154014cd07269ff3327db5d37f4 ;;
*) echo "no pinned age release for $(uname -m)"; exit 1 ;;
esac
curl -fsSL -o /tmp/age.tgz "https://github.com/FiloSottile/age/releases/download/v1.3.2/age-v1.3.2-linux-$a.tar.gz"
echo "$sum /tmp/age.tgz" | sha256sum -c -
tar -xzf /tmp/age.tgz -C /tmp age/age
/tmp/age/age --version
echo "AGE_BIN=/tmp/age/age" >> "$GITHUB_ENV"
- name: gofmt, go vet on linux/darwin/windows, build, go test
working-directory: client-go
run: |
echo "== gofmt"; out="$(gofmt -l .)"; [ -z "$out" ] || { echo "$out"; exit 1; }
for os in linux darwin windows; do echo "== go vet ($os)"; GOOS=$os go vet ./...; done
echo "== go build"; go build -o /tmp/privatae-bastion .
echo "== go test"; go test -count=1 ./...
- name: the same for deploy/release-sign
working-directory: deploy/release-sign
run: |
out="$(gofmt -l .)"; [ -z "$out" ] || { echo "$out"; exit 1; }
for os in linux darwin windows; do GOOS=$os go vet ./...; done
go test -count=1 ./...
- name: staticcheck on linux, darwin and windows
# The pinned staticcheck release, built with this toolchain from the
# module proxy and checked against the checksum database.
run: sh deploy/staticcheck.sh
- name: the release workflow's GitHub step, against a fake API
run: python3 -I deploy/test_github_release.py
personal-data:
runs-on: ubuntu-24.04
timeout-minutes: 10
container:
image: golang@sha256:27f2d3b83cfc5803c8078f8e730af0f7c431bf067afaec8f60a36fb763187cba
steps:
- name: tools
run: apk add -q --no-cache bash git python3
- name: fetch the commit
run: |
set -o pipefail
git config --global --add safe.directory "$PWD"
git init -q .
git fetch -q --depth=1 "$REPO_URL" "$WANT"
git checkout -q --detach FETCH_HEAD
test "$(git rev-parse HEAD)" = "$WANT"
- name: this file's image is the image releases are built with
# Each job here names the digest itself, so a change to GO_IMAGE in
# deploy/build.sh that missed this file fails here.
run: |
want="$(sed -n 's/^GO_IMAGE="\(golang@sha256:[0-9a-f]\{64\}\)"$/\1/p' deploy/build.sh)"
test -n "$want"
test "$(grep -c "^ image: $want\$" .github/workflows/ci.yml)" -eq 3
test "$(grep -c '^ image: ' .github/workflows/ci.yml)" -eq 3
grep -q "^ GO_IMAGE: $want\$" .gitea/workflows/ci.yml
grep -q "^ GO_IMAGE: $want\$" .github/workflows/ci.yml
- name: no person's data in the tree
# Reads files, runs nothing from the tree. The term check needs the
# maintainers' key and the entity-id check the reserved ids, which
# this job never holds, so both are skipped and the run prints a NOTE
# for each; the provenance, email and phone checks run.
run: python3 -I scripts/test_no_personal_data.py
- name: every test the claims table cites exists
# docs/claims.md names, for each security claim, the test that fails
# if the claim stops being true. A renamed or deleted test would leave
# the claim looking covered; this fails the run instead. Reads files
# and greps; runs nothing from the tree.
run: bash deploy/check-claims.sh
- name: no Host source file names in the tree
# The member repository publishes this tree as a release was built,
# and its release step refuses a path into the Host's Python
# (server/... or brain/... .py): this repo has no such files, so the
# path can only name another program's source. git grep exits 1 when
# nothing matches and above 1 when it could not search; only 1 passes.
run: |
rc=0
git grep -nIE 'server/[a-z_/]+\.py|brain/[a-z_/]+\.py' || rc=$?
case "$rc" in
1) ;;
0) echo "these lines name another program's source file; describe it instead" >&2
exit 1 ;;
*) echo "git grep could not search the tree (exit $rc), so it was not checked" >&2
exit 1 ;;
esac
fuzz:
needs: test
runs-on: ubuntu-24.04
timeout-minutes: 20
container:
image: golang@sha256:27f2d3b83cfc5803c8078f8e730af0f7c431bf067afaec8f60a36fb763187cba
steps:
- name: tools
run: apk add -q --no-cache bash git
- name: fetch the commit
run: |
set -o pipefail
git config --global --add safe.directory "$PWD"
git init -q .
git fetch -q --depth=1 "$REPO_URL" "$WANT"
git checkout -q --detach FETCH_HEAD
test "$(git rev-parse HEAD)" = "$WANT"
- name: the fuzz runner fails on a failing input and on nothing else
# fuzz.sh does not fail on Go's own -fuzztime report, on the Go
# releases whose fuzzer reports its deadline as a failure, and fails on
# anything else; this checks both, with planted failing inputs, before
# the targets run, and fails once the image's Go no longer needs the
# workaround.
run: bash deploy/test_fuzz.sh
- name: every fuzz target, 5 seconds each
# A short pass on every commit. Run deploy/fuzz.sh with its default of
# 20 seconds before changing a wall a host's input goes through.
run: bash deploy/fuzz.sh 5
claims-on-a-machine:
# The claim tests a container cannot run: the op and action-op sandboxes
# (root, systemd units, overlayfs, a cgroup with the pids controller),
# the egress tests in private user and network namespaces, and the root
# command watch (nftables). They run here on the runner's own virtual
# machine, as root, because that is what they test: the bastion as a
# networked machine's system service. The machine is GitHub's and is
# discarded after the job. The toolchain is the one in the pinned image.
# Every test docs/claims.md cites is run, and the job fails if any of
# them skips.
needs: test
runs-on: ubuntu-24.04
timeout-minutes: 30
# This job runs on the runner itself, not in the image, and Ubuntu
# 24.04's sh is dash 0.5.12-6ubuntu5, which has no pipefail (Debian added
# it in 0.5.12-7): `set -o pipefail` there is a fatal error. bash has it.
defaults:
run:
shell: bash -eu {0}
env:
GO_IMAGE: golang@sha256:27f2d3b83cfc5803c8078f8e730af0f7c431bf067afaec8f60a36fb763187cba
steps:
- name: fetch the commit
run: |
set -o pipefail
git init -q .
git fetch -q --depth=1 "$REPO_URL" "$WANT"
git checkout -q --detach FETCH_HEAD
test "$(git rev-parse HEAD)" = "$WANT"
- name: tools, and the pinned toolchain on the machine
run: |
sudo apt-get update -qq
sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends nftables iproute2 poppler-utils >/dev/null
# The forge-token claim test needs git 2.46 or later; Ubuntu 24.04's
# own git is 2.43.
if ! git version | awk '{split($3, v, "."); exit !(v[1] > 2 || (v[1] == 2 && v[2] >= 46))}'; then
sudo add-apt-repository -y ppa:git-core/ppa >/dev/null
sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq git >/dev/null
fi
git version
docker create --name go "$GO_IMAGE" >/dev/null
sudo docker cp go:/usr/local/go /opt/go-pinned
docker rm go >/dev/null
/opt/go-pinned/bin/go version
- name: age
# The same pinned age release as the test job.
run: |
case "$(uname -m)" in
x86_64) a=amd64; sum=cbe24006683f8eb669266162894b9a522a1af52f2665fbc63a4bb032ed26ac10 ;;
aarch64) a=arm64; sum=6b8dc4333c53a5a57c9e5834e3a48f92605d7154014cd07269ff3327db5d37f4 ;;
*) echo "no pinned age release for $(uname -m)"; exit 1 ;;
esac
curl -fsSL -o "$RUNNER_TEMP/age.tgz" "https://github.com/FiloSottile/age/releases/download/v1.3.2/age-v1.3.2-linux-$a.tar.gz"
echo "$sum $RUNNER_TEMP/age.tgz" | sha256sum -c -
tar -xzf "$RUNNER_TEMP/age.tgz" -C "$RUNNER_TEMP" age/age
echo "AGE_BIN=$RUNNER_TEMP/age/age" >> "$GITHUB_ENV"
- name: every cited test, as root
working-directory: client-go
run: |
set -o pipefail
tests="$(grep -o '_test\.go`: `[A-Za-z0-9_]*`' ../docs/claims.md | sed 's/.*`: `//; s/`$//' | sort -u | paste -sd'|' -)"
test -n "$tests"
sudo env PATH="/opt/go-pinned/bin:$PATH" HOME="$HOME" AGE_BIN="$AGE_BIN" PRIVATAE_REQUIRE_AGE=1 GOTOOLCHAIN=local CGO_ENABLED=0 \
GOFLAGS=-mod=readonly GOMODCACHE="$HOME/go/pkg/mod" GOCACHE="$HOME/.cache/go-build" \
go test -count=1 -json -run "^($tests)\$" . > "$RUNNER_TEMP/claims.json" || {
grep -h '"Action":"fail"' "$RUNNER_TEMP/claims.json" | head -40; exit 1; }
- name: none of them skipped
run: bash deploy/check-claims.sh --skipped "$RUNNER_TEMP/claims.json"
+67
View File
@@ -0,0 +1,67 @@
# Attaches each release's binaries, its signed manifest and the manifest's
# signature to the release on GitHub, for the copy of this repository there.
# Gitea reads .gitea/workflows when it exists and then ignores this directory.
#
# Nothing here is trusted to say what the release is. deploy/release-assets.sh
# rebuilds the binaries from the release tag with the tag's own pinned build,
# writes the release manifest from them, and fetches the signature published
# for that manifest, which only the release key holder can make and which is
# named by the manifest's digest. Unless the rebuilt bytes are exactly the
# signed release, there is no signature by that name and nothing is attached.
# The release key never comes near this workflow.
#
# A tag push runs it once the release is published. Run it by hand
# (workflow_dispatch, from main) to attach the files to an earlier release
# whose tag is here; its signature is still published, so that works for any
# release from 0.53.2 on.
#
# No action from the marketplace is used: the commit is fetched with git from
# this repository's public URL, and the build runs in the Go toolchain image
# deploy/build.sh pins by digest. The job's token can write the release and
# is passed only to the last step.
name: release
on:
push:
tags: ['v[0-9]+.[0-9]+.[0-9]+']
workflow_dispatch:
inputs:
tag:
description: the release tag, vX.Y.Z
required: true
permissions:
contents: read
defaults:
run:
shell: bash -euo pipefail {0}
jobs:
assets:
runs-on: ubuntu-24.04
timeout-minutes: 40
permissions:
contents: write
env:
TAG: ${{ github.event_name == 'workflow_dispatch' && inputs.tag || github.ref_name }}
REPO_URL: ${{ github.server_url }}/${{ github.repository }}.git
steps:
- name: fetch this workflow's commit and the release tags
run: |
git init -q .
git fetch -q --tags "$REPO_URL" "$GITHUB_SHA"
git checkout -q --detach FETCH_HEAD
test "$(git rev-parse HEAD)" = "$GITHUB_SHA"
- name: rebuild the release and check it against its signature
run: deploy/release-assets.sh "$TAG" "$RUNNER_TEMP/assets"
- name: attach the files to the release
# The one step with the token. Its shell, git and Python are named by
# path, and Python runs isolated (-I), so nothing an earlier step put
# on PATH or in the environment chooses what runs here.
shell: /usr/bin/bash -euo pipefail {0}
env:
GH_TOKEN: ${{ github.token }}
run: |
/usr/bin/git show "refs/tags/$TAG:CHANGELOG.md" > "$RUNNER_TEMP/CHANGELOG.md"
/usr/bin/python3 -I deploy/github-release.py "$GITHUB_REPOSITORY" "$TAG" "$RUNNER_TEMP/assets" "$RUNNER_TEMP/CHANGELOG.md"
+403 -7
View File
@@ -23,6 +23,402 @@ From 0.39.1 on, a heading marks a version *not published* or *held* once
that is settled. The newest entries may not be published yet; the manifest that is settled. The newest entries may not be published yet; the manifest
says which version is. Older entries are not marked. says which version is. Older entries are not marked.
## 0.56.0
- **New: a Host with no platform can use the model servers on this
computer.** A Host running on this computer, or in a virtual machine on it,
can reach the model servers here (Ollama, a local language model server, a
personal-data filter) by name through the bastion, and is told which of
them it can use. It starts working once your Host supports it.
- **Changed: for a Host with no platform, model management reaches only this
computer.** Listing, downloading and removing models for such a Host now
reach the model server set here only when that server is on this computer.
One set to another computer is refused, and the bastion's log says which
setting is not used and what to set. Such a Host is also told only of
the model servers it can use, and no longer of the service slots, or of
coder runs when it is in a virtual machine, since neither runs for it.
- **Changed: on a local machine, the internet lane reaches only public
addresses.** A Host can no longer use a local machine's internet lane (the
egress lane) to reach devices on your home or office network at their
private addresses, such as 192.168.x.x or 10.x.x.x, or the private ranges
many VPNs use, as was already the case on an anchored machine. Your own
machines are reached through their approved lanes. A device that has a
public address, as many do over IPv6, is still reached at that address.
- **Changed: plain-HTTP requests through the bastion are checked one per
connection.** On the internet lane and a coder run's web proxy, a
plain-HTTP (not HTTPS) connection now carries one request, which is checked
before it is sent on; anything sent after it closes the connection. Before,
later requests on the same connection reached the destination unchecked. A
malformed or ambiguous request is refused, and every refusal now says why.
- **Safer: file access in a folder you shared stays inside it.** When a Host
lists, reads or moves files in a folder you shared with it, the bastion
stays inside that folder. If the folder, or any folder on its path, is later
swapped for a link to somewhere else, the bastion refuses until the folder
is shared again, and a file or folder inside it swapped for a link is never
followed out of it. Search, which goes through the folder's own search
agent, is not part of this change.
- **Safer: the bastion limits how much it holds of what a Host sends.** On
the internet lane, the terminal and root commands, coder runs, the memory
mirror (the copy of your Host's database kept here) and the bastion's
check-in with its Host, a message from the Host is read only up to a set
size and refused past it, or passed along or dropped as it arrives. A Host
can no longer make the bastion hold a message of any size in memory. A
coder run's web proxy now carries at most 1024 connections at once, and the
memory mirror at most 8.
- **Safer: asking to share a folder with a Host has limits.** Any user on the
computer can send that ask to the bastion, so each ask is now read only up
to a set size, and the number of asks open at once is capped, per user and
in total; past the cap, the ask is told the bastion is busy and to try
again.
- **For contributors: the fuzz check in CI no longer fails on a timing bug in
Go's own fuzzer.** A real failure still fails it. The program itself is
unchanged by this.
## 0.55.3
- **A pairing change reaches every Host on the machine at once.** Each pairing
beats its Host every 60 seconds, and the beat is how every Host on the
machine learns that a pairing went active or was revoked. Approving or
revoking a pairing changed it on disk immediately but triggered no beat, so
every Host kept the old state for up to a minute: a connection request to a
Host the anchor had just approved was refused as not on a shared machine. Now
a grant that makes a pairing active, and a revoke, make every running
pairing beat right away. A refused grant, or a re-sync of a pairing that is
already active, beats nothing.
## 0.55.2
- **A search reply says how its text was sanitized.** Every successful reply
from the bastion agent (search, read range, catalog) was stamped
`sanitized: true`, while the agent the bastion deploys runs in `flag` mode,
which replaces e-mail addresses, phone numbers, card, bank and ID numbers,
IP and MAC addresses and keys, and leaves names, street addresses and
dates of birth in. A reply now carries `sanitize_mode` (the mode this
bastion wrote into the agent's manifest, or for an agent run by hand the
mode it reports, else `unknown`), `sanitized` true only for `sterilize`,
and `sanitize_note` saying what the mode leaves in. A reply cannot raise
the mode this bastion wrote.
- **The README says what the bastion needs.** It said "no Docker, no Python".
The bastion itself needs neither, and the README now names the optional
features that do: coder runs, the service slot and the memory mirror need a
container runtime, folder search needs Python 3.8 or later, Python ops need
`/usr/bin/python3`, and the Linux accessibility reader needs python3 with
AT-SPI. It also said Go 1.25; `client-go/go.mod` needs 1.26.
- **GitHub releases carry their files.** A new workflow,
`.github/workflows/release.yml`, runs on each release tag (and by hand for
an earlier one): `deploy/release-assets.sh` rebuilds the release from its
tag, writes the release manifest from the rebuilt binaries with
`deploy/manifest.py` (which `publish.sh` now uses too, so the two cannot
differ), and fetches the signature published for that manifest. Only when
the signature verifies are the six binaries, `version.json`, its signature
and `SHA256SUMS` attached to the release. Releases 0.53.2, 0.54.0 and
0.54.1 were rebuilt this way and each matched its signed manifest.
- **The lab harness is out of the release build.** `mode=lab` drove one test
lab's own cluster and database, and its layout was in the published source.
It is now built only with `-tags lab` and is not in the published source; a
release bastion answers a lab run with exit 126, as a bastion without a lab
always did.
- **staticcheck is clean** on linux, darwin and windows, for both Go modules,
and CI runs it (`deploy/staticcheck.sh`, a pinned staticcheck release, with
the per-OS runs merged so code used on one OS only is not reported unused).
Unused code is removed and error strings no longer end in punctuation; no
behaviour changes.
- **The claim tests a container cannot run now run on GitHub.** 25 tests the
claims table cites (the op and action-op sandboxes, the egress tests in
private namespaces, the root command watch) skipped in CI, because a CI
container has no systemd, no namespaces of its own and no nftables. A new
job runs every cited test as root on GitHub's discarded virtual machine and
fails if any skips; `deploy/check-claims.sh --skipped` names them.
## 0.55.1
- **A root op's tasks are capped.** An op's tasks were capped at 64 with
`RLIMIT_NPROC`, which the kernel never applies to root, so a root op could
start as many tasks as it liked until its time limit. A root op now runs
in a systemd scope of its own in `privatae_ops.slice` with `TasksMax=96`,
which counts every task in it, root's included: the op's 64 and the
bastion's two processes that start it. Those two run with one CPU's worth
of Go runtime threads (`GOMAXPROCS=1`), so they take the same share of the
cap on a machine of any size: with the runtime set for 64 CPUs they took
44 of the 96, and set for 256 no root op could start. The scope's process
reads its own cgroup's `pids.max` before the op starts, and the op's
report carries `tasks_capped` only when that is the cap. Where no scope
can be made (no systemd, no pids controller, or `systemd-run` failing),
the op runs as before and its report's `tasks_note` says its tasks were
not capped, and why. A user op's tasks are held by `RLIMIT_NPROC` as
before, and its report says so. A scope is killed with the run at the
time limit, when the bastion stops, and by the startup sweep. Two ops can
run at once, so an action op's memory watch now reads each other op's
unit or scope in the slice as another program: it yields when a root op
beside it makes the machine wait on memory, as it did when root ops ran
in the bastion's own cgroup, and its own unit still never counts.
## 0.55.0
- **A local machine's egress lane never reaches the machine itself.** Before,
on a machine with no anchor the lane vetted no destination, so a host could
dial this machine's own services through it: `POST
http://127.0.0.1:11434/api/pull` reached ollama whatever the `models` switch
said. It now refuses `localhost` names, loopback and unspecified addresses
(`0.0.0.0/8` and `::` included), link-local addresses (the cloud metadata
address included), the rest of `64:ff9b::/32`, the deprecated IPv6 forms
below, and any address this machine takes as its own, as the anchored lane
checks them, also inside an IPv4-mapped, NAT64 or 6to4 address. The LAN
stays reachable: a private, CGNAT or reserved address that is not this
machine's is dialled. Every address a name resolves to is checked, and only
those addresses are dialled, never the name again.
- **A container or VM on this machine is this machine, not the LAN.** On
Linux, an address the kernel routes straight onto a container's veth, or
onto a bridge whose every port is a veth, a tap or a dummy (the bridge
networks Docker, Podman, LXD and libvirt make for their containers and
VMs), is refused as this machine. A route through a gateway on such a link
leaves, and a link an IPv4 or IPv6 default route of the main table goes
out of (any nexthop's, for a multipath default) is never inside: a bastion
that is itself in a container reaches the internet through the machine it
runs on, and dials the network on its own veth, its LAN included, as
before. Before, the local egress lane dialled such an address as LAN,
so a host could reach a service in a container here, `172.17.0.2:11434`
for one, though the same service on loopback was refused. A bridge with a
network card, a bond or a VLAN among its ports is still the LAN. Every lane
reads the same check, so a coder run's proxy, the anchored lane and the
backup lanes now also refuse a public address on such a network; their
private addresses were refused already. macOS and Windows have no route
lookup here, so there such an address is still dialled.
- **Every lane refuses the deprecated IPv6 forms.** IPv4-compatible `::/96`,
IPv4-translated `::ffff:0:0/96` and site-local `fec0::/10` are no
destination of their own, and where each is delivered depends on the
platform. A coder run's proxy and the backup lanes already refused them; the
egress lane now does too, anchored or local, by address and in any answer a
name resolves to. The anchored lane refuses nothing else it did not refuse
before. All the lanes now read one address classifier.
- **Both egress lanes try each vetted address, as Happy Eyeballs does.** A
name with more than one address is dialled at each in order, the next
starting as soon as an attempt fails or when none has answered within
300 ms, and the first to answer is used; the others are cancelled, and one
that connects anyway is closed. Before, the anchored lane dialled only the
first address, and a coder run's proxy gave each address an even share of
the dial budget, so an address whose packets go nowhere held up the next
one for seconds. The next one now starts within 300 ms, about what the OS
dialler took when it was given the name.
- **`pair` says to enrol a second factor after every pairing that has none.**
It said so only after it anchored a networked machine, so a local pairing,
one over TCP or a unix socket, and a later host on an anchored machine were
told nothing, and on a local machine a folder grant then installs on the
host's signed grant alone. The line names `--host` when more than one host
is paired, and `sudo -H` only when `pair` itself ran with machine authority,
so a local install is told to enrol as its own user. A pairing that keeps
its factor (a `--replace`), or a machine with an approval device pinned, is
not told.
- **A root-command grant lasts at most 30 days.** It was a year. `root-exec
grant --for 31d` is refused and says the longest term; grant again before
it ends. A grant given under the longer term ends 30 days after it was
given, and `root-exec` shows that end. A grant file with no grant time is
refused, since it cannot be held to the term.
- **SECURITY.md, the README's license section and `docs/host-on-this-machine.md`
name each kind of Host.** They described only a Host on Privatae's platform.
They now say that a C-Series Host runs on your own machine in its own Linux
virtual machine, is open source (Privatae/cseries, AGPL-3.0-only), needs no
Privatae account, and that `npx cseries` sets the bastion up networked,
paired, anchored and with a second factor on every pairing (the commands are
shown); that C-Series Cloud and a network membership are optional and paid;
and that a Host on Privatae's own platform is the case where the Host's
software is not open source.
- **Security reports go to security@privatae.ai,** the address
privatae.ai/.well-known/security.txt names. SECURITY.md said info@.
- **`docs/models.md` says what stops a host changing your models on a local
machine.** It told you to scope the pairing on your host, which a local
machine never reads. There, switching `models` off for the host is what
filters the ollama forward, and the egress lane is no way around it. On an
anchored machine the forward filters for every pairing, which the page now
says too.
- **A tunnel endpoint the host sends must be one host and port.** The
pairing response's `endpoint` and `endpoints`, and the endpoints in a
heartbeat's rotate directive, were written into the WireGuard
configuration unchecked. That configuration is one setting per line, so an
endpoint with a newline in it added settings of its own:
`203.0.113.7:51820` followed by a newline and `allowed_ip=::/0` let the
tunnel carry every IPv6 address, not only the host's. An endpoint must now
be an IP address or a host name with a port from 1 to 65535, and nothing
else, no space or control character. A pairing response with any other is
refused and nothing is saved, and a rotate directive with one is not acted
on. A WireGuard key must be 32 bytes. The tunnel's peer is now configured in
one place, which checks every value it writes.
- **A pairing file that holds such an endpoint is listed but never served.**
An older build may have saved one. Its tunnel and lanes never come up, and
`run` names the file and says why. Every other pairing still loads and
serves, `status` shows the pairing as `invalid` with the same reason, and
`unpair` removes it, as does pairing that host again with `pair --replace`.
One such file no longer stops `run`, `status` and `unpair` for the whole
machine.
- **Such a pairing cannot hold the anchor or be approved, and no host is told
of it.** `claim-anchor` marked it active and anchored the machine to it; it
now refuses, says why, and changes nothing. An approval from the anchor
host marked it active and started it with the endpoint its file holds; it
is now refused. The heartbeat and the anchor host's pairing list leave it
out: a host knows only the states active, pending and revoked, and could
show a pairing in any other state as active.
- **A harness run's agent credential is readable only by the harness.** The
credential file was readable by every user on the machine (0644), in a
folder anyone could open and write (0777), for the length of the run. A
bastion running as root now hands the folder (0700) and the file (0600) to
the user the harness image runs its agent as (uid 10005). One that is not
root cannot hand them over, so it leaves them open for the agent, and the
folder sits inside one that only the bastion's own user can enter.
- **A harness run's tree is out of other users' reach.** In a worktree run the
tree mounted at `/work` was opened to every user (0777), its top folder
included, in the shared temporary folder. Any local user could read or
change the Host's tree during the run. It now sits in a folder only the
bastion's own user can enter, as a git run's tree already did, and a run
refuses a tree whose folder other users can enter. Opening the tree for the
agent no longer follows a link in it. Before, on a bastion running as root,
a link in the repository, to `/etc/shadow` for example, made the linked
file world-writable. A file is opened as 0666, or 0777 if it was
executable, so a git run no longer commits every file as executable.
- **A worktree run returns only what is in its tree.** A file swapped for a
link while the tree was being packed, or a folder swapped for a link, was
read through the link, so a bastion running as root could return any file
on the machine to the Host. Each name is now opened inside the folder
already open above it, never through a link, and goes in only if it is the
plain file or folder that was looked at. On Linux a device or pipe swapped
in for a file in that gap is not opened either: opening some devices does
something by itself. Links, pipes, devices and sockets are left out
(before, each came back as an empty entry of its kind), and the run's
result names them in `worktree_skipped`, with `worktree_skipped_count`.
- **A coding agent can no longer run code as the bastion through the
repository's git settings.** In a git run the bastion committed and pushed
in the agent's own folder, `.git` included, so a hook, a file system
monitor, a filter, a signing program or a changed remote the agent set ran
as the bastion's own user, which can be root, or sent the push, with its
token, to another server. The bastion now makes its own fresh clone, in a
folder the agent cannot reach, copies in the agent's plain files and
folders (never a link, never anything of its `.git`), and commits and
pushes only there. The branch gets one commit with the agent's final
files; the agent's own commits are not pushed one by one. A link the
agent made is left out and named in `worktree_skipped`; a link the
repository already holds stays as it is. Every git the bastion runs for a
harness or a mirror runs with no hooks, no file system monitor, no `ext::`
transport, and no system or user git settings of the machine.
- **The agent in a git run no longer holds the push token, and no process
argument shows it.** The agent's clone named the repository with the
run's token in it, so the agent could push straight to the repository
during the run, past the bastion's copy, and every git the bastion ran
for the run had the token in its arguments, which any local user can
read. Every git call now names the repository without the token and gets
it from its environment, as an `Authorization` header sent to that
repository's address only; no clone's config holds it, and a run is
refused if the token is in any file of the agent's clone's `.git`.
- **A user-privilege op is no longer refused because other programs run as
its user.** An op's tasks are capped at 64, and the cap counted every task
the machine ran as the op's user (`privatae-session`, or `nobody`, which
system daemons share), so on a busy machine a user op could not start or
could not fork at all. The op now runs in a user namespace of its own,
as the same user and seeing every file's real owner, where the cap counts
only its own tasks. Where the machine has no user namespaces, the op runs
as before.
- **A harness run's git calls carry the harness identity.** git made one
from this machine's user and host name instead, which cost a host name
lookup per call (seconds where there is no resolver) and wrote this
machine's name into the clone handed to the agent.
- **A run whose agent container cannot be removed uses nothing of its
tree.** A failed removal was ignored, so an agent still running could go
on writing the tree while the bastion read it. The run now fails, says
so, and copies, returns and pushes nothing.
- **A git run whose commit fails pushes nothing and fails.** A failed add or
commit was ignored and the branch pushed anyway, and the run read as
successful. It now ends the run with an error and no push. A run in which
the agent changed no file pushes no branch and reports `changed: false`.
- **Repositories that keep files in Git LFS work in git runs again.** No
user git settings are read any more, so the LFS filter `git lfs install`
sets there was lost. When `git-lfs` is installed, the bastion now supplies
the standard LFS filter itself on its own git commands, sends the large
files before the push, and sends them only to the repository's own LFS
address, whatever a `.lfsconfig` the agent wrote names. A filter of any
other name in a `.gitattributes` still runs nothing. Tested with git-lfs
3.7.1.
- **Names that open as `.git` on Windows or macOS are left out of a git
run's copy.** Besides `.git` in any letter case, the copy now leaves out
`.git` followed by dots or spaces or a `:`, the short name `GIT~1`, and
`.git` with the characters HFS+ ignores, the names git itself refuses, so
no file the agent made can be written into the clean clone's own `.git`.
- **A run's tree has a size limit, and a file is never read into memory
whole.** A file of any size in the tree, even one that takes no disk
space, was read whole into memory, so an agent could make the bastion run
out of memory. A worktree run now returns at most 9 MiB of tree, which is
what fits in one message to the Host beside the transcript; a larger tree
is refused with `worktree_too_large` and none of it is returned. A git run
copies at most 2 GiB, and refuses a larger tree with `tree_too_large`
before it pushes anything; each file and folder counts 512 bytes and its
name besides its content, so a tree of millions of empty files is refused
too, and the copy keeps nothing per file in memory. Of the entries left
out, the count is kept in full and the first 100 names. A folder is read
512 names at a time, each counted as it is read, so a folder of millions
of names is refused without being read whole first. A result too large for one message to the Host
is refused with `result_too_large` instead of being lost on the way.
- **The local console shows a pairing that is not served as "invalid".** A
pairing whose file holds an endpoint this build refuses showed as active
on the console when its stored state was empty, as it is in older files.
It now reads as the setup page and `status` read it.
- **Built with Go 1.26.10, golang.org/x/net 0.61.0, x/crypto 0.58.0 and x/sys
0.49.0.** `govulncheck` finds 13 known vulnerabilities this code reaches in
Go 1.25.14 and x/net 0.57.0, which 0.54.1 was built with: in crypto/x509
(intermediate certificates since found compromised were still trusted),
crypto/tls, net/http and HTTP/2, net/textproto, os and html/template. With
these versions it finds none. The
build image is still pinned by digest, and CI runs in the same one. The
module now needs Go 1.26, as x/net's fixed release does. A published release
is still rebuilt by `deploy/verify-release.sh` with the toolchain its own
tag pins.
- **`go test ./...` passes for a contributor who is not root.** A root
command now starts in `/` when the process cannot enter root's home, as it
already did when that home is missing; the bastion runs these commands only
as root, so on a machine nothing changes, and the root-command tests now run
as any user. `TestTreeCheckoutMovesOnlyForward`, which needs a tree root
owns, skips as another user, and `TestClaimAnchorWithNoToken` checks the
refusal without machine authority and then skips the rest, where it passed
having checked nothing more.
- **The tests name no folder of our own build machine.** The sandbox tests'
fixture folder is `/srv/privatae-optest` (`/tmp` cannot hold an op's
effects), and the action tests' run note asks for any folder outside
`/tmp`.
- **How this repository is published (docs only).** From this release on,
GitHub receives each change merged since the last release as its own commit,
titled as the change was, instead of one commit per release. The README,
CONTRIBUTING.md and SECURITY.md say so, and say that releases from 0.53.2 on
can be verified from a clone, since the earlier tags are not public.
- **The personal-data check reads what it used to miss.** A listed term
written right after a backslash escape (`\n`, `\t`, `\u00e9`) is found, and
so is any value inside a run of base64 or base64url that decodes to text.
The reserved entity ids are read from `/etc/privatae/reserved-entity-ids`
(one per line, `RESERVED_ENTITY_IDS_FILE` to point elsewhere) instead of
being written in the script; where that file is absent, as in public CI,
the entity-id check is skipped and the run prints a NOTE, as the term check
already does without its key. A file that holds no id, or a line that is
not one, fails the run.
- **The former-name check reads comments and Markdown.** A kept identifier
is set aside only in Go code; in a comment, this changelog or any other
file it is a hit like the bare word, and the two entries that spelled the
device-fingerprint prefix out now name `fpSANPrefix` instead.
- **CI on GitHub.** `.github/workflows/ci.yml` runs the jobs of
`.gitea/workflows/ci.yml` on GitHub Actions in the same Go image, pinned by
the same digest, with no action from the marketplace and a token that can
only read. A step fails if its digest stops matching `deploy/build.sh`.
- Tests use documentation addresses (`2001:db8::/32`, `203.0.113.0/24`) and
neutral paths, and comments state their reason instead of citing a pull
request by number; a test keeps published source free of such numbers.
## 0.54.1
- **No change to what the bastion does.** The README is now a short front
page: install, quick start, local and networked modes, a summary of the
security model, how this repository is published, and how to build and
verify a release. The reference text moved into one page per topic under
`docs/`, listed in `docs/README.md`, with only its headings and links
changed. The README and CONTRIBUTING.md now say that issues and pull
requests are welcome on GitHub.
- **The claims table is its own file, `docs/claims.md`,** and
`deploy/check-claims.sh` reads it there. Two rows that cited two tests each
now cite one test per row, so the check passes again.
## 0.54.0 ## 0.54.0
- **Every audit row also goes to the system log.** The rows written to - **Every audit row also goes to the system log.** The rows written to
@@ -212,13 +608,13 @@ says which version is. Older entries are not marked.
- **The device fingerprint is sent under a second name to a Host with no - **The device fingerprint is sent under a second name to a Host with no
platform.** A pairing with such a Host (`--redeem-url unix://…` or platform.** A pairing with such a Host (`--redeem-url unix://…` or
`https://…`) sends an operator certificate request that carries the `https://…`) sends an operator certificate request that carries the
fingerprint as two URI SANs: first `urn:skema:hwfp:<fp>` (`fpSANPrefix`), fingerprint as two URI SANs: first under the original device-fingerprint
then `urn:privatae:hwfp:<fp>`, which that Host reads. A platform pairing's URN prefix (`fpSANPrefix` in pair.go), then `urn:privatae:hwfp:<fp>`, which
request names it once, under `urn:skema:hwfp:`, as before: the platform that Host reads. A platform pairing's request names it once, under
signs a request with that one name and refuses one with any other. A `fpSANPrefix`, as before: the platform signs a request with that one name
signed accounting verifies with the fingerprint under either name, and a and refuses one with any other. A signed accounting verifies with the
signer certificate that names two different fingerprints is refused rather fingerprint under either name, and a signer certificate that names two
than resolved. different fingerprints is refused rather than resolved.
## 0.52.0 ## 0.52.0
+16 -4
View File
@@ -7,10 +7,15 @@ their approval on that machine.
## What you can do today ## What you can do today
Sign-in on this forge is not reachable from the internet, so you cannot file Issues and pull requests are welcome on GitHub, at
an issue or open a pull request here. You can read, clone, build and verify a [github.com/Privatae/bastion](https://github.com/Privatae/bastion).
release, and fork under the GPL. Report a bug or a security problem by email Development happens in a private repository, so an accepted change is carried
(see [SECURITY.md](SECURITY.md)). into the development repository and arrives here with the next release, as
its own commit. Sign-in on the git.privatae.ai forge is not reachable from
the internet, so issues and pull requests cannot be opened there. You can
also read, clone, build and verify a release, and fork under the GPL. Report
a security problem by email, not in a public issue (see
[SECURITY.md](SECURITY.md)).
## Checks a change has to pass ## Checks a change has to pass
@@ -20,8 +25,15 @@ gofmt -l . # must print nothing
go vet ./... go vet ./...
CGO_ENABLED=0 go build . CGO_ENABLED=0 go build .
go test ./... go test ./...
cd .. && deploy/staticcheck.sh # must find nothing, on linux, darwin and windows
``` ```
Some claim tests need a real machine and root: the op sandboxes, the
namespace tests and the root command watch skip in a container and as a
user. GitHub's CI runs every test `docs/claims.md` cites as root on its own
virtual machine and fails if one skips; to run them yourself, use a machine
you can throw away.
The backup tests also check this code against the age command itself when The backup tests also check this code against the age command itself when
one is on PATH or named by `AGE_BIN`; CI installs a pinned age release and one is on PATH or named by `AGE_BIN`; CI installs a pinned age release and
sets `PRIVATAE_REQUIRE_AGE`, so there they fail rather than skip. sets `PRIVATAE_REQUIRE_AGE`, so there they fail rather than skip.
+167 -1362
View File
File diff suppressed because it is too large Load Diff
+34 -11
View File
@@ -4,7 +4,7 @@
Please do not open a public issue for a security problem. Please do not open a public issue for a security problem.
Email **info@privatae.ai** with: Email **security@privatae.ai** with:
- the version (`privatae-bastion version`) and platform, - the version (`privatae-bastion version`) and platform,
- what you found and how to reproduce it, - what you found and how to reproduce it,
@@ -22,13 +22,15 @@ notes if you wish.
machine do more than the grant and the local approval allow: reading files machine do more than the grant and the local approval allow: reading files
outside a granted store, deleting where only archiving is allowed, running a outside a granted store, deleting where only archiving is allowed, running a
program that is not in the fixed table, bypassing the second factor, opening program that is not in the fixed table, bypassing the second factor, opening
a listener that is not documented in the README's security model. a listener that is not documented in the security model
([docs/security-model.md](docs/security-model.md)).
## What is out of scope ## What is out of scope
- The Privatae platform the bastion connects to. It is separate software with - The Host the bastion connects to. It is separate software: a C-Series Host
its own reporting path; reports about it sent to the address above are is [Privatae/cseries](https://github.com/Privatae/cseries), and a Host on
forwarded. Privatae's own platform is software that is not open source. Reports about
either sent to the address above are forwarded.
- Findings that require root on the machine already. The bastion's root ops - Findings that require root on the machine already. The bastion's root ops
assume the machine's administrator is trusted. assume the machine's administrator is trusted.
@@ -100,7 +102,7 @@ one installed.
**Checking our work.** Release builds are reproducible. **Checking our work.** Release builds are reproducible.
`deploy/verify-release.sh` fetches the published manifest, checks its `deploy/verify-release.sh` fetches the published manifest, checks its
signature, checks out the release's source, rebuilds it, and compares every signature, checks out the release's source, rebuilds it, and compares every
binary's digest. From 0.43.0 on, each public snapshot in this repository binary's digest. From 0.53.2 on, each published release in this repository
carries the annotated tag `v<version>`, whose message records the release's carries the annotated tag `v<version>`, whose message records the release's
`built_from`. That field names the commit in the repository the release was `built_from`. That field names the commit in the repository the release was
built in, which is not a commit here, so the script uses the tag and prints built in, which is not a commit here, so the script uses the tag and prints
@@ -113,9 +115,30 @@ the folder you granted. The agent is a separate program in its own repository
(`privatae/bastion-agent`), and it reads your files in order to index them. What (`privatae/bastion-agent`), and it reads your files in order to index them. What
leaves your machine is what its manifest and your grant allow. Its extraction leaves your machine is what its manifest and your grant allow. Its extraction
and PII redaction happen inside it, so check that repository, not this one, and PII redaction happen inside it, so check that repository, not this one,
for how well those work. for how well those work. The bastion runs it in `flag` mode, which does not
find names, street addresses or dates of birth, and every search reply says so
([DCN stores](docs/context.md#distributed-context-network-dcn-stores)).
**The platform.** Your host runs on our platform, which is not open source. The **The Host.** The bastion holds your Host to your grant and your approvals.
bastion keeps the host to your grant and your approvals whatever the platform What the Host does with what you share with it is the Host's own software, and
does. What the host does with what you share with it is governed by software which software that is depends on where the Host runs.
you cannot read here.
- **A C-Series Host** runs on your own machine, in its own Linux virtual
machine. Its source is [Privatae/cseries](https://github.com/Privatae/cseries),
under AGPL-3.0-only; the bastion and the bastion agent are GPL-3.0-or-later.
`npx cseries` installs the bastion beside it as the machine's system service
and, on every pairing, runs `set-mode networked`, then
`pair --redeem-url <the Host's https enroll address> --host-ca <its CA> -`
with the token on standard input, then `claim-anchor` and
`stepup-enroll --console-only --host <the Host>`: the machine is anchored to
that Host and you approve its writes here with your own authenticator. No
Privatae account is needed
([A Host in a VM on this machine](docs/host-on-this-machine.md#a-host-in-a-vm-on-this-machine)).
- **Optional and paid, from Privatae:** C-Series Cloud, which runs a Host on a
server of its own that Privatae sets up and keeps patched, and a Privatae
network membership, which joins a Host on your own machine to the Privatae
network.
- **A Host on Privatae's own platform** is the case where the Host's software
is not open source. There, the bastion keeps the Host to your grant and your
approvals whatever the platform does, and what the Host does with what you
share with it is governed by software you cannot read here.
+63 -5
View File
@@ -512,6 +512,30 @@ func resolveLLMIPs() map[string]string {
// to the anchor host, because approving is the anchor's job. "" = no pairing // to the anchor host, because approving is the anchor's job. "" = no pairing
// identity (tests, tools): mode, anchor and unapproved attempts still travel. // identity (tests, tools): mode, anchor and unapproved attempts still travel.
func buildHeartbeatBody(self, hostname, platform, version, wgPub, fingerprint, tunnelIP, endpoint string, func buildHeartbeatBody(self, hostname, platform, version, wgPub, fingerprint, tunnelIP, endpoint string,
savePending bool, mirror *mirrorReport, enforcing map[string]any, peers []peerAnnounce) string {
return buildHeartbeatBodyOn(beatLanes{}, self, hostname, platform, version, wgPub, fingerprint, tunnelIP, endpoint,
savePending, mirror, enforcing, peers)
}
// beatLanes is which of a pairing's lanes run, so its beat advertises those
// and no others. The zero value is a tunnel pairing: the forwards listen on
// the tunnel's stack and the service slots are reachable there.
type beatLanes struct {
// local: a Host with no platform (run.go runUnixPairing). The forwards and
// the service slots are absent; the model servers ride the serve lane
// (servelane.go), and the beat says which in lane_serves.
local bool
// noCoders: a tcp pairing, whose coder runs have no control plane to use
// (pair_tcp.go coderTransportRefusal), so it is never offered one.
noCoders bool
}
func beatLanesFor(cfg *Config) beatLanes {
return beatLanes{local: cfg.localTransport(), noCoders: cfg.tcpTransport()}
}
// buildHeartbeatBodyOn is buildHeartbeatBody for a pairing whose lanes are lanes.
func buildHeartbeatBodyOn(lanes beatLanes, self, hostname, platform, version, wgPub, fingerprint, tunnelIP, endpoint string,
savePending bool, mirror *mirrorReport, enforcing map[string]any, peers []peerAnnounce) string { savePending bool, mirror *mirrorReport, enforcing map[string]any, peers []peerAnnounce) string {
caps := []string{} caps := []string{}
// WHY SOME CAPABILITIES ARE WITHHELD HERE AND OTHERS ARE NOT — the asymmetry // WHY SOME CAPABILITIES ARE WITHHELD HERE AND OTHERS ARE NOT — the asymmetry
@@ -549,7 +573,17 @@ func buildHeartbeatBody(self, hostname, platform, version, wgPub, fingerprint, t
// gets exactly what it got before: scope.filterModels is the identity then. // gets exactly what it got before: scope.filterModels is the identity then.
scope := pairingModelScope(self) scope := pairingModelScope(self)
reachable, models := false, []string(nil) reachable, models := false, []string(nil)
if inference { // A local pairing has no forward to advertise: its model servers ride the
// serve lane, and each lane below is advertised only when lane_serves
// carries its service (the target set and this machine's, the server
// answering, the family held). Its probes are of the address the lane
// dials, never the tunnel's guesses.
var served laneServeReport
if lanes.local {
served = laneServes(self)
reachable = containsStr(served.served, "ollama")
models = served.models
} else if inference {
reachable, models = ollamaModels(tunnelIP) reachable, models = ollamaModels(tunnelIP)
models = scope.filterModels(models) models = scope.filterModels(models)
// Reachable but nothing shared: advertising `embed` would put this box // Reachable but nothing shared: advertising `embed` would put this box
@@ -572,7 +606,15 @@ func buildHeartbeatBody(self, hostname, platform, version, wgPub, fingerprint, t
// box running both. // box running both.
llmModels := []string{} llmModels := []string{}
llmPort := llmForwardPort llmPort := llmForwardPort
if !inference { if lanes.local {
// No port listens for it here: the Host reaches the server through the
// serve lane, so llm_port is the "no port" value.
llmPort = 0
if containsStr(served.served, "llm") {
caps = append(caps, "llm-openai")
llmModels = served.llmModels
}
} else if !inference {
// The host stores 0 as NULL; with llm-openai withheld and // The host stores 0 as NULL; with llm-openai withheld and
// llm_models empty nothing on the host resolves a port for this row. // llm_models empty nothing on the host resolves a port for this row.
llmPort = 0 llmPort = 0
@@ -598,7 +640,9 @@ func buildHeartbeatBody(self, hostname, platform, version, wgPub, fingerprint, t
// probe: an up forwarder with a dead upstream is a distinct state the // probe: an up forwarder with a dead upstream is a distinct state the
// host wants before committing a fail-closed write path to it. // host wants before committing a fail-closed write path to it.
piiAdvertised, piiOK := false, false piiAdvertised, piiOK := false, false
if inference && scope.allowsPII() { if lanes.local {
piiAdvertised, piiOK = containsStr(served.served, "pii"), served.piiOK
} else if inference && scope.allowsPII() {
piiAdvertised, piiOK = piiProbe() piiAdvertised, piiOK = piiProbe()
} }
if piiAdvertised { if piiAdvertised {
@@ -638,7 +682,7 @@ func buildHeartbeatBody(self, hostname, platform, version, wgPub, fingerprint, t
// "build a server on this machine" on it, and an offer the box would refuse // "build a server on this machine" on it, and an offer the box would refuse
// is a control that fails after the user chose it. The slot list rides with // is a control that fails after the user chose it. The slot list rides with
// it, withheld (absent, not empty) from a pairing outside the family. // it, withheld (absent, not empty) from a pairing outside the family.
servicesOn := servicesCapable(self) servicesOn := !lanes.local && servicesCapable(self)
if servicesOn { if servicesOn {
caps = append(caps, serviceCapability) caps = append(caps, serviceCapability)
} }
@@ -646,7 +690,10 @@ func buildHeartbeatBody(self, hostname, platform, version, wgPub, fingerprint, t
// only when this machine would accept a run-up from this pairing — Linux, // only when this machine would accept a run-up from this pairing — Linux,
// root, coders on, a runtime, the family, and (anchored) a standing coder // root, coders on, a runtime, the family, and (anchored) a standing coder
// grant. Absent, never empty, otherwise. // grant. Absent, never empty, otherwise.
coders := coderHeartbeat(self) var coders map[string]any
if !lanes.noCoders {
coders = coderHeartbeat(self)
}
if coders != nil { if coders != nil {
caps = append(caps, coderCapability) caps = append(caps, coderCapability)
} }
@@ -774,6 +821,13 @@ func buildHeartbeatBody(self, hostname, platform, version, wgPub, fingerprint, t
// absence as a client too old to say. // absence as a client too old to say.
"endpoint_save_pending": savePending, "endpoint_save_pending": savePending,
} }
// What this pairing serves over the serve lane (servelane.go), on a local
// pairing only: ["ollama","llm","pii"] or fewer, [] when nothing is. Absent
// on a tunnel pairing, which has the forwards instead, and from a bastion
// that predates the lane.
if lanes.local {
body["lane_serves"] = served.served
}
// The service slots this pairing may reach, with the tunnel port for each // The service slots this pairing may reach, with the tunnel port for each
// (service.go). Absent when the capability is withheld, never empty-for-no. // (service.go). Absent when the capability is withheld, never empty-for-no.
if servicesOn { if servicesOn {
@@ -1004,7 +1058,11 @@ func buildHeartbeatBody(self, hostname, platform, version, wgPub, fingerprint, t
} }
} }
if piiAdvertised { if piiAdvertised {
// pii_port is the tunnel's port; a local pairing has none and reaches
// the sterilizer through the serve lane.
if !lanes.local {
body["pii_port"] = piiForwardPort body["pii_port"] = piiForwardPort
}
body["pii_ok"] = piiOK body["pii_ok"] = piiOK
} }
if faAdvertised { if faAdvertised {
+15 -3
View File
@@ -810,6 +810,11 @@ type pairingEntry struct {
// --replace, so a pending host re-presenting a token keeps its original age // --replace, so a pending host re-presenting a token keeps its original age
// (the 10-minute unapproved clock does not restart). nil when the directory // (the 10-minute unapproved clock does not restart). nil when the directory
// cannot be read — the host keeps what it has, as with peers. // cannot be read — the host keeps what it has, as with peers.
//
// A pairing whose file this build refuses (Config.invalid) is left out: it is
// never served, so to a host it is not on this box. A host knows the states
// active, pending and revoked and may read any other as active, so "invalid"
// is never sent. The machine's own status and setup page still show it.
func pairingsReport() []pairingEntry { func pairingsReport() []pairingEntry {
cfgs, err := loadPairings() cfgs, err := loadPairings()
if err != nil { if err != nil {
@@ -817,7 +822,7 @@ func pairingsReport() []pairingEntry {
} }
out := make([]pairingEntry, 0, len(cfgs)) out := make([]pairingEntry, 0, len(cfgs))
for _, c := range cfgs { for _, c := range cfgs {
if c.HostName == "" { if c.HostName == "" || c.invalid != "" {
continue continue
} }
out = append(out, pairingEntry{HostName: c.HostName, Handle: handleOf(c.HostName), out = append(out, pairingEntry{HostName: c.HostName, Handle: handleOf(c.HostName),
@@ -849,7 +854,7 @@ func unapprovedAttempts(now time.Time) []peerAnnounce {
if err != nil || now.Sub(since) < unapprovedAfter { if err != nil || now.Sub(since) < unapprovedAfter {
continue continue
} }
out = append(out, peerAnnounce{HostName: e.HostName, Handle: e.Handle, State: e.State, Since: e.Since}) out = append(out, peerAnnounce(e))
} }
return out return out
} }
@@ -1113,9 +1118,16 @@ func cmdClaimAnchor(token string, rotate bool, owner, host string) error {
case len(cfgs) == 0: case len(cfgs) == 0:
return fmt.Errorf("this box has no pairing yet — pair it to your host first (`privatae-bastion pair <token>`, token from Network → Set up a bastion), then `privatae-bastion claim-anchor` with no token") return fmt.Errorf("this box has no pairing yet — pair it to your host first (`privatae-bastion pair <token>`, token from Network → Set up a bastion), then `privatae-bastion claim-anchor` with no token")
default: default:
return fmt.Errorf("this box serves %d hosts — say which one should hold the anchor:\n%s\n e.g. `privatae-bastion claim-anchor --host %s`\nThe anchor host approves every pairing made here from then on; the other pairings keep the lanes they already have.", return fmt.Errorf("this box serves %d hosts — say which one should hold the anchor:\n%s\n e.g. `privatae-bastion claim-anchor --host %s`\nThe anchor host approves every pairing made here from then on; the other pairings keep the lanes they already have",
len(cfgs), " "+strings.Join(hostNames(cfgs), "\n "), hostNames(cfgs)[0]) len(cfgs), " "+strings.Join(hostNames(cfgs), "\n "), hostNames(cfgs)[0])
} }
// A pairing whose file this build refuses is never served, so the box
// could not reach the host it would be anchored to, and that host would
// approve every pairing made here. Refused with the reason, before
// anything is written.
if cfg.invalid != "" {
return fmt.Errorf("%s cannot hold the anchor: %s", cfg.HostName, cfg.invalid)
}
// REVOKED is not "waiting", it is a decision, and it must not be undone by // REVOKED is not "waiting", it is a decision, and it must not be undone by
// a command the user ran to do something else. pair.go holds the same line // a command the user ran to do something else. pair.go holds the same line
// for a re-pair (pending again, never straight back to active), // for a re-pair (pending again, never straight back to active),
+1 -1
View File
@@ -606,7 +606,7 @@ func TestClaimAnchorWithNoToken(t *testing.T) {
if err := cmdClaimAnchor("", false, "", ""); err == nil || !contains(err.Error(), "machine authority") { if err := cmdClaimAnchor("", false, "", ""); err == nil || !contains(err.Error(), "machine authority") {
t.Fatalf("claim-anchor must need machine authority whether or not a token is given: %v", err) t.Fatalf("claim-anchor must need machine authority whether or not a token is given: %v", err)
} }
return t.Skip("the rest needs root: without machine authority claim-anchor stops at the refusal checked above")
} }
t.Run("local mode refuses — the token is not what gates this", func(t *testing.T) { t.Run("local mode refuses — the token is not what gates this", func(t *testing.T) {
-6
View File
@@ -32,9 +32,3 @@ func systemAuditWrite(line []byte) {
auditSyslog = nil auditSyslog = nil
} }
} }
// registerAuditEventSource has nothing to register outside Windows: syslog
// takes any tag.
func registerAuditEventSource() error { return nil }
func removeAuditEventSource() {}
-43
View File
@@ -16,7 +16,6 @@ import (
"path/filepath" "path/filepath"
"strings" "strings"
"testing" "testing"
"time"
"github.com/gorilla/websocket" "github.com/gorilla/websocket"
) )
@@ -64,48 +63,6 @@ func drive(t *testing.T, backupRoot string, req backupFetchReq) (string, string,
} }
} }
func awaitJob(t *testing.T, backupRoot, id string) backupJob {
t.Helper()
deadline := time.Now().Add(10 * time.Second)
for time.Now().Before(deadline) {
out, _, code := drive(t, backupRoot, backupFetchReq{Op: "status", JobID: id})
if code != 0 {
t.Fatalf("status exited %d", code)
}
var j backupJob
if err := json.Unmarshal([]byte(out), &j); err != nil {
t.Fatalf("status is not JSON: %v (%s)", err, out)
}
if j.State != "running" {
return j
}
time.Sleep(25 * time.Millisecond)
}
t.Fatal("job never left running")
return backupJob{}
}
func startAndAwait(t *testing.T, root string, req backupFetchReq) backupJob {
t.Helper()
out, errOut, code := drive(t, root, req)
if code != 0 {
t.Fatalf("start exited %d: %s", code, errOut)
}
var started struct {
ID string `json:"id"`
State string `json:"state"`
}
if err := json.Unmarshal([]byte(out), &started); err != nil {
t.Fatalf("start reply is not JSON: %v (%s)", err, out)
}
if started.State == "done" { // already-present short circuit
var j backupJob
_ = json.Unmarshal([]byte(out), &j)
return j
}
return awaitJob(t, root, started.ID)
}
func artifactServer(t *testing.T, body []byte) (*httptest.Server, string) { func artifactServer(t *testing.T, body []byte) (*httptest.Server, string) {
t.Helper() t.Helper()
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+3
View File
@@ -655,6 +655,9 @@ func cmdBackupDecide(host, prefix string, approve bool) error {
} }
} }
r, err = resealApprove(dir, who, id, code) r, err = resealApprove(dir, who, id, code)
if err != nil {
return err
}
} else { } else {
r, err = resealRefuse(dir, resealWho{Name: cfg.HostName}, id, "") r, err = resealRefuse(dir, resealWho{Name: cfg.HostName}, id, "")
} }
+34
View File
@@ -10,6 +10,7 @@ package main
import ( import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"os"
"path/filepath" "path/filepath"
"strings" "strings"
"testing" "testing"
@@ -149,3 +150,36 @@ func TestHostNumberComesFromTheTunnelAddress(t *testing.T) {
} }
} }
} }
// With a second factor enrolled, a wrong code is refused by resealApprove,
// and cmdBackupDecide returns that refusal. It once checked the outer err
// while the approval's error went to the one its `who, err :=` declared, so
// a wrong code fell through to r.State on a nil request and panicked.
func TestResealDecideWithAWrongCodeReturnsTheRefusal(t *testing.T) {
dirs := withFixturePairings(t, map[string]int64{fixtureHost: fixtureEntity})
fx := newBackupFixture(t, "")
fx.installAsBastionKey(t)
own, _ := age.GenerateX25519Identity()
id := resealStartAs(t, fixtureHost, dirs[fixtureHost], resealReq{Op: "start", Sidecar: string(fx.sidecar(t)),
To: own.Recipient().String(), Stamp: fixtureStamp, HostLabel: "B"})
if err := stepupSave(dirs[fixtureHost], &stepupFile{SecretB32: "GEZDGNBVGY3TQOJQGEZDGNBVGY3TQOJQ",
EnrolledAt: "2026-08-29T00:00:00Z", LastCounter: -1}); err != nil {
t.Fatal(err)
}
r, w, err := os.Pipe()
if err != nil {
t.Fatal(err)
}
_, _ = w.WriteString("not-a-code\n")
_ = w.Close()
realStdin := os.Stdin
os.Stdin = r
t.Cleanup(func() { os.Stdin = realStdin; _ = r.Close() })
if err := cmdBackupDecide(fixtureHost, id, true); err == nil {
t.Fatal("a wrong second-factor code approved the request")
}
if st := resealStatusAs(t, fixtureHost, dirs[fixtureHost], id); st["state"] == "done" || st["sidecar"] != "" {
t.Fatalf("a wrong code handed the backup key over: %v", st)
}
}
+80
View File
@@ -0,0 +1,80 @@
package main
import (
"context"
"sync"
"time"
)
// A beat out of turn, for every pairing on this box.
//
// Each running pairing beats its own Host on a 60-second ticker, and every
// beat carries the box's pairings as `peers` with the state of each (peers.go).
// That is how every Host on the machine learns that a pairing went active or was
// revoked. A pairing_grant_sync or pairing_grant_drop changes that state on disk
// at once, but nothing beat: every Host kept the old state for up to a minute.
// So a pairing approved a second after a beat still read as pending for the
// next minute, and a Host asking to connect to it in that time was refused. The
// anchor Host can read its own confirmed grant, but no other Host on the
// machine has one, and a revoke reached the other Hosts no sooner.
//
// So a change to a pairing's state asks every running pairing's beat loop to beat
// now. The signal carries nothing: the beat reads the pairings from disk as it
// always does, so it reports the state the change wrote. One pending signal per
// loop is kept and the rest are dropped, so a burst of changes is one extra beat
// per Host, never one per change.
var beatNow = struct {
sync.Mutex
subs map[chan struct{}]struct{}
}{subs: map[chan struct{}]struct{}{}}
// subscribeBeatNow registers one beat loop. The channel holds at most one
// pending signal; stop unregisters it.
func subscribeBeatNow() (<-chan struct{}, func()) {
ch := make(chan struct{}, 1)
beatNow.Lock()
beatNow.subs[ch] = struct{}{}
beatNow.Unlock()
return ch, func() {
beatNow.Lock()
delete(beatNow.subs, ch)
beatNow.Unlock()
}
}
// kickBeats asks every registered beat loop to beat now. It never blocks: a
// loop that already has a signal pending, or is mid-beat, beats once more.
func kickBeats() {
beatNow.Lock()
defer beatNow.Unlock()
for ch := range beatNow.subs {
select {
case ch <- struct{}{}:
default:
}
}
}
// runBeatLoop beats every `every`, and also whenever kickBeats asks. beforeTick
// runs before a ticked beat only (the tunnel's path failover, which belongs to
// the clock, not to a pairing change); it may be nil. It returns when ctx ends.
func runBeatLoop(ctx context.Context, every time.Duration, beforeTick, beat func()) {
kick, stop := subscribeBeatNow()
defer stop()
t := time.NewTicker(every)
defer t.Stop()
for {
select {
case <-ctx.Done():
return
case <-t.C:
if beforeTick != nil {
beforeTick()
}
beat()
case <-kick:
beat()
}
}
}
+66
View File
@@ -0,0 +1,66 @@
//go:build !windows
// Tagged !windows like anchor_test.go, whose box helpers (setupBox, driveOp,
// signTerms) it uses; beatnow_test.go runs the signal and the loop everywhere.
package main
import (
"crypto/ed25519"
"os"
"testing"
"time"
)
// optest v0.16.145: an approval left every Host on the machine reading
// "pending" until the next 60-second beat. A grant that flips a pairing active,
// and a revoke, ask every pairing to beat now; a refused grant, and a re-sync
// of a pairing that is already active, change no state and ask nothing.
func TestPairingStateChangeBeatsEveryHost(t *testing.T) {
priv := setupBox(t)
writePairingState(t, tMember, pairingPending)
hn, _ := os.Hostname()
future := time.Now().Add(time.Hour).UTC().Format(time.RFC3339)
kick, stop := subscribeBeatNow()
defer stop()
_, otherPriv, _ := ed25519.GenerateKey(nil)
terms, sig := signTerms(t, otherPriv, grantTerms(tMember, []string{"observe"}, future))
if _, _, code := driveOp(t, tAnchor, "pairing_grant_sync", []string{terms, sig}); code != 126 {
t.Fatalf("a grant under another key must be refused (126): %d", code)
}
if signalled(kick, 100*time.Millisecond) {
t.Fatal("a refused grant asked for a beat")
}
terms, sig = signTerms(t, priv, grantTerms(tMember, []string{"observe"}, future))
if _, errOut, code := driveOp(t, tAnchor, "pairing_grant_sync", []string{terms, sig}); code != 0 {
t.Fatalf("a good grant must be accepted: exit %d, %s", code, errOut)
}
if loadState(t, tMember) != pairingActive {
t.Fatal("grant_sync must flip the pairing active")
}
if !signalled(kick, time.Second) {
t.Fatal("a grant that flipped a pairing active did not ask every pairing to beat now")
}
terms, sig = signTerms(t, priv, grantTerms(tMember, []string{"observe", "dcn"}, future))
if _, errOut, code := driveOp(t, tAnchor, "pairing_grant_sync", []string{terms, sig}); code != 0 {
t.Fatalf("a re-sync of an active pairing must be accepted: exit %d, %s", code, errOut)
}
if signalled(kick, 100*time.Millisecond) {
t.Fatal("a re-sync that changed no pairing state asked for a beat")
}
terms, sig = signTerms(t, priv, map[string]any{"anchor": tAnchor, "issued_at": time.Now().UTC().Format(time.RFC3339),
"kind": "pairing_revoke", "server": hn, "subject": tMember})
if _, errOut, code := driveOp(t, tAnchor, "pairing_grant_drop", []string{tMember, terms, sig}); code != 0 {
t.Fatalf("a good revoke must be accepted: %d %s", code, errOut)
}
if loadState(t, tMember) != pairingRevoked {
t.Fatal("grant_drop must mark the pairing revoked")
}
if !signalled(kick, time.Second) {
t.Fatal("a revoke did not ask every pairing to beat now")
}
}
+79
View File
@@ -0,0 +1,79 @@
package main
import (
"context"
"sync/atomic"
"testing"
"time"
)
// signalled reports whether ch carries a signal within d.
func signalled(ch <-chan struct{}, d time.Duration) bool {
select {
case <-ch:
return true
case <-time.After(d):
return false
}
}
// One signal reaches every registered loop, a burst is one pending signal per
// loop, and a loop that stopped is not signalled.
func TestKickBeatsReachesEveryLoopOnce(t *testing.T) {
a, stopA := subscribeBeatNow()
defer stopA()
b, stopB := subscribeBeatNow()
gone, stopGone := subscribeBeatNow()
stopGone()
kickBeats()
kickBeats()
kickBeats()
stopB()
for name, ch := range map[string]<-chan struct{}{"a": a, "b": b} {
if !signalled(ch, time.Second) {
t.Fatalf("loop %s was not asked to beat", name)
}
if signalled(ch, 50*time.Millisecond) {
t.Fatalf("loop %s holds more than one pending beat after a burst", name)
}
}
if signalled(gone, 50*time.Millisecond) {
t.Fatal("a loop that unregistered was still signalled")
}
}
// The beat loop beats on a kick without waiting for its tick, and the
// tick-only step (path failover) does not run for it.
func TestRunBeatLoopBeatsOnKick(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
var beats, ticks atomic.Int32
done := make(chan struct{})
go func() {
defer close(done)
runBeatLoop(ctx, time.Hour, func() { ticks.Add(1) }, func() { beats.Add(1) })
}()
// The loop registers inside its goroutine; keep asking until it hears.
deadline := time.Now().Add(3 * time.Second)
for beats.Load() == 0 && time.Now().Before(deadline) {
kickBeats()
time.Sleep(10 * time.Millisecond)
}
if beats.Load() == 0 {
t.Fatal("a kicked beat loop did not beat before its 1-hour tick")
}
if ticks.Load() != 0 {
t.Fatal("a kicked beat ran the tick-only step")
}
cancel()
select {
case <-done:
case <-time.After(2 * time.Second):
t.Fatal("the beat loop did not return when its context ended")
}
beatNow.Lock()
left := len(beatNow.subs)
beatNow.Unlock()
if left != 0 {
t.Fatalf("a returned beat loop stayed registered (%d left)", left)
}
}
+53
View File
@@ -0,0 +1,53 @@
package main
import (
"os"
"path/filepath"
"regexp"
"testing"
)
// The Go toolchain image is pinned by digest in deploy/build.sh, which builds
// releases, and again in each forge's CI, so CI tests with the compiler the
// release is built with. GitHub's workflow checks that the three agree, but
// only GitHub runs that file, and only once a commit is published there. This
// test runs wherever go test runs, so a digest changed in one place and not the
// others fails before the change is merged.
var (
releaseImageRe = regexp.MustCompile(`(?m)^GO_IMAGE="(golang@sha256:[0-9a-f]{64})"$`)
giteaImageRe = regexp.MustCompile(`(?m)^ GO_IMAGE: (\S+)$`)
githubImageRe = regexp.MustCompile(`(?m)^ image: (\S+)$`)
)
func TestCIUsesTheReleaseGoImage(t *testing.T) {
read := func(rel string) string {
data, err := os.ReadFile(filepath.Join("..", rel))
if err != nil {
t.Fatal(err)
}
return string(data)
}
release := releaseImageRe.FindAllStringSubmatch(read("deploy/build.sh"), -1)
if len(release) != 1 {
t.Fatalf(`deploy/build.sh: want one GO_IMAGE="golang@sha256:<64 hex>" line, found %d`, len(release))
}
want := release[0][1]
for _, c := range []struct {
rel string
re *regexp.Regexp
n int
}{
{".gitea/workflows/ci.yml", giteaImageRe, 1},
{".github/workflows/ci.yml", githubImageRe, 3},
} {
got := c.re.FindAllStringSubmatch(read(c.rel), -1)
if len(got) != c.n {
t.Errorf("%s: want %d image line(s), found %d", c.rel, c.n, len(got))
}
for _, g := range got {
if g[1] != want {
t.Errorf("%s names %s; deploy/build.sh builds releases with %s", c.rel, g[1], want)
}
}
}
}
+64
View File
@@ -90,6 +90,70 @@ func TestATunnelAddressThatIsNotOneHostIsRefused(t *testing.T) {
} }
} }
// Tunnel endpoints as the host may send them. The first is the hostile
// shape: the configuration text is one setting per line, so a newline ends
// the endpoint and starts a setting of the sender's choosing.
var (
badTunnelEndpoints = []string{
"203.0.113.7:51820\nallowed_ip=::/0", "[fe80::1%x\nallowed_ip=::/0]:51820",
"203.0.113.7:51820\r", "203.0.113.7:51820\t", " 203.0.113.7:51820", "203.0.113.7:51820 ",
"relay\x00.example.net:51820", "relay a.example.net:51820", "", "203.0.113.7",
"203.0.113.7:0", "203.0.113.7:65536", "203.0.113.7:+80", "relay.example.net:port",
"-relay.example.net:51820", "relay..example.net:51820", "[relay.example.net]:51820",
"[fe80::1%eth0]:51820", "0.0.0.0:51820", "[::]:51820", "2001:db8::7:51820",
}
goodTunnelEndpoints = []string{"203.0.113.7:51820", "[2001:db8::7]:51820", "relay-a.example.net:51820"}
)
// An endpoint that is not one host:port is refused at every place the peer
// is configured, and nothing is set on the device.
func TestATunnelEndpointThatIsNotHostPortIsRefused(t *testing.T) {
for _, bad := range badTunnelEndpoints {
cfg := &Config{WGPrivateKey: testWGKey(t), EndpointPubKey: testWGKey(t),
Endpoint: bad, HostTunnelIP: "2001:db8::e:3"}
dev := testWGDevice(t, "2001:db8::a:1")
if err := configureTunnelPeer(dev, cfg); err == nil {
t.Errorf("%q: configureTunnelPeer must refuse", bad)
}
if err := swapPeer(cfg, dev, bad, testWGKey(t)); err == nil {
t.Errorf("%q: swapPeer must refuse", bad)
}
if peers := wgPeers(t, dev); len(peers) != 0 {
t.Errorf("%q: a refused endpoint must leave the device with no peer, it holds %v", bad, peers)
}
}
}
// uapiRecorder keeps the configuration text it is given, in place of a device.
type uapiRecorder struct{ texts []string }
func (r *uapiRecorder) IpcSet(s string) error { r.texts = append(r.texts, s); return nil }
// Whatever endpoint the pairing holds, the text the device is given names
// one peer with exactly one allowed address, the host's alias, or the
// pairing is refused before any text is built.
func TestThePeerConfigurationHasExactlyOneAllowedAddress(t *testing.T) {
for _, ep := range append(append([]string{}, goodTunnelEndpoints...), badTunnelEndpoints...) {
rec := &uapiRecorder{}
cfg := &Config{WGPrivateKey: testWGKey(t), EndpointPubKey: testWGKey(t),
Endpoint: ep, HostTunnelIP: "2001:db8::e:3"}
if err := configureTunnelPeer(rec, cfg); err != nil {
continue
}
for _, text := range rec.texts {
seen := map[string]int{}
for _, line := range strings.Split(strings.TrimSuffix(text, "\n"), "\n") {
k, _, _ := strings.Cut(line, "=")
seen[k]++
}
if seen["allowed_ip"] != 1 || seen["public_key"] != 1 || seen["endpoint"] != 1 ||
!strings.Contains(text, "\nallowed_ip=2001:db8::e:3/128\n") {
t.Errorf("%q: the configuration must name one peer with one allowed address, got %q", ep, text)
}
}
}
}
func TestTheTunnelReachesOnlyThePairedHost(t *testing.T) { func TestTheTunnelReachesOnlyThePairedHost(t *testing.T) {
for _, hostIP := range []string{"2001:db8::e:3", "2001:db8::e:ffff", "2001:db8::1"} { for _, hostIP := range []string{"2001:db8::e:3", "2001:db8::e:ffff", "2001:db8::1"} {
cfg := &Config{ cfg := &Config{
+70 -79
View File
@@ -10,8 +10,8 @@ package main
// over the machine's own connection. // over the machine's own connection.
// //
// THE VETTING APPLIES ALWAYS, anchored box or not. The egress lane // THE VETTING APPLIES ALWAYS, anchored box or not. The egress lane
// (egress.go) only vets on an anchored box, because on a local box the host // (egress.go) keeps a local box's LAN reachable, because on a local box the
// using it is the person's own. A coding agent is not the person: it runs // host using it is the person's own. A coding agent is not the person: it runs
// whatever a prompt or a repository talks it into, so this proxy reaches the // whatever a prompt or a repository talks it into, so this proxy reaches the
// public internet and nothing else — not this machine, not its LAN, not the // public internet and nothing else — not this machine, not its LAN, not the
// cloud metadata address, nor any address this machine takes as its own (a // cloud metadata address, nor any address this machine takes as its own (a
@@ -23,7 +23,7 @@ package main
import ( import (
"bufio" "bufio"
"context" "context"
"errors" "io"
"net" "net"
"os" "os"
"path/filepath" "path/filepath"
@@ -50,35 +50,12 @@ var (
coderDialOut egressDialer = (&net.Dialer{Timeout: egressDialTimeout}).DialContext coderDialOut egressDialer = (&net.Dialer{Timeout: egressDialTimeout}).DialContext
) )
// coderInternalIP is internalIP (egress.go) plus the IPv6 forms that carry an // coderInternalIP is internalIP (egress.go): a coder run's proxy and the
// IPv4 address and are not covered there: IPv4-compatible ::a.b.c.d (::/96), // backup lanes refuse the same address classes as the egress lane, from the
// IPv4-translated ::ffff:0:a.b.c.d, and the deprecated site-local fec0::/10. // one classifier (classifyIP), the IPv4-compatible, IPv4-translated and
// IPv4-mapped ::ffff:a.b.c.d is covered by internalIP through To4. // site-local fec0::/10 forms included.
func coderInternalIP(ip net.IP) bool { func coderInternalIP(ip net.IP) bool {
if internalIP(ip) { return internalIP(ip)
return true
}
if ip.To4() != nil || len(ip) != net.IPv6len {
return false
}
zero := true
for _, b := range ip[:12] {
if b != 0 {
zero = false
break
}
}
if zero {
return true // ::/96 — never a public host
}
if ip[0] == 0 && ip[1] == 0 && ip[2] == 0 && ip[3] == 0 && ip[4] == 0 && ip[5] == 0 &&
ip[6] == 0 && ip[7] == 0 && ip[8] == 0xff && ip[9] == 0xff && ip[10] == 0 && ip[11] == 0 {
return true // ::ffff:0:0/96
}
if ip[0] == 0xfe && ip[1]&0xc0 == 0xc0 {
return true // fec0::/10
}
return false
} }
// coderLANAllowed: the owner's allow_lan names this requested host (an entry // coderLANAllowed: the owner's allow_lan names this requested host (an entry
@@ -174,17 +151,32 @@ func coderVettedAddrs(ctx context.Context, hostport string, allowLAN []string) (
} }
func coderRefuse(c net.Conn, why string) { func coderRefuse(c net.Conn, why string) {
_, _ = c.Write([]byte("HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\nConnection: close\r\nX-Privatae-Refused: " + writeEgressRefusal(c, "403 Forbidden", why, why)
strings.NewReplacer("\r", " ", "\n", " ").Replace(why) + "\r\n\r\n"))
} }
// handleCoderEgressConn serves one connection from a run: HTTP CONNECT or an // handleCoderEgressConn serves one connection from a run: HTTP CONNECT or an
// absolute-form plain HTTP request, the same parsing as handleEgressConn. // absolute-form plain HTTP request, the same parsing as handleEgressConn.
// Destinations are never logged; refusals are counted on the run. // Destinations are never logged; refusals are counted on the run.
// coderEgressMaxHeadBytes bounds a request's line and headers together.
const coderEgressMaxHeadBytes = 64 << 10
// coderEgressMaxConns bounds the connections one run's proxy carries at once.
// A whole Chromium profile opens at most 256 sockets at a time (its socket
// pool limit), and package managers far fewer, so four times the browser's
// figure never refuses real work; past it a connection is answered 503 and
// closed, and each one held costs this machine at most its head and two copy
// buffers.
const coderEgressMaxConns = 1024
func handleCoderEgressConn(client net.Conn, allowLAN []string, refused *int64) { func handleCoderEgressConn(client net.Conn, allowLAN []string, refused *int64) {
defer client.Close() defer client.Close()
_ = client.SetReadDeadline(time.Now().Add(30 * time.Second)) _ = client.SetReadDeadline(time.Now().Add(30 * time.Second))
br := bufio.NewReader(client) // The request line and headers are read under one bound, the head limit
// below has always named: the run's own processes send them, and a line
// that never ends must not grow in this machine's memory before that
// limit is looked at.
headLim := &io.LimitedReader{R: client, N: coderEgressMaxHeadBytes}
br := bufio.NewReader(headLim)
line, err := br.ReadString('\n') line, err := br.ReadString('\n')
if err != nil { if err != nil {
return return
@@ -196,7 +188,6 @@ func handleCoderEgressConn(client net.Conn, allowLAN []string, refused *int64) {
method, target := parts[0], parts[1] method, target := parts[0], parts[1]
var head strings.Builder var head strings.Builder
head.WriteString(line) head.WriteString(line)
hostHdr := ""
for { for {
h, err := br.ReadString('\n') h, err := br.ReadString('\n')
if err != nil { if err != nil {
@@ -206,38 +197,41 @@ func handleCoderEgressConn(client net.Conn, allowLAN []string, refused *int64) {
if h == "\r\n" || h == "\n" { if h == "\r\n" || h == "\n" {
break break
} }
if head.Len() > 64<<10 { if head.Len() > coderEgressMaxHeadBytes {
return return
} }
if len(h) >= 5 && strings.EqualFold(h[:5], "host:") {
hostHdr = strings.TrimSpace(h[5:])
}
} }
_ = client.SetReadDeadline(time.Time{}) _ = client.SetReadDeadline(time.Time{})
// The head is in; what follows is the stream itself. What the reader
// already buffered past the head stays first in line.
headLim.N = 1<<63 - 1
ctx, cancel := context.WithTimeout(context.Background(), egressDialTimeout) ctx, cancel := context.WithTimeout(context.Background(), egressDialTimeout)
defer cancel() defer cancel()
// Read the way the bastion's own egress lane reads it (egress_http.go): a
// CONNECT target through parseEgressAuthority, and plain HTTP as ONE
// request, rebuilt and sent with `Connection: close`, so the run's later
// requests on a keep-alive connection never reach the destination unvetted.
connect := strings.EqualFold(method, "CONNECT") connect := strings.EqualFold(method, "CONNECT")
hp := "" hp := ""
var req egressRequest
if connect { if connect {
hp = ensurePort(target, "443") host, port, ok := parseEgressAuthority(target, 443)
} else { if !ok {
host := hostHdr atomic.AddInt64(refused, 1)
if i := strings.Index(target, "://"); i >= 0 { writeEgressRefusal(client, "400 Bad Request", egressNoHost, egressNoHost)
rest := target[i+3:]
if s := strings.IndexAny(rest, "/?#"); s >= 0 {
host = rest[:s]
} else {
host = rest
}
if at := strings.LastIndexByte(host, '@'); at >= 0 {
host = host[at+1:]
}
}
if host == "" {
return return
} }
hp = ensurePort(host, "80") hp = egressHostPort(host, port)
} else {
r, bad := parseEgressRequest(head.String())
if bad != nil {
atomic.AddInt64(refused, 1)
writeEgressRefusal(client, bad.status, bad.body, bad.body)
return
}
req = r
hp = egressHostPort(req.host, req.port)
} }
dialAddrs, why := coderVettedAddrs(ctx, hp, allowLAN) dialAddrs, why := coderVettedAddrs(ctx, hp, allowLAN)
if why != "" { if why != "" {
@@ -245,36 +239,19 @@ func handleCoderEgressConn(client net.Conn, allowLAN []string, refused *int64) {
coderRefuse(client, why) coderRefuse(client, why)
return return
} }
var upstream net.Conn upstream, err := dialEach(ctx, coderDialOut, dialAddrs)
err = errors.New("no address")
for i, a := range dialAddrs {
// Each address gets an even share of what is left of the dial budget,
// so one that black-holes cannot use it all.
actx, acancel := ctx, context.CancelFunc(func() {})
if left := len(dialAddrs) - i; left > 1 {
if dl, ok := ctx.Deadline(); ok {
actx, acancel = context.WithTimeout(ctx, time.Until(dl)/time.Duration(left))
}
}
upstream, err = coderDialOut(actx, "tcp", a)
acancel()
if err == nil {
break
}
}
if err != nil { if err != nil {
_, _ = client.Write([]byte("HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")) writeEgressRefusal(client, "502 Bad Gateway", egressUnreachable, "")
return
}
if !connect {
oneEgressExchange(br, client, upstream, req)
return return
} }
if connect {
if _, err := client.Write([]byte("HTTP/1.1 200 Connection established\r\n\r\n")); err != nil { if _, err := client.Write([]byte("HTTP/1.1 200 Connection established\r\n\r\n")); err != nil {
_ = upstream.Close() _ = upstream.Close()
return return
} }
} else if _, err := upstream.Write([]byte(head.String())); err != nil {
_ = upstream.Close()
return
}
splice(br, client, upstream) splice(br, client, upstream)
} }
@@ -317,6 +294,7 @@ func coderListenEgress(runID, dir string, uid int) error {
} }
run := &coderEgressRun{ln: ln} run := &coderEgressRun{ln: ln}
coderEgress[runID] = run coderEgress[runID] = run
slots := make(chan struct{}, coderEgressMaxConns)
go func() { go func() {
for { for {
c, err := ln.Accept() c, err := ln.Accept()
@@ -325,7 +303,20 @@ func coderListenEgress(runID, dir string, uid int) error {
} }
// allow_lan is read per connection: the owner's edit applies to // allow_lan is read per connection: the owner's edit applies to
// running runs at once. // running runs at once.
go handleCoderEgressConn(c, loadCoderConfig().AllowLAN, &run.refused) select {
case slots <- struct{}{}:
go func() {
defer func() { <-slots }()
handleCoderEgressConn(c, loadCoderConfig().AllowLAN, &run.refused)
}()
default:
go func() {
defer c.Close()
_ = c.SetWriteDeadline(time.Now().Add(time.Second))
why := "this run already has " + strconv.Itoa(coderEgressMaxConns) + " connections open"
writeEgressRefusal(c, "503 Service Unavailable", why, why)
}()
}
} }
}() }()
return nil return nil
+8 -2
View File
@@ -474,8 +474,14 @@ func TestCoderEgressProxyOverTheSocket(t *testing.T) {
} { } {
head, c := ask(req) head, c := ask(req)
c.Close() c.Close()
if !strings.HasPrefix(head, "HTTP/1.1 403") || !strings.Contains(head, "X-Privatae-Refused:") { want := "HTTP/1.1 403"
t.Fatalf("must be refused with 403 + X-Privatae-Refused: %q -> %q", req, head) if strings.Contains(req, "user@") {
// Userinfo is refused outright, before any lookup, as the Host's
// egress gate refuses it (parseEgressAuthority); still counted.
want = "HTTP/1.1 400"
}
if !strings.HasPrefix(head, want) || !strings.Contains(head, "X-Privatae-Refused:") {
t.Fatalf("must be refused with %s + X-Privatae-Refused: %q -> %q", want, req, head)
} }
} }
head, c := ask("CONNECT public.example:443 HTTP/1.1\r\n\r\n") head, c := ask("CONNECT public.example:443 HTTP/1.1\r\n\r\n")
+1 -1
View File
@@ -47,7 +47,7 @@ func TestBoundTreeRecountsAndCaps(t *testing.T) {
t.Fatalf("full tree: count=%d trunc=%v %+v", count, trunc, out) t.Fatalf("full tree: count=%d trunc=%v %+v", count, trunc, out)
} }
// the cap holds on THIS side even if a helper lied about truncation // the cap holds on THIS side even if a helper lied about truncation
out, count, trunc = boundTree([]*computerNode{win}, 3) _, count, trunc = boundTree([]*computerNode{win}, 3)
if count != 3 || !trunc { if count != 3 || !trunc {
t.Fatalf("capped: count=%d trunc=%v", count, trunc) t.Fatalf("capped: count=%d trunc=%v", count, trunc)
} }
+37 -22
View File
@@ -76,12 +76,20 @@ type Config struct {
// that its file does not hold yet (rotate.go, setUnsaved; guarded by // that its file does not hold yet (rotate.go, setUnsaved; guarded by
// rotation.mu). Unexported: never in the file. // rotation.mu). Unexported: never in the file.
savePending bool savePending bool
// invalid is why loadPairings found this pairing's file unusable (an
// endpoint that is not one host:port), naming the file; "" for a usable
// one. Such a pairing is listed, so status shows it and unpair removes it,
// but it never serves: active() is false. Unexported: never in the file.
invalid string
} }
const ( const (
pairingActive = "active" pairingActive = "active"
pairingPending = "pending" pairingPending = "pending"
pairingRevoked = "revoked" pairingRevoked = "revoked"
// pairingInvalid is reported, never stored: the file holds a value this
// build refuses (Config.invalid).
pairingInvalid = "invalid"
// transportUnix is Config.Transport for a Host on this machine. // transportUnix is Config.Transport for a Host on this machine.
transportUnix = "unix" transportUnix = "unix"
@@ -103,11 +111,15 @@ func (c *Config) localTransport() bool { return c.unixTransport() || c.tcpTransp
// active is the ONE place the state field is interpreted: "" and "active" serve, // active is the ONE place the state field is interpreted: "" and "active" serve,
// anything else does not. run.go, the heartbeat and the mgmt lane all ask this. // anything else does not. run.go, the heartbeat and the mgmt lane all ask this.
func (c *Config) active() bool { func (c *Config) active() bool {
return c.State == "" || c.State == pairingActive return c.invalid == "" && (c.State == "" || c.State == pairingActive)
} }
// stateName is the state as reported: the empty legacy value reads "active". // stateName is the state as reported: the empty legacy value reads "active",
// and a pairing whose file is unusable reads "invalid" whatever its state.
func (c *Config) stateName() string { func (c *Config) stateName() string {
if c.invalid != "" {
return pairingInvalid
}
if c.State == "" { if c.State == "" {
return pairingActive return pairingActive
} }
@@ -316,17 +328,36 @@ func (c *Config) updateOnDiskIf(apply func(disk *Config) error) error {
// updatePairingOnDisk is updateOnDisk for a caller that only has the host // updatePairingOnDisk is updateOnDisk for a caller that only has the host
// name (the mgmt handlers). Returns the pairing as written, or nil with // name (the mgmt handlers). Returns the pairing as written, or nil with
// errPairingGone when no such file exists. // errPairingGone when no such file exists.
//
// The copy is checked as loadPairings checks a file, so a caller that starts
// the pairing from it never serves one whose file this build refuses.
func updatePairingOnDisk(host string, apply func(disk *Config)) (*Config, error) { func updatePairingOnDisk(host string, apply func(disk *Config)) (*Config, error) {
pd, err := pairingsDir()
if err != nil {
return nil, err
}
c := &Config{HostName: host} c := &Config{HostName: host}
var out *Config var out *Config
err := c.updateOnDisk(func(d *Config) { err = c.updateOnDisk(func(d *Config) {
apply(d) apply(d)
cp := *d cp := *d
cp.invalid = invalidReason(&cp, filepath.Join(pd, host+".json"))
out = &cp out = &cp
}) })
return out, err return out, err
} }
// invalidReason is why the pairing c, read from file, is never served, naming
// the file, or "" when it may be served. It is the check the pairing response
// and the rotate directive get, for a file an older build saved from either.
func invalidReason(c *Config, file string) string {
if err := checkEndpoints(append([]string{c.Endpoint}, c.Endpoints...)...); err != nil {
return fmt.Sprintf("pairing %s: %v — it is not served; pair it again (`privatae-bastion pair --replace <token>`) or remove it (`privatae-bastion unpair %s`)",
file, err, c.HostName)
}
return ""
}
// unpairHost tears down and deletes one pairing: its file and its grant file. // unpairHost tears down and deletes one pairing: its file and its grant file.
// The context store and DCN dir are the USER's data and are left in place — // The context store and DCN dir are the USER's data and are left in place —
// deleting a pairing must not delete what the user put on this machine. // deleting a pairing must not delete what the user put on this machine.
@@ -495,31 +526,15 @@ func loadPairings() ([]*Config, error) {
if err := json.Unmarshal(raw, &c); err != nil { if err := json.Unmarshal(raw, &c); err != nil {
return nil, fmt.Errorf("pairing %s: %w", e.Name(), err) return nil, fmt.Errorf("pairing %s: %w", e.Name(), err)
} }
// One such file must not stop every other pairing, status or unpair:
// it loads, never serves.
c.invalid = invalidReason(&c, filepath.Join(pd, e.Name()))
cfgs = append(cfgs, &c) cfgs = append(cfgs, &c)
} }
sort.Slice(cfgs, func(i, j int) bool { return cfgs[i].HostName < cfgs[j].HostName }) sort.Slice(cfgs, func(i, j int) bool { return cfgs[i].HostName < cfgs[j].HostName })
return cfgs, nil return cfgs, nil
} }
// loadConfig keeps single-pairing call sites working: with exactly one pairing it
// returns it. With several it REFUSES — a straggler caller must name a host and
// fail safe, not silently act on an arbitrary one (that ambiguity is the same
// family of bug as the single-config unpair).
func loadConfig() (*Config, error) {
cfgs, err := loadPairings()
if err != nil {
return nil, err
}
switch len(cfgs) {
case 0:
return nil, fmt.Errorf("no pairings")
case 1:
return cfgs[0], nil
default:
return nil, fmt.Errorf("%d hosts are paired — this operation must name one (see --host)", len(cfgs))
}
}
// hostNames lists the paired host names (for error messages). // hostNames lists the paired host names (for error messages).
func hostNames(cfgs []*Config) []string { func hostNames(cfgs []*Config) []string {
names := make([]string, 0, len(cfgs)) names := make([]string, 0, len(cfgs))
+1 -1
View File
@@ -135,7 +135,7 @@ func maybeContain(pairing, dcnDir string, heartbeatResponse []byte) {
// already passed the agent checks above, and an agent whose own // already passed the agent checks above, and an agent whose own
// fingerprint is in the accounting (one running as a service, say) // fingerprint is in the accounting (one running as a service, say)
// would otherwise be spared in silence — killTree returns 0 for a // would otherwise be spared in silence — killTree returns 0 for a
// spared root and a zero kill is not recorded (review of #100). // spared root and a zero kill is not recorded.
root := pid root := pid
killed := killTree(pid, func(p int) bool { killed := killTree(pid, func(p int) bool {
if p == root { if p == root {
+6 -6
View File
@@ -104,7 +104,7 @@ func recordContainResult(pairing string, o findingOrder, outcome string, stopped
// "Carried out once" held only while this process lived: a bastion that // "Carried out once" held only while this process lived: a bastion that
// restarted after a kill and before the beat carried the answer (a self-update, // restarted after a kill and before the beat carried the answer (a self-update,
// a crash) would receive the order again, run it again, and report // a crash) would receive the order again, run it again, and report
// not_a_finding — or kill a restarted copy (review of #100). So every result // not_a_finding — or kill a restarted copy. So every result
// is written to the config dir as it is recorded and read back on first use. // is written to the config dir as it is recorded and read back on first use.
var containResultsLoaded bool var containResultsLoaded bool
@@ -165,7 +165,7 @@ func containResultsSaveLocked() {
} }
if b, err := json.Marshal(disk); err == nil { if b, err := json.Marshal(disk); err == nil {
// Atomic: a crash mid-write must not corrupt the record and make the // Atomic: a crash mid-write must not corrupt the record and make the
// box forget answers it gave (review of #100). // box forget answers it gave.
_ = atomicWrite(p, b) _ = atomicWrite(p, b)
} }
} }
@@ -231,7 +231,7 @@ type findingView struct {
protected map[int]bool protected map[int]bool
// identity: each finding pid's identity read straight after the walk. The // identity: each finding pid's identity read straight after the walk. The
// kill checks it before touching the root, so a pid reused between the walk // kill checks it before touching the root, so a pid reused between the walk
// and the kill is never signalled (review of #100). // and the kill is never signalled.
identity map[int]string identity map[int]string
} }
@@ -441,9 +441,9 @@ func maybeContainFindings(pairing, dcnDir string, heartbeatResponse []byte) {
// process in the tree, not only the ones an order names — an accounting entry // process in the tree, not only the ones an order names — an accounting entry
// (accountingPIDs), an app the user is running, anything this bastion started, // (accountingPIDs), an app the user is running, anything this bastion started,
// init, the bastion. killTree does not signal a protected process or walk // init, the bastion. killTree does not signal a protected process or walk
// beneath it (review of #100: stopping a replaced sshd would otherwise have // beneath it: without that, stopping a replaced sshd would take every ssh
// taken every ssh session with it). It also returns the bastion's ANCESTORS — // session with it. It also returns the bastion's ANCESTORS — its launch
// its launch chain — which the finding path refuses by name: if any process in // chain — which the finding path refuses by name: if any process in
// a tree were an ancestor of the bastion, the tree's root would be one too, so // a tree were an ancestor of the bastion, the tree's root would be one too, so
// the named check always fires first and a tree-level check could never be // the named check always fires first and a tree-level check could never be
// reached. // reached.
+11 -11
View File
@@ -290,7 +290,7 @@ func TestProcessFindingsFromTheRealComparison(t *testing.T) {
} }
} }
// THE GUARDS HOLD ACROSS THE WHOLE TREE (review of #100, blocking). A finding // THE GUARDS HOLD ACROSS THE WHOLE TREE. A finding
// whose tree holds an accounting entry, an app, or something the bastion // whose tree holds an accounting entry, an app, or something the bastion
// started: those processes and everything beneath them are spared, the rest // started: those processes and everything beneath them are spared, the rest
// of the tree is stopped. // of the tree is stopped.
@@ -334,8 +334,8 @@ func TestContainFindingRefusesTheBastionsOwnAncestors(t *testing.T) {
} }
// THE REFUSALS liveProcessFindings MAKES, through the real function with only // THE REFUSALS liveProcessFindings MAKES, through the real function with only
// the machine replaced (review of #100: every other test replaced the whole // the machine replaced. Every other test replaces the whole function, so
// function, so none of these was tested). // this is the one that tests these refusals.
// //
// MUTATION: drop the root check. RESULT: red (no_authority not returned). // MUTATION: drop the root check. RESULT: red (no_authority not returned).
// MUTATION: drop !digestOK. RESULT: red (an edited accounting is used). // MUTATION: drop !digestOK. RESULT: red (an edited accounting is used).
@@ -375,7 +375,7 @@ func TestLiveProcessFindingsRefusesWhatItCannotTrust(t *testing.T) {
} }
} }
// RESULTS SURVIVE A RESTART (review of #100): an answer recorded before the // RESULTS SURVIVE A RESTART: an answer recorded before the
// process died is still there — and still stops a re-run — after it starts. // process died is still there — and still stops a re-run — after it starts.
// //
// MUTATION: skip containResultsSaveLocked. RESULT: red. // MUTATION: skip containResultsSaveLocked. RESULT: red.
@@ -413,7 +413,7 @@ func TestContainFindingRunsAnOrderOnce(t *testing.T) {
} }
} }
// THE REAL FILTER IS WIRED IN (review of #100): with containFilter left at its // THE REAL FILTER IS WIRED IN: with containFilter left at its
// default, a new process the bastion itself started is filtered by the same // default, a new process the bastion itself started is filtered by the same
// deltaFilter the pass uses, and so is never containable; the same process // deltaFilter the pass uses, and so is never containable; the same process
// started by anyone else is. // started by anyone else is.
@@ -422,7 +422,7 @@ func TestContainFindingRunsAnOrderOnce(t *testing.T) {
func TestContainFilterIsTheDeltaPassFilter(t *testing.T) { func TestContainFilterIsTheDeltaPassFilter(t *testing.T) {
// The process's AGE is supplied (an hour), not read from a real process: // The process's AGE is supplied (an hour), not read from a real process:
// a real pid's age depends on where the suite runs — pid 1 in a fresh PID // a real pid's age depends on where the suite runs — pid 1 in a fresh PID
// namespace is seconds old and reads as short-lived (review of #100). // namespace is seconds old and reads as short-lived.
self := os.Getpid() self := os.Getpid()
prevTree, prevAge := processTree, deltaAgeFn prevTree, prevAge := processTree, deltaAgeFn
t.Cleanup(func() { processTree, deltaAgeFn = prevTree, prevAge }) t.Cleanup(func() { processTree, deltaAgeFn = prevTree, prevAge })
@@ -446,7 +446,7 @@ func TestContainFilterIsTheDeltaPassFilter(t *testing.T) {
} }
} }
// THE ROOT'S IDENTITY IS CARRIED FROM THE WALK (review of #100): a pid that // THE ROOT'S IDENTITY IS CARRIED FROM THE WALK: a pid that
// changed hands between the finding walk and the kill is not signalled at all. // changed hands between the finding walk and the kill is not signalled at all.
// //
// MUTATION: killTree(pid, protected) instead of killTreeExpect with the view's // MUTATION: killTree(pid, protected) instead of killTreeExpect with the view's
@@ -471,7 +471,7 @@ func TestContainFindingDoesNotKillARootThatChangedHandsAfterTheWalk(t *testing.T
} }
} }
// THE AGENT PATH HAS THE SAME TREE GUARD (review of #100): an accounting entry // THE AGENT PATH HAS THE SAME TREE GUARD: an accounting entry
// or an app inside an agent's tree is spared with its subtree. // or an app inside an agent's tree is spared with its subtree.
// //
// MUTATION: killTree(pid, spareFromContain) in maybeContain. RESULT: red. // MUTATION: killTree(pid, spareFromContain) in maybeContain. RESULT: red.
@@ -497,9 +497,9 @@ func TestAgentContainSparesProtectedProcessesInTheTree(t *testing.T) {
} }
} }
// AN AGENT WHOSE OWN FINGERPRINT IS IN THE ACCOUNTING IS STILL STOPPED (review // AN AGENT WHOSE OWN FINGERPRINT IS IN THE ACCOUNTING IS STILL STOPPED: the
// of #100, round 3): the tree guard applies BENEATH the agent, never to the // tree guard applies BENEATH the agent, never to the agent the host named.
// agent the host named. Before this, such an agent was spared in silence. // Before this, such an agent was spared in silence.
// //
// MUTATION: killTree(pid, guard) — the guard on the root too. RESULT: red. // MUTATION: killTree(pid, guard) — the guard on the root too. RESULT: red.
func TestAgentContainStopsAnAgentThatIsItselfInTheAccounting(t *testing.T) { func TestAgentContainStopsAnAgentThatIsItselfInTheAccounting(t *testing.T) {
+79 -7
View File
@@ -43,9 +43,11 @@ import (
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
"io"
"net" "net"
"os" "os"
"path/filepath" "path/filepath"
"sync"
"time" "time"
) )
@@ -126,16 +128,79 @@ func startContextSock(done <-chan struct{}) {
_ = ln.Close() _ = ln.Close()
_ = os.Remove(p) _ = os.Remove(p)
}() }()
go func() { go contextSockAccept(ln)
uilogf("context socket listening on %s\n", p)
}
// The socket is 0666 and the daemon runs as root (or SYSTEM), so every user on
// the machine can make it read. These bound what they can make it hold: an ask
// is a folder path and a display name, far under contextAskMaxBytes, and no
// more than contextSockMaxInFlight asks are read at once. Past that a
// connection is told the daemon is busy and closed; the asker tries again.
//
// One person may hold at most contextSockMaxPerUser of those slots, keyed on
// the identity the kernel gives (never on anything the asker says): without
// it, one user opening contextSockMaxInFlight idle connections and dialling
// again as each deadline ran out kept every other user's asks "busy" for as
// long as they liked. A person adds a folder at a time; four at once is far
// past that.
const (
contextAskMaxBytes = 64 << 10
contextSockMaxInFlight = 32
)
// contextSockMaxPerUser is a var only so the test of the overall cap can lift
// it (every test connection is the same user).
var contextSockMaxPerUser = 4
var contextSockUsers = struct {
sync.Mutex
held map[string]int
}{held: map[string]int{}}
// contextSockTakeUser takes one of user's slots, or reports there is none.
func contextSockTakeUser(user string) bool {
contextSockUsers.Lock()
defer contextSockUsers.Unlock()
if contextSockUsers.held[user] >= contextSockMaxPerUser {
return false
}
contextSockUsers.held[user]++
return true
}
func contextSockReleaseUser(user string) {
contextSockUsers.Lock()
defer contextSockUsers.Unlock()
if contextSockUsers.held[user]--; contextSockUsers.held[user] <= 0 {
delete(contextSockUsers.held, user)
}
}
// contextSockAccept serves ln until it is closed, at most
// contextSockMaxInFlight connections at a time.
func contextSockAccept(ln net.Listener) {
slots := make(chan struct{}, contextSockMaxInFlight)
for { for {
c, err := ln.Accept() c, err := ln.Accept()
if err != nil { if err != nil {
return return
} }
go contextSockServe(c) select {
} case slots <- struct{}{}:
go func() {
defer func() { <-slots }()
contextSockServe(c)
}() }()
uilogf("context socket listening on %s\n", p) default:
go func() {
defer c.Close()
_ = c.SetDeadline(time.Now().Add(time.Second))
_ = json.NewEncoder(c).Encode(contextAskReply{Why: "the bastion is busy " +
"answering other asks on this machine; try again in a moment"})
}()
}
}
} }
func contextSockServe(c net.Conn) { func contextSockServe(c net.Conn) {
@@ -154,8 +219,15 @@ func contextSockServe(c net.Conn) {
return return
} }
if !contextSockTakeUser(user) {
_ = json.NewEncoder(c).Encode(contextAskReply{User: user, Why: "the bastion is busy " +
"answering other asks of yours; try again in a moment"})
return
}
defer contextSockReleaseUser(user)
var req contextAskRequest var req contextAskRequest
if err := json.NewDecoder(c).Decode(&req); err != nil { if err := json.NewDecoder(io.LimitReader(c, contextAskMaxBytes)).Decode(&req); err != nil {
_ = json.NewEncoder(c).Encode(contextAskReply{User: user, Why: "the request could not be read"}) _ = json.NewEncoder(c).Encode(contextAskReply{User: user, Why: "the request could not be read"})
return return
} }
@@ -200,10 +272,10 @@ func contextAskViaSock(root, name string) (sent bool, host, note string, availab
return true, rep.Host, "", true return true, rep.Host, "", true
} }
// contextSockUnsupported is the note used where the platform has no // errContextSockUnsupported is the note used where the platform has no
// implementation yet, so the reason a person sees names the platform rather // implementation yet, so the reason a person sees names the platform rather
// than pretending the feature is merely broken. // than pretending the feature is merely broken.
var contextSockUnsupported = errors.New("peer identity over a local socket is not " + var errContextSockUnsupported = errors.New("peer identity over a local socket is not " +
"implemented for this platform yet") "implemented for this platform yet")
func contextSockFmtUser(uid uint32, name string) string { func contextSockFmtUser(uid uint32, name string) string {
+130
View File
@@ -0,0 +1,130 @@
package main
// The context socket is 0666: every user on the machine may connect, and the
// daemon behind it runs as root (or SYSTEM). So what one connection can make
// the daemon hold must be bounded by the daemon, not by the asker.
import (
"net"
"path/filepath"
"runtime"
"strings"
"testing"
"time"
)
// An ask is a folder path and a display name. A client that streams a request
// with no end gets the daemon to read only a bounded amount of it, then a
// reply and a close; before, the decoder kept every byte until the 30 s
// deadline, and nothing capped how many such connections were open.
func TestAContextAskWithNoEndIsReadOnlyToABound(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("no unix-socket peer credentials on windows; contextsock_windows.go says why")
}
sock := filepath.Join(t.TempDir(), "s.sock")
ln, err := net.Listen("unix", sock)
if err != nil {
t.Fatal(err)
}
defer ln.Close()
go func() {
c, err := ln.Accept()
if err != nil {
return
}
contextSockServe(c)
}()
c, err := net.Dial("unix", sock)
if err != nil {
t.Fatal(err)
}
defer c.Close()
const total = 16 << 20
chunk := []byte(strings.Repeat("a", 64<<10))
sent := 0
if _, err := c.Write([]byte(`{"root":"`)); err != nil {
t.Fatal(err)
}
for sent < total {
_ = c.SetWriteDeadline(time.Now().Add(2 * time.Second))
n, err := c.Write(chunk)
sent += n
if err != nil {
break
}
}
// The kernel's socket buffers hold a few hundred KiB the daemon never
// read; anything near the whole stream means the daemon read it all.
if sent >= 4<<20 {
t.Fatalf("the daemon accepted %d MiB of one ask with no end; it must stop at a bound", sent>>20)
}
}
// Asks in flight are capped: a connection past the cap is closed at once, so
// many idle or endless connections cannot multiply the bound above.
func TestContextAsksInFlightAreCapped(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("no unix-socket peer credentials on windows; contextsock_windows.go says why")
}
sock := filepath.Join(t.TempDir(), "s.sock")
ln, err := net.Listen("unix", sock)
if err != nil {
t.Fatal(err)
}
defer ln.Close()
// Every connection here is the same user; lift the per-user share so the
// overall cap is what this test meets (TestOneUserCannotHoldEveryContextSockSlot
// holds the share).
oldShare := contextSockMaxPerUser
contextSockMaxPerUser = contextSockMaxInFlight
t.Cleanup(func() { contextSockMaxPerUser = oldShare })
go contextSockAccept(ln)
var held []net.Conn
defer func() {
for _, h := range held {
_ = h.Close()
}
}()
// Each of these sends nothing and holds its slot until its deadline.
for i := 0; i < contextSockMaxInFlight; i++ {
h, err := net.Dial("unix", sock)
if err != nil {
t.Fatal(err)
}
held = append(held, h)
}
// Give the daemon's accept loop time to take every held connection.
time.Sleep(200 * time.Millisecond)
extra, err := net.Dial("unix", sock)
if err != nil {
t.Fatal(err)
}
defer extra.Close()
_ = extra.SetReadDeadline(time.Now().Add(2 * time.Second))
buf := make([]byte, 512)
n, rerr := extra.Read(buf)
if ne, ok := rerr.(net.Error); ok && ne.Timeout() {
t.Fatalf("a connection past the cap of %d was left open; it must be refused", contextSockMaxInFlight)
}
if n > 0 && !strings.Contains(string(buf[:n]), "busy") {
t.Fatalf("a connection past the cap got %q; it must be told the daemon is busy", buf[:n])
}
// Once a held connection goes, a new ask gets a slot again.
_ = held[0].Close()
held = held[1:]
time.Sleep(200 * time.Millisecond)
again, err := net.Dial("unix", sock)
if err != nil {
t.Fatal(err)
}
defer again.Close()
_ = again.SetReadDeadline(time.Now().Add(500 * time.Millisecond))
n, rerr = again.Read(buf)
if ne, ok := rerr.(net.Error); !ok || !ne.Timeout() {
t.Fatalf("after a slot was freed, a new ask was not served (read %q, %v)", buf[:n], rerr)
}
}
+1 -1
View File
@@ -7,4 +7,4 @@ package main
// to tell who is on the other end: contextPeerUser then fails, and // to tell who is on the other end: contextPeerUser then fails, and
// contextSockServe answers every connection with that reason and forwards // contextSockServe answers every connection with that reason and forwards
// nothing. An ask that cannot be attributed is refused, never let through. // nothing. An ask that cannot be attributed is refused, never let through.
func contextPeerUID(uintptr) (uint32, error) { return 0, contextSockUnsupported } func contextPeerUID(uintptr) (uint32, error) { return 0, errContextSockUnsupported }
+86
View File
@@ -0,0 +1,86 @@
package main
import (
"net"
"path/filepath"
"runtime"
"strings"
"testing"
"time"
)
// One person cannot take every slot: past their own share they are told the
// daemon is busy at once, while slots remain for everyone else. Before, one
// user's idle connections, re-dialled as each deadline ran out, held all of
// them and every other user's ask was refused.
func TestOneUserCannotHoldEveryContextSockSlot(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("no unix-socket peer credentials on windows; contextsock_windows.go says why")
}
const share = 4 // contextSockMaxPerUser
// A test before this one may still be releasing its slots.
if n := contextSockSlotsHeld(t, 0); n != 0 {
t.Fatalf("%d slots still held from an earlier test", n)
}
sock := filepath.Join(t.TempDir(), "s.sock")
ln, err := net.Listen("unix", sock)
if err != nil {
t.Fatal(err)
}
defer ln.Close()
go contextSockAccept(ln)
var held []net.Conn
defer func() {
for _, h := range held {
_ = h.Close()
}
}()
for i := 0; i < share; i++ {
h, err := net.Dial("unix", sock)
if err != nil {
t.Fatal(err)
}
held = append(held, h)
}
// Wait until the daemon has given those four their slots (a fixed sleep
// lost this race on a loaded machine and handed the 5th one first).
if n := contextSockSlotsHeld(t, share); n != share {
t.Fatalf("the daemon gave %d slots to %d idle connections of one user", n, share)
}
extra, err := net.Dial("unix", sock)
if err != nil {
t.Fatal(err)
}
defer extra.Close()
_ = extra.SetReadDeadline(time.Now().Add(2 * time.Second))
buf := make([]byte, 512)
n, rerr := extra.Read(buf)
if ne, ok := rerr.(net.Error); ok && ne.Timeout() {
t.Fatalf("one user held %d idle connections and a %dth was still given a slot: "+
"one user can take all %d", share, share+1, contextSockMaxInFlight)
}
if !strings.Contains(string(buf[:n]), "busy") {
t.Fatalf("past one user's share the ask got %q; it must be told the daemon is busy", buf[:n])
}
}
// contextSockSlotsHeld waits up to 10 s for the per-user slots held across
// all users to reach want, and returns the count it last saw.
func contextSockSlotsHeld(t *testing.T, want int) int {
t.Helper()
n := -1
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(10 * time.Millisecond) {
contextSockUsers.Lock()
n = 0
for _, k := range contextSockUsers.held {
n += k
}
contextSockUsers.Unlock()
if n == want {
break
}
}
return n
}
+2 -2
View File
@@ -33,11 +33,11 @@ const contextSockDir = `C:\ProgramData\privatae-bastion`
func contextSockDialable(string) string { return "" } func contextSockDialable(string) string { return "" }
func contextSockListen(path string) (net.Listener, error) { func contextSockListen(path string) (net.Listener, error) {
return nil, contextSockUnsupported return nil, errContextSockUnsupported
} }
func contextPeerUser(c net.Conn) (string, error) { func contextPeerUser(c net.Conn) (string, error) {
return "", contextSockUnsupported return "", errContextSockUnsupported
} }
var _ = os.Remove var _ = os.Remove
+1 -4
View File
@@ -859,10 +859,7 @@ func sttIsNonSpeechMark(text string) bool {
// ONE group, not two. "(yes) and (no)" ends in a bracket and is speech; the // ONE group, not two. "(yes) and (no)" ends in a bracket and is speech; the
// opening bracket must be closed only at the very end. // opening bracket must be closed only at the very end.
inner := t[1 : len(t)-1] inner := t[1 : len(t)-1]
if strings.IndexByte(inner, closer) >= 0 { return strings.IndexByte(inner, closer) < 0
return false
}
return true
} }
type sttSegment struct { type sttSegment struct {
+1 -3
View File
@@ -26,9 +26,7 @@ import (
"github.com/gorilla/websocket" "github.com/gorilla/websocket"
) )
// uvicornWSMaxSize is the host's WebSocket frame ceiling: the server default, // uvicornWSMaxSize (harness.go) is the host's WebSocket frame ceiling.
// which nothing on the host overrides, so this is the live value.
const uvicornWSMaxSize = 16 << 20
func TestMeasureCtxPutCapacity(t *testing.T) { func TestMeasureCtxPutCapacity(t *testing.T) {
if os.Getenv("MEASURE_CTX_PUT") == "" { if os.Getenv("MEASURE_CTX_PUT") == "" {
+68 -7
View File
@@ -26,6 +26,7 @@ import (
"encoding/base64" "encoding/base64"
"encoding/json" "encoding/json"
"fmt" "fmt"
"io/fs"
"os" "os"
"path/filepath" "path/filepath"
"runtime" "runtime"
@@ -74,6 +75,10 @@ type dcnGrantRec struct {
// the signed bytes (a public key; the terms stay exactly what was // the signed bytes (a public key; the terms stay exactly what was
// signed). The managed local agent seals the store's catalog to it. // signed). The managed local agent seals the store's catalog to it.
SealRecipient string `json:"seal_recipient,omitempty"` SealRecipient string `json:"seal_recipient,omitempty"`
// RootReal is where the granted folder led when this box installed the
// grant: its real path, pinned here (dcn_store.go), outside the signed
// bytes. An op whose root leads anywhere else is refused.
RootReal string `json:"root_real,omitempty"`
} }
// dcnTerms is parsed FROM the verified canonical bytes — never from anything else. // dcnTerms is parsed FROM the verified canonical bytes — never from anything else.
@@ -524,6 +529,17 @@ func handleDcnGrantSync(ws *websocket.Conn, dir string, args []string) {
dcnRefuse(ws, dir, "grant_sync", payload.StoreID, "", "grant does not verify: "+err.Error()) dcnRefuse(ws, dir, "grant_sync", payload.StoreID, "", "grant does not verify: "+err.Error())
return return
} }
// The folder this grant names is pinned as it resolves now; a re-sync or a
// renewal of the same root keeps the pin it already has (dcn_store.go).
if prev, held := f.Stores[fmt.Sprint(payload.StoreID)]; held {
prevRoot := ""
if pt, perr := dcnVerifyRec(f.VerifyPubkey, prev); perr == nil {
prevRoot = pt.Root
}
rec.RootReal = dcnPinFor(&prev, prevRoot, terms.Root)
} else {
rec.RootReal = dcnPinFor(nil, "", terms.Root)
}
if hn, _ := os.Hostname(); hn != "" && terms.Server != "" && terms.Server != hn { if hn, _ := os.Hostname(); hn != "" && terms.Server != "" && terms.Server != hn {
dcnRefuse(ws, dir, "grant_sync", payload.StoreID, "", dcnRefuse(ws, dir, "grant_sync", payload.StoreID, "",
fmt.Sprintf("grant is for server '%s', this box is '%s'", terms.Server, hn)) fmt.Sprintf("grant is for server '%s', this box is '%s'", terms.Server, hn))
@@ -658,15 +674,26 @@ func handleDcnList(ws *websocket.Conn, dir string, args []string) {
if terms == nil { if terms == nil {
return return
} }
base := terms.Root base := "."
if a.Prefix != "" { if a.Prefix != "" {
p, ok := dcnResolvePath(terms.Root, a.Prefix) p, ok := dcnResolvePath(terms.Root, a.Prefix)
if !ok { if !ok {
dcnRefuse(ws, dir, "list", storeID, a.Prefix, "prefix escapes the store root") dcnRefuse(ws, dir, "list", storeID, a.Prefix, "prefix escapes the store root")
return return
} }
base = p rel, err := dcnRel(terms.Root, p)
if err != nil {
dcnRefuse(ws, dir, "list", storeID, a.Prefix, "prefix escapes the store root")
return
} }
base = filepath.ToSlash(rel)
}
r, err := dcnOpenStore(dir, storeID, terms)
if err != nil {
dcnRefuse(ws, dir, "list", storeID, a.Prefix, err.Error())
return
}
defer r.Close()
limit := a.Limit limit := a.Limit
if limit <= 0 || limit > dcnMaxListEntries { if limit <= 0 || limit > dcnMaxListEntries {
limit = dcnMaxListEntries limit = dcnMaxListEntries
@@ -678,12 +705,14 @@ func handleDcnList(ws *websocket.Conn, dir string, args []string) {
} }
var files []entry var files []entry
truncated := false truncated := false
_ = filepath.WalkDir(base, func(p string, d os.DirEntry, err error) error { // Walked through the store's handle: a link that leads out of it is not
// followed, whatever the names do during the walk.
_ = fs.WalkDir(r.FS(), base, func(p string, d fs.DirEntry, err error) error {
if err != nil { if err != nil {
return nil // unreadable subtree: skip, never abort the listing return nil // unreadable subtree: skip, never abort the listing
} }
rel, rerr := filepath.Rel(terms.Root, p) rel := filepath.FromSlash(p)
if rerr != nil || rel == "." { if rel == "." {
return nil return nil
} }
if dcnExcluded(terms.Excludes, rel) { if dcnExcluded(terms.Excludes, rel) {
@@ -727,8 +756,23 @@ func handleDcnStat(ws *websocket.Conn, dir string, args []string) {
dcnRefuse(ws, dir, "stat", storeID, a.Path, "path escapes the store root") dcnRefuse(ws, dir, "stat", storeID, a.Path, "path escapes the store root")
return return
} }
info, err := os.Stat(abs) rel, err := dcnRel(terms.Root, abs)
if err != nil { if err != nil {
dcnRefuse(ws, dir, "stat", storeID, a.Path, err.Error())
return
}
r, err := dcnOpenStore(dir, storeID, terms)
if err != nil {
dcnRefuse(ws, dir, "stat", storeID, a.Path, err.Error())
return
}
defer r.Close()
info, err := r.Stat(rel)
if err != nil {
if dcnEscaped(err) {
dcnRefuse(ws, dir, "stat", storeID, a.Path, "path escapes the store root")
return
}
if os.IsNotExist(err) { if os.IsNotExist(err) {
dcnResult(ws, dir, "stat", storeID, a.Path, map[string]any{ dcnResult(ws, dir, "stat", storeID, a.Path, map[string]any{
"state": "gone", "path": a.Path}) "state": "gone", "path": a.Path})
@@ -771,8 +815,25 @@ func handleDcnReadRange(ws *websocket.Conn, dir string, args []string) {
dcnRefuse(ws, dir, "read_range", storeID, a.Path, "path escapes the store root") dcnRefuse(ws, dir, "read_range", storeID, a.Path, "path escapes the store root")
return return
} }
fh, err := os.Open(abs) rel, err := dcnRel(terms.Root, abs)
if err != nil { if err != nil {
dcnRefuse(ws, dir, "read_range", storeID, a.Path, err.Error())
return
}
r, err := dcnOpenStore(dir, storeID, terms)
if err != nil {
dcnRefuse(ws, dir, "read_range", storeID, a.Path, err.Error())
return
}
defer r.Close()
// Opened through the store's handle: a name swapped for a link to a file
// outside the store after the check above is refused, never followed.
fh, err := r.Open(rel)
if err != nil {
if dcnEscaped(err) {
dcnRefuse(ws, dir, "read_range", storeID, a.Path, "path escapes the store root")
return
}
if os.IsNotExist(err) { if os.IsNotExist(err) {
dcnResult(ws, dir, "read_range", storeID, a.Path, map[string]any{ dcnResult(ws, dir, "read_range", storeID, a.Path, map[string]any{
"state": "gone", "path": a.Path}) "state": "gone", "path": a.Path})
+56 -20
View File
@@ -191,10 +191,28 @@ func dcnReportPath(root string, planID int64) string {
return p return p
} }
// dcnApplyPlan runs the plan: check, rename in order (each re-checked just // dcnApplyPlan is dcnApplyPlanIn on root opened where it leads now (the op
// before it runs; rolling back on the first failure), verify, report. Every // handler opens the PINNED folder instead, dcn_store.go).
// path it touches has passed dcnCheckPlan.
func dcnApplyPlan(root string, excludes []string, plan *dcnPlanTerms) map[string]any { func dcnApplyPlan(root string, excludes []string, plan *dcnPlanTerms) map[string]any {
real, err := dcnRealRoot(root)
if err != nil {
return map[string]any{"state": "refused", "mode": "apply", "plan_id": plan.PlanID,
"reason": "the store's folder cannot be opened: " + err.Error()}
}
r, err := dcnOpenRootAt(real)
if err != nil {
return map[string]any{"state": "refused", "mode": "apply", "plan_id": plan.PlanID, "reason": err.Error()}
}
defer r.Close()
return dcnApplyPlanIn(r, root, excludes, plan)
}
// dcnApplyPlanIn runs the plan: check, rename in order (each re-checked just
// before it runs; rolling back on the first failure), verify, report. Every
// path it touches has passed dcnCheckPlan, and every change is made through
// r, the store's handle, so a folder that becomes a link to somewhere else
// between the check and the rename is refused rather than followed.
func dcnApplyPlanIn(r *os.Root, root string, excludes []string, plan *dcnPlanTerms) map[string]any {
problems := dcnCheckPlan(root, excludes, plan.Moves) problems := dcnCheckPlan(root, excludes, plan.Moves)
if len(problems) > 0 { if len(problems) > 0 {
return map[string]any{"state": "refused", "mode": "apply", "plan_id": plan.PlanID, return map[string]any{"state": "refused", "mode": "apply", "plan_id": plan.PlanID,
@@ -207,11 +225,11 @@ func dcnApplyPlan(root string, excludes []string, plan *dcnPlanTerms) map[string
// and what is NOT (so the user knows exactly what is still displaced). // and what is NOT (so the user knows exactly what is still displaced).
rollback := func() (back, stuck []string) { rollback := func() (back, stuck []string) {
for i := len(applied) - 1; i >= 0; i-- { for i := len(applied) - 1; i >= 0; i-- {
if _, err := os.Lstat(applied[i].src); err == nil { if _, err := r.Lstat(applied[i].src); err == nil {
stuck = append(stuck, applied[i].from) // something now sits at the source; never replace it stuck = append(stuck, applied[i].from) // something now sits at the source; never replace it
continue continue
} }
if err := os.Rename(applied[i].dst, applied[i].src); err == nil { if err := r.Rename(applied[i].dst, applied[i].src); err == nil {
back = append(back, applied[i].from) back = append(back, applied[i].from)
} else { } else {
stuck = append(stuck, applied[i].from) stuck = append(stuck, applied[i].from)
@@ -230,24 +248,30 @@ func dcnApplyPlan(root string, excludes []string, plan *dcnPlanTerms) map[string
return rep return rep
} }
for i, m := range plan.Moves { for i, m := range plan.Moves {
src, _ := dcnResolvePath(root, m.From) srcAbs, _ := dcnResolvePath(root, m.From)
dst, _ := cleanRel(root, m.To) dstAbs, _ := cleanRel(root, m.To)
if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { src, serr := dcnRel(root, srcAbs)
dst, derr := dcnRel(root, dstAbs)
if serr != nil || derr != nil {
return fail(i, m, "path escapes the store root")
}
if d := filepath.Dir(dst); d != "." {
if err := r.MkdirAll(d, 0o755); err != nil {
return fail(i, m, "could not create the target folder: "+err.Error()) return fail(i, m, "could not create the target folder: "+err.Error())
} }
}
// Re-check the instant before the rename: the world may have changed since // Re-check the instant before the rename: the world may have changed since
// the pre-check (a file appeared at the target; a parent became a symlink). // the pre-check (a file appeared at the target; a parent became a symlink).
// rename(2) would replace silently — this lane never lets it. // rename(2) would replace silently — this lane never lets it.
if _, ok := dcnResolvePath(root, m.To); !ok { if _, ok := dcnResolvePath(root, m.To); !ok {
return fail(i, m, "target's parent no longer resolves inside the store root") return fail(i, m, "target's parent no longer resolves inside the store root")
} }
if _, err := os.Lstat(dst); err == nil { if _, err := r.Lstat(dst); err == nil {
return fail(i, m, "target appeared since the check (a plan never overwrites)") return fail(i, m, "target appeared since the check (a plan never overwrites)")
} }
if err := os.Rename(src, dst); err != nil { if err := r.Rename(src, dst); err != nil {
why := err.Error() why := err.Error()
var le *os.LinkError if errors.Is(err, syscall.EXDEV) {
if errors.As(err, &le) && errors.Is(le.Err, syscall.EXDEV) {
why = "target is on a different filesystem than the source (a plan only renames; it never copies and removes)" why = "target is on a different filesystem than the source (a plan only renames; it never copies and removes)"
} }
return fail(i, m, why) return fail(i, m, why)
@@ -257,14 +281,16 @@ func dcnApplyPlan(root string, excludes []string, plan *dcnPlanTerms) map[string
// Verify pass: every target present, every source gone, declared sizes hold. // Verify pass: every target present, every source gone, declared sizes hold.
var mismatches []dcnPlanProblem var mismatches []dcnPlanProblem
for i, m := range plan.Moves { for i, m := range plan.Moves {
src, _ := dcnResolvePath(root, m.From) srcAbs, _ := dcnResolvePath(root, m.From)
dst, _ := cleanRel(root, m.To) dstAbs, _ := cleanRel(root, m.To)
info, err := os.Lstat(dst) src, _ := dcnRel(root, srcAbs)
dst, _ := dcnRel(root, dstAbs)
info, err := r.Lstat(dst)
if err != nil { if err != nil {
mismatches = append(mismatches, dcnPlanProblem{i, m.From, m.To, "target missing after move"}) mismatches = append(mismatches, dcnPlanProblem{i, m.From, m.To, "target missing after move"})
continue continue
} }
if _, err := os.Lstat(src); err == nil { if _, err := r.Lstat(src); err == nil {
mismatches = append(mismatches, dcnPlanProblem{i, m.From, m.To, "source still present after move"}) mismatches = append(mismatches, dcnPlanProblem{i, m.From, m.To, "source still present after move"})
} }
if m.SizeBytes != nil && !info.IsDir() && info.Size() != *m.SizeBytes { if m.SizeBytes != nil && !info.IsDir() && info.Size() != *m.SizeBytes {
@@ -372,9 +398,19 @@ func handleDcnPlanApply(ws *websocket.Conn, dir string, args []string) {
// a retry (a lost reply, a host re-run) gets the earlier report back BEFORE // a retry (a lost reply, a host re-run) gets the earlier report back BEFORE
// any gate can turn it into a refusal. (The signed expiry is the guard that // any gate can turn it into a refusal. (The signed expiry is the guard that
// does not depend on a file the user can remove.) // does not depend on a file the user can remove.)
store, err := dcnOpenStore(dir, storeID, terms)
if err != nil {
dcnRefuse(ws, dir, op, storeID, planKey, err.Error())
return
}
defer store.Close()
repPath := dcnReportPath(terms.Root, plan.PlanID) repPath := dcnReportPath(terms.Root, plan.PlanID)
repRel := ""
if repPath != "" { if repPath != "" {
if prev, err := os.ReadFile(repPath); err == nil { repRel, _ = dcnRel(terms.Root, repPath)
}
if repRel != "" {
if prev, err := store.ReadFile(repRel); err == nil {
var old map[string]any var old map[string]any
if json.Unmarshal(prev, &old) == nil && old["state"] == "ok" { if json.Unmarshal(prev, &old) == nil && old["state"] == "ok" {
old["note"] = "already applied on this machine; this is the earlier report" old["note"] = "already applied on this machine; this is the earlier report"
@@ -398,13 +434,13 @@ func handleDcnPlanApply(ws *websocket.Conn, dir string, args []string) {
return return
} }
} }
rep := dcnApplyPlan(terms.Root, terms.Excludes, plan) rep := dcnApplyPlanIn(store, terms.Root, terms.Excludes, plan)
if rep["state"] == "ok" && plan.Approval != "" { if rep["state"] == "ok" && plan.Approval != "" {
markApprovalUsed(dir, plan.Approval) // single use: the files have moved markApprovalUsed(dir, plan.Approval) // single use: the files have moved
} }
if repPath != "" { if repRel != "" {
if raw, err := json.MarshalIndent(rep, "", " "); err == nil { if raw, err := json.MarshalIndent(rep, "", " "); err == nil {
_ = atomicWrite(repPath, raw) // the machine's own record; best-effort _ = dcnRootAtomicWrite(store, repRel, raw) // the machine's own record; best-effort
} }
} }
dcnResult(ws, dir, op, storeID, planKey, rep) dcnResult(ws, dir, op, storeID, planKey, rep)
+322
View File
@@ -0,0 +1,322 @@
package main
// A machine store is the folder the person granted, for as long as the grant
// lives. The bastion often runs as root and the folder belongs to someone who
// is not, so whoever can write in it, or above it, must not be able to point
// a read or a move somewhere else: by turning the granted folder into a link
// after the grant (no race needed), or by swapping a file for a link between
// the check and the open.
import (
"crypto/ed25519"
"encoding/base64"
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strconv"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/gorilla/websocket"
)
// dcnRunOp runs one DCN handler on the far side of a real WebSocket and
// returns what it sent: the O payload, the R text and the exit code.
func dcnRunOp(t *testing.T, h func(ws *websocket.Conn)) (out, errText string, exit int) {
t.Helper()
up := websocket.Upgrader{}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
ws, err := up.Upgrade(w, r, nil)
if err != nil {
return
}
defer ws.Close()
h(ws)
}))
defer srv.Close()
ws, _, err := websocket.DefaultDialer.Dial("ws"+strings.TrimPrefix(srv.URL, "http"), nil)
if err != nil {
t.Fatal(err)
}
defer ws.Close()
_ = ws.SetReadDeadline(time.Now().Add(10 * time.Second))
exit = -1
for {
_, data, err := ws.ReadMessage()
if err != nil || len(data) == 0 {
return
}
switch data[0] {
case 'O':
out += string(data[1:])
case 'R':
errText += string(data[1:])
case 'X':
exit, _ = strconv.Atoi(string(data[1:]))
return
}
}
}
type dcnStoreFixture struct {
dir string // the pairing's dcn dir
base string
store string // the granted folder
other string // a folder the grant never named
priv ed25519.PrivateKey
pub ed25519.PublicKey
}
func newDcnStoreFixture(t *testing.T) *dcnStoreFixture {
t.Helper()
fx := &dcnStoreFixture{dir: t.TempDir(), base: t.TempDir()}
fx.store = filepath.Join(fx.base, "store")
fx.other = filepath.Join(fx.base, "other")
for _, d := range []string{fx.store, fx.other, filepath.Join(fx.store, "docs")} {
if err := os.MkdirAll(d, 0o755); err != nil {
t.Fatal(err)
}
}
if err := os.WriteFile(filepath.Join(fx.store, "a.txt"), []byte("MINE"), 0o644); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(fx.other, "a.txt"), []byte("OTHER"), 0o600); err != nil {
t.Fatal(err)
}
fx.pub, fx.priv, _ = ed25519.GenerateKey(nil)
return fx
}
// sync installs a read grant for the store through the real grant_sync op.
func (fx *dcnStoreFixture) sync(t *testing.T, mask ...string) {
t.Helper()
if len(mask) == 0 {
mask = []string{"read"}
}
rec := signedRec(t, fx.priv, map[string]any{"kind": "dcn_store_grant", "class": "machine",
"root": fx.store, "mask": mask,
"expires": time.Now().Add(24 * time.Hour).UTC().Format(time.RFC3339)})
payload, _ := json.Marshal(map[string]any{"store_id": 1, "terms_canonical": rec.TermsCanonical,
"signature": rec.Signature, "verify_pubkey": base64.StdEncoding.EncodeToString(fx.pub)})
_, why, exit := dcnRunOp(t, func(ws *websocket.Conn) {
handleDcnGrantSync(ws, fx.dir, []string{base64.StdEncoding.EncodeToString(payload)})
})
if exit != 0 {
t.Fatalf("grant_sync: exit %d: %s", exit, why)
}
}
func (fx *dcnStoreFixture) op(t *testing.T, h func(*websocket.Conn, string, []string), args map[string]any) (map[string]any, string, int) {
t.Helper()
raw, _ := json.Marshal(args)
out, why, exit := dcnRunOp(t, func(ws *websocket.Conn) {
h(ws, fx.dir, []string{"1", string(raw)})
})
var obj map[string]any
_ = json.Unmarshal([]byte(out), &obj)
return obj, why, exit
}
func (fx *dcnStoreFixture) read(t *testing.T, path string) (string, string, int) {
t.Helper()
obj, why, exit := fx.op(t, handleDcnReadRange, map[string]any{"path": path})
b, _ := base64.StdEncoding.DecodeString(func() string { s, _ := obj["content_b64"].(string); return s }())
return string(b), why, exit
}
// repoint turns the granted folder into a link to the other one.
func (fx *dcnStoreFixture) repoint(t *testing.T) {
t.Helper()
if err := os.Rename(fx.store, fx.store+".was"); err != nil {
t.Fatal(err)
}
if err := os.Symlink(fx.other, fx.store); err != nil {
t.Skipf("symlinks unavailable: %v", err)
}
}
func TestDcnStoreRepointedAfterTheGrantIsRefused(t *testing.T) {
fx := newDcnStoreFixture(t)
fx.sync(t)
if got, why, exit := fx.read(t, "a.txt"); exit != 0 || got != "MINE" {
t.Fatalf("the granted file must read: exit %d %q %s", exit, got, why)
}
fx.repoint(t)
if got, _, exit := fx.read(t, "a.txt"); exit == 0 || got == "OTHER" {
t.Fatalf("after the granted folder became a link elsewhere, read_range returned %q (exit %d); it must refuse", got, exit)
}
if obj, _, exit := fx.op(t, handleDcnStat, map[string]any{"path": "a.txt"}); exit == 0 {
t.Fatalf("stat through the repointed folder must refuse, got %v", obj)
}
if obj, _, exit := fx.op(t, handleDcnList, map[string]any{}); exit == 0 {
t.Fatalf("list through the repointed folder must refuse, got %v", obj)
}
}
// A folder inside the store that becomes a link to somewhere else after the
// grant is the same escape one level down.
func TestDcnSubfolderRepointedIsRefused(t *testing.T) {
fx := newDcnStoreFixture(t)
fx.sync(t)
if err := os.RemoveAll(filepath.Join(fx.store, "docs")); err != nil {
t.Fatal(err)
}
if err := os.Symlink(fx.other, filepath.Join(fx.store, "docs")); err != nil {
t.Skipf("symlinks unavailable: %v", err)
}
if got, _, exit := fx.read(t, "docs/a.txt"); exit == 0 || got == "OTHER" {
t.Fatalf("a link inside the store to a folder outside it read %q (exit %d)", got, exit)
}
}
// The race: a file swapped for a link to a file outside the store between the
// check and the open. Run for long enough that the old check-then-open loses.
func TestDcnReadRangeSwapRaceNeverLeavesTheStore(t *testing.T) {
fx := newDcnStoreFixture(t)
fx.sync(t)
target := filepath.Join(fx.store, "a.txt")
realTmp := filepath.Join(fx.store, ".real")
linkTmp := filepath.Join(fx.store, ".link")
var stop atomic.Bool
done := make(chan struct{})
go func() {
defer close(done)
for !stop.Load() {
_ = os.Symlink(filepath.Join(fx.other, "a.txt"), linkTmp)
_ = os.Rename(linkTmp, target)
_ = os.WriteFile(realTmp, []byte("MINE"), 0o644)
_ = os.Rename(realTmp, target)
}
}()
deadline := time.Now().Add(3 * time.Second)
n := 0
for time.Now().Before(deadline) {
n++
if got, _, _ := fx.read(t, "a.txt"); got == "OTHER" {
stop.Store(true)
<-done
t.Fatalf("read %d returned the file outside the store through a swapped link", n)
}
}
stop.Store(true)
<-done
}
// A move plan applies through the store too: a target folder that leads
// outside the store for an instant between the re-check and the rename must
// not receive the file.
func TestDcnPlanMoveRaceNeverLeavesTheStore(t *testing.T) {
fx := newDcnStoreFixture(t)
if err := os.Remove(filepath.Join(fx.other, "a.txt")); err != nil {
t.Fatal(err)
}
docs := filepath.Join(fx.store, "docs")
var stop atomic.Bool
done := make(chan struct{})
go func() {
defer close(done)
for !stop.Load() {
_ = os.Rename(docs, docs+".real")
_ = os.Symlink(fx.other, docs+".link")
_ = os.Rename(docs+".link", docs)
_ = os.Rename(docs, docs+".link")
_ = os.Remove(docs + ".link")
_ = os.Rename(docs+".real", docs)
}
}()
defer func() { stop.Store(true); <-done }()
deadline := time.Now().Add(3 * time.Second)
for i := 0; time.Now().Before(deadline); i++ {
_ = os.WriteFile(filepath.Join(fx.store, "a.txt"), []byte("MINE"), 0o644)
plan := &dcnPlanTerms{PlanID: int64(i), StoreID: 1, Root: fx.store,
Moves: []dcnMove{{From: "a.txt", To: "docs/a.txt"}}}
_ = dcnApplyPlan(fx.store, nil, plan)
if _, err := os.Lstat(filepath.Join(fx.other, "a.txt")); err == nil {
t.Fatalf("move %d landed outside the store, through a folder that was a link for an instant", i)
}
_ = os.Remove(filepath.Join(fx.store, "docs.real", "a.txt"))
_ = os.Remove(filepath.Join(docs, "a.txt"))
}
}
// The plan report is written beside its name and renamed into place. Whoever
// can write in the store could plant a link at that side name; the write must
// never follow it, or the root bastion truncates whatever file the link names
// (an excluded .ssh/authorized_keys included) and fills it with the report.
func TestDcnReportWriteNeverFollowsAPlantedSideName(t *testing.T) {
store := t.TempDir()
victim := filepath.Join(store, ".ssh", "authorized_keys")
if err := os.MkdirAll(filepath.Dir(victim), 0o700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(victim, []byte("ssh-ed25519 AAAAexample user@example\n"), 0o600); err != nil {
t.Fatal(err)
}
plans := filepath.Join(store, ".agent", "plans")
if err := os.MkdirAll(plans, 0o700); err != nil {
t.Fatal(err)
}
rel := filepath.Join(".agent", "plans", "7.json")
// Every side name a writer could guess: this process's pid, and a spread
// of others, all pointing at the victim.
for _, pid := range []int{os.Getpid(), 1, 2, 100, 1000} {
side := filepath.Join(store, rel+".tmp-"+strconv.Itoa(pid))
if err := os.Symlink(filepath.Join("..", "..", ".ssh", "authorized_keys"), side); err != nil {
t.Fatal(err)
}
}
r, err := os.OpenRoot(store)
if err != nil {
t.Fatal(err)
}
defer r.Close()
if err := dcnRootAtomicWrite(r, rel, []byte(`{"state":"ok"}`)); err != nil {
t.Fatalf("report write: %v", err)
}
if got, _ := os.ReadFile(victim); string(got) != "ssh-ed25519 AAAAexample user@example\n" {
t.Fatalf("the report write followed a planted link: the victim now holds %q", got)
}
if got, err := os.ReadFile(filepath.Join(store, rel)); err != nil || string(got) != `{"state":"ok"}` {
t.Fatalf("report not in place: %q %v", got, err)
}
if fi, err := os.Lstat(filepath.Join(store, rel)); err != nil || fi.Mode()&os.ModeSymlink != 0 || fi.Mode().Perm() != 0o600 {
t.Fatalf("report is not a 0600 regular file: %v %v", fi, err)
}
}
// A grant installed before pins existed has no root_real. It still works,
// pins where its folder leads at first use, keeps that pin, and a repoint
// after that is refused like any other.
func TestDcnGrantFromBeforePinsPinsAtFirstUse(t *testing.T) {
fx := newDcnStoreFixture(t)
fx.sync(t)
f, err := dcnLoadFile(fx.dir)
if err != nil {
t.Fatal(err)
}
rec := f.Stores["1"]
rec.RootReal = ""
f.Stores["1"] = rec
if err := dcnSaveFile(fx.dir, f); err != nil {
t.Fatal(err)
}
if got, why, exit := fx.read(t, "a.txt"); exit != 0 || got != "MINE" {
t.Fatalf("a grant from before pins no longer reads: exit %d %q %s", exit, got, why)
}
f, err = dcnLoadFile(fx.dir)
if err != nil {
t.Fatal(err)
}
want, _ := filepath.EvalSymlinks(fx.store)
if got := f.Stores["1"].RootReal; got != want {
t.Fatalf("first use pinned %q, want %q", got, want)
}
fx.repoint(t)
if got, _, exit := fx.read(t, "a.txt"); exit == 0 || got == "OTHER" {
t.Fatalf("after the first-use pin a repointed folder was read: exit %d %q", exit, got)
}
}
+199
View File
@@ -0,0 +1,199 @@
package main
// dcn_store.go — every byte a machine-store op touches is reached through ONE
// handle on the folder the person granted.
//
// WHY. The bastion is often root (a networked install, the macOS and Windows
// system service), and the granted folder is often someone else's. Checking a
// path and then opening it by name leaves two ways out of the store for
// whoever can write in it or above it:
//
// - the granted folder itself turned into a link to another folder after the
// grant. The old check resolved the root again on every op and judged the
// path against wherever it led NOW, so no race was needed;
// - a file or folder inside the store swapped for a link between the check
// and the open (or the rename, for a move plan).
//
// So: the folder's real path is PINNED at grant sync (rec.RootReal, this box's
// own field, outside the signed bytes), an op whose root now leads elsewhere
// is refused, and the op opens the pinned folder as an os.Root, which never
// follows a link out of it, whatever happens to the names afterwards. That
// handle is then checked to be the folder at the pinned path reached with no
// link at all (dcnNoLinkStat: an open that refuses a link in every component
// on Linux and macOS), so a swap of a folder above the store between the two
// opens is refused too.
import (
"crypto/rand"
"encoding/hex"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"strconv"
)
// errDcnRepointed is the refusal for a granted folder that leads elsewhere now.
var errDcnRepointed = errors.New("the granted folder now leads somewhere other than the folder that was granted; " +
"a new grant for it is the repair")
// dcnRealRoot is where the grant's root leads now: its links resolved.
func dcnRealRoot(root string) (string, error) {
real, err := filepath.EvalSymlinks(filepath.Clean(root))
if err != nil {
return "", err
}
if !filepath.IsAbs(real) {
return "", errors.New("the granted folder is not an absolute path")
}
return filepath.Clean(real), nil
}
// dcnPinFor is the real root a sync records: the one already pinned for the
// same root while the store is held (a re-sync, a renewed grant), else where
// the root leads now ("" when it cannot be resolved yet; pinned at first use).
func dcnPinFor(prev *dcnGrantRec, prevRoot, root string) string {
if prev != nil && prev.RootReal != "" && filepath.Clean(prevRoot) == filepath.Clean(root) {
return prev.RootReal
}
real, err := dcnRealRoot(root)
if err != nil {
return ""
}
return real
}
// dcnStorePin is the pinned real root of a held store, pinning it now (and
// saving it) for a grant synced before pins existed.
func dcnStorePin(dir string, storeID int64, root string) (string, error) {
grantsMu.Lock()
defer grantsMu.Unlock()
f, err := dcnLoadFile(dir)
if err != nil {
return "", err
}
key := strconv.FormatInt(storeID, 10)
rec, ok := f.Stores[key]
if !ok {
return "", errors.New("no grant for this store on this box")
}
if rec.RootReal != "" {
return rec.RootReal, nil
}
real, err := dcnRealRoot(root)
if err != nil {
return "", err
}
rec.RootReal = real
f.Stores[key] = rec
if err := dcnSaveFile(dir, f); err != nil {
return "", err
}
return real, nil
}
// dcnOpenStore opens the store a verified grant names, as the folder that was
// granted, or refuses.
func dcnOpenStore(dir string, storeID int64, terms *dcnTerms) (*os.Root, error) {
pin, err := dcnStorePin(dir, storeID, terms.Root)
if err != nil {
return nil, fmt.Errorf("the granted folder cannot be opened: %w", err)
}
real, err := dcnRealRoot(terms.Root)
if err != nil {
return nil, fmt.Errorf("the granted folder cannot be opened: %w", err)
}
if real != pin {
return nil, errDcnRepointed
}
return dcnOpenRootAt(pin)
}
// dcnOpenRootAt opens real (a resolved, absolute path) as an os.Root and
// checks that the handle is the folder reached at real with no link in any
// component, so nothing above the folder was swapped while it was opened.
func dcnOpenRootAt(real string) (*os.Root, error) {
r, err := os.OpenRoot(real)
if err != nil {
return nil, fmt.Errorf("the granted folder cannot be opened: %w", err)
}
rfi, err := r.Stat(".")
if err != nil {
_ = r.Close()
return nil, fmt.Errorf("the granted folder cannot be opened: %w", err)
}
nfi, err := dcnNoLinkStat(real)
if err != nil || !nfi.IsDir() || !os.SameFile(rfi, nfi) {
_ = r.Close()
return nil, errDcnRepointed
}
return r, nil
}
// dcnRel is a path the sandbox accepted (cleanRel/dcnResolvePath's joined
// form) as a name inside the root handle.
func dcnRel(root, joined string) (string, error) {
rel, err := filepath.Rel(filepath.Clean(root), joined)
if err != nil || rel == ".." || filepath.IsAbs(rel) || len(rel) >= 3 && rel[:3] == ".."+string(os.PathSeparator) {
return "", errors.New("path escapes the store root")
}
return rel, nil
}
// dcnEscaped: the root handle refused a name because it leads out of the store.
func dcnEscaped(err error) bool {
var pe *os.PathError
return errors.As(err, &pe) && pe.Err != nil && pe.Err.Error() == "path escapes from parent"
}
// dcnRootAtomicWrite is atomicWrite inside the store's handle: folders 0700,
// the file 0600, written beside its name and renamed into place.
func dcnRootAtomicWrite(r *os.Root, rel string, data []byte) error {
if d := filepath.Dir(rel); d != "." {
if err := r.MkdirAll(d, 0o700); err != nil {
return err
}
}
// The side name is unguessable and opened O_EXCL, as atomicWrite's
// CreateTemp is: whoever can write in the store could otherwise plant a
// link at a guessable name, and an open that follows it would truncate
// and fill whatever file in the store the link names. O_EXCL refuses any
// existing name, a link included, without following it.
var f *os.File
var tmp string
for try := 0; ; try++ {
var nb [12]byte
if _, err := rand.Read(nb[:]); err != nil {
return err
}
tmp = filepath.Join(filepath.Dir(rel), ".dcn-"+hex.EncodeToString(nb[:])+".tmp")
var err error
f, err = r.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
if err == nil {
break
}
if !errors.Is(err, fs.ErrExist) || try >= 8 {
return err
}
}
if _, err := f.Write(data); err != nil {
_ = f.Close()
_ = r.Remove(tmp)
return err
}
if err := f.Sync(); err != nil {
_ = f.Close()
_ = r.Remove(tmp)
return err
}
if err := f.Close(); err != nil {
_ = r.Remove(tmp)
return err
}
if err := r.Rename(tmp, rel); err != nil {
_ = r.Remove(tmp)
return err
}
return nil
}
+21
View File
@@ -0,0 +1,21 @@
//go:build darwin
package main
import (
"os"
"golang.org/x/sys/unix"
)
// dcnNoLinkStat describes the directory at p opened with O_NOFOLLOW_ANY, which
// fails on a link in any component (macOS 11 and later).
func dcnNoLinkStat(p string) (os.FileInfo, error) {
fd, err := unix.Open(p, unix.O_RDONLY|unix.O_DIRECTORY|unix.O_NOFOLLOW_ANY|unix.O_CLOEXEC, 0)
if err != nil {
return nil, err
}
f := os.NewFile(uintptr(fd), p)
defer f.Close()
return f.Stat()
}
+48
View File
@@ -0,0 +1,48 @@
//go:build linux
package main
import (
"errors"
"os"
"strings"
"golang.org/x/sys/unix"
)
// dcnNoLinkStat describes the directory at p (absolute, clean) reached one
// component at a time with O_NOFOLLOW, so a link anywhere in the path fails
// instead of being followed. O_PATH: a component the bastion may traverse but
// not list (a home at 0711) still opens.
func dcnNoLinkStat(p string) (os.FileInfo, error) {
if !strings.HasPrefix(p, "/") {
return nil, errors.New("not an absolute path")
}
fd, err := unix.Open("/", unix.O_PATH|unix.O_DIRECTORY|unix.O_CLOEXEC, 0)
if err != nil {
return nil, err
}
for _, c := range strings.Split(p, "/") {
if c == "" {
continue
}
if c == "." || c == ".." {
_ = unix.Close(fd)
return nil, errors.New("not a clean path")
}
next, err := unix.Openat(fd, c, unix.O_PATH|unix.O_NOFOLLOW|unix.O_DIRECTORY|unix.O_CLOEXEC, 0)
_ = unix.Close(fd)
if err != nil {
return nil, err
}
var st unix.Stat_t
if err := unix.Fstat(next, &st); err != nil || st.Mode&unix.S_IFMT != unix.S_IFDIR {
_ = unix.Close(next)
return nil, errors.New("a component is not a directory")
}
fd = next
}
f := os.NewFile(uintptr(fd), p)
defer f.Close()
return f.Stat()
}
+14
View File
@@ -0,0 +1,14 @@
//go:build !linux && !darwin
package main
import "os"
// dcnNoLinkStat on a platform with no open that refuses a link in every
// component (Windows): the final component is judged without following it.
// The store is still reached only through its os.Root and only while its real
// path matches the pin; what this leaves is a swap of a folder above the
// store, by someone who can rename it, timed between the two opens.
func dcnNoLinkStat(p string) (os.FileInfo, error) {
return os.Lstat(p)
}
-18
View File
@@ -296,13 +296,6 @@ func nonceSave(m map[string]string) error {
return atomicWrite(p, raw) return atomicWrite(p, raw)
} }
func nonceSeen(nonce string) bool {
deviceMu.Lock()
defer deviceMu.Unlock()
_, ok := nonceLoad()[nonce]
return ok
}
// burnNonce records the nonce until its approval's expiry (after which a // burnNonce records the nonce until its approval's expiry (after which a
// replay fails on expiry anyway). // replay fails on expiry anyway).
func burnNonce(a *deviceApproval) { func burnNonce(a *deviceApproval) {
@@ -331,17 +324,6 @@ func approvalOK(dir, approval, purpose string) error {
return stepupCheckApproval(dir, approval, purpose) return stepupCheckApproval(dir, approval, purpose)
} }
// approvalKind names how an approval was given, for the audit line.
func approvalKind(approval string) string {
switch {
case strings.TrimSpace(approval) == "":
return ""
case isDeviceApproval(approval):
return "device"
}
return "console"
}
// approvalRequired: does a change on this box need the person's approval? // approvalRequired: does a change on this box need the person's approval?
// Always on an anchored box (writes.go); on a local box only when the person // Always on an anchored box (writes.go); on a local box only when the person
// enrolled a factor here — the rule the DCN lanes have had since stepup.go. // enrolled a factor here — the rule the DCN lanes have had since stepup.go.
+283 -95
View File
@@ -3,8 +3,10 @@ package main
import ( import (
"bufio" "bufio"
"context" "context"
"errors"
"io" "io"
"net" "net"
"net/http"
"strings" "strings"
"time" "time"
) )
@@ -23,94 +25,110 @@ type egressDialer func(ctx context.Context, network, addr string) (net.Conn, err
// egress is refused (403, before any dial) unless its pairing grant carries // egress is refused (403, before any dial) unless its pairing grant carries
// the "egress" cap: this lane makes requests leave THIS machine's connection // the "egress" cap: this lane makes requests leave THIS machine's connection
// under its address, and nothing in the mgmt op table gates it. // under its address, and nothing in the mgmt op table gates it.
func handleEgressConn(client net.Conn, dialOut egressDialer, self string) { //
// It reports whether it took the job: true once the request passed every check
// and was dialled out, whatever the destination then answered; false when the
// worker closed before a request, or the request was malformed or refused. The
// worker pool (runEgressWorkers) parks again at once after a job it took, and
// waits after one it did not.
func handleEgressConn(client net.Conn, dialOut egressDialer, self string) bool {
defer client.Close() defer client.Close()
br := bufio.NewReader(client) // The request line and headers are read under a bound, the one Go's own HTTP
// server uses (http.DefaultMaxHeaderBytes, 1 MiB): the host sends them, and a
// head that never ends must not grow in this machine's memory.
headLim := &io.LimitedReader{R: client, N: egressMaxHeadBytes}
br := bufio.NewReader(headLim)
line, err := br.ReadString('\n') line, err := br.ReadString('\n')
if err != nil { if err != nil {
return return false
} }
parts := strings.Fields(strings.TrimSpace(line)) parts := strings.Fields(strings.TrimSpace(line))
if len(parts) < 2 { if len(parts) < 2 {
return return false
} }
method, target := parts[0], parts[1] method, target := parts[0], parts[1]
// Buffer the head so a plain-HTTP request can be replayed verbatim; capture Host. // Buffer the head: a plain-HTTP request is parsed from it and rebuilt
// (parseEgressRequest), never replayed verbatim.
var head strings.Builder var head strings.Builder
head.WriteString(line) head.WriteString(line)
hostHdr := ""
for { for {
h, err := br.ReadString('\n') h, err := br.ReadString('\n')
if err != nil { if err != nil {
return return false
} }
head.WriteString(h) head.WriteString(h)
if h == "\r\n" || h == "\n" { if h == "\r\n" || h == "\n" {
break break
} }
if len(h) >= 5 && strings.EqualFold(h[:5], "host:") {
hostHdr = strings.TrimSpace(h[5:])
}
} }
// The head is in; what follows is the stream itself, unbounded. What the
// reader already buffered past the head stays first in line.
headLim.N = 1<<63 - 1
if why := capsAllow(self, "egress"); why != "" { if why := capsAllow(self, "egress"); why != "" {
_, _ = client.Write([]byte("HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\nX-Privatae-Refused: " + why + "\r\n\r\n")) writeEgressRefusal(client, "403 Forbidden", why, why)
return return false
} }
ctx, cancel := context.WithTimeout(context.Background(), egressDialTimeout) ctx, cancel := context.WithTimeout(context.Background(), egressDialTimeout)
defer cancel() defer cancel()
if strings.EqualFold(method, "CONNECT") { if strings.EqualFold(method, "CONNECT") {
dialAddr, why := egressVettedAddr(ctx, ensurePort(target, "443")) // The target is read the way the Host's egress gate reads it
if why != "" { // (parseEgressAuthority): userinfo, an unbracketed IPv6 literal or a
_, _ = client.Write([]byte("HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\nX-Privatae-Refused: " + why + "\r\n\r\n")) // malformed port is refused, never guessed at.
return host, port, ok := parseEgressAuthority(target, 443)
if !ok {
writeEgressRefusal(client, "400 Bad Request", egressNoHost, egressNoHost)
return false
} }
upstream, err := dialOut(ctx, "tcp", dialAddr) dialAddrs, why := egressVettedAddrs(ctx, egressHostPort(host, port))
if why != "" {
writeEgressRefusal(client, "403 Forbidden", why, why)
return false
}
upstream, err := dialEach(ctx, dialOut, dialAddrs)
if err != nil { if err != nil {
_, _ = client.Write([]byte("HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\n\r\n")) writeEgressRefusal(client, "502 Bad Gateway", egressUnreachable, "")
return return true
} }
if _, err := client.Write([]byte("HTTP/1.1 200 Connection established\r\n\r\n")); err != nil { if _, err := client.Write([]byte("HTTP/1.1 200 Connection established\r\n\r\n")); err != nil {
_ = upstream.Close() _ = upstream.Close()
return return true
} }
splice(br, client, upstream) splice(br, client, upstream)
return return true
} }
// Plain HTTP, absolute-form: "GET http://host/path HTTP/1.1". // Plain HTTP: ONE request per connection, parsed and rebuilt so the
host := hostHdr // destination reads the host, port and body the lane vetted, sent with
if i := strings.Index(target, "://"); i >= 0 { // `Connection: close` (egress_http.go). Anything the Host sends after that
rest := target[i+3:] // request's body closes the connection unforwarded: a keep-alive connection
if s := strings.IndexByte(rest, '/'); s >= 0 { // can no longer carry later requests past the vetting.
host = rest[:s] req, refused := parseEgressRequest(head.String())
} else { if refused != nil {
host = rest writeEgressRefusal(client, refused.status, refused.body, refused.body)
return false
} }
} dialAddrs, why := egressVettedAddrs(ctx, egressHostPort(req.host, req.port))
if host == "" {
return
}
dialAddr, why := egressVettedAddr(ctx, ensurePort(host, "80"))
if why != "" { if why != "" {
_, _ = client.Write([]byte("HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\nX-Privatae-Refused: " + why + "\r\n\r\n")) writeEgressRefusal(client, "403 Forbidden", why, why)
return return false
} }
upstream, err := dialOut(ctx, "tcp", dialAddr) upstream, err := dialEach(ctx, dialOut, dialAddrs)
if err != nil { if err != nil {
_, _ = client.Write([]byte("HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\n\r\n")) writeEgressRefusal(client, "502 Bad Gateway", egressUnreachable, "")
return return true
} }
if _, err := upstream.Write([]byte(head.String())); err != nil { oneEgressExchange(br, client, upstream, req)
_ = upstream.Close() return true
return
}
splice(br, client, upstream)
} }
// egressMaxHeadBytes bounds an egress request's line and headers together.
const egressMaxHeadBytes = http.DefaultMaxHeaderBytes
// ensurePort returns host:port, adding defPort when target has no explicit port. // ensurePort returns host:port, adding defPort when target has no explicit port.
func ensurePort(target, defPort string) string { func ensurePort(target, defPort string) string {
target = strings.TrimSpace(target) target = strings.TrimSpace(target)
@@ -145,47 +163,68 @@ func splice(clientReader io.Reader, client, upstream net.Conn) {
_ = client.Close() _ = client.Close()
} }
// egressInternalRefusal: on an ANCHORED box the egress lane reaches the public // egressInternalRefusal is the refusal egressVettedAddrs gives, or "".
// internet and nothing else. Loopback, link-local and private addresses — this //
// machine's own services, the LAN it sits on, the mesh — are refused before // The lane is USE of the machine's connection (writes.go), and that holds only
// any dial, whether named by address or by a name that resolves to one. The
// lane is USE of the machine's connection (writes.go), and that holds only
// while it cannot be turned inward: `POST http://127.0.0.1:11434/api/pull` // while it cannot be turned inward: `POST http://127.0.0.1:11434/api/pull`
// down this lane would reach ollama around the forward's filter, and anything // down this lane would reach ollama around the forward's filter and around the
// on the LAN around every other gate. So is any address the machine takes as // `models` switch, and this machine's other services around theirs. So on
// its own (ownaddr.go): a public one on its NIC, or a range a `local` route // every box the lane refuses this machine: `localhost` names, loopback and
// delivers here. A local box keeps the lane it had. // unspecified addresses, link-local ones (the cloud metadata address
// Resolution failures refuse too — a name this box cannot resolve is not one // included), and any address the machine takes as its own (ownaddr.go), such
// as a public one on its NIC or a range a `local` route delivers here.
//
// The network around it is refused too, on every box: the lane reaches the
// public internet and nothing else, so private, CGNAT and reserved addresses
// (the LAN, the mesh) are refused, by address and by any address a name
// resolves to. A LOCAL box used to keep its LAN reachable here; it no longer
// does (2026-10-26): the Host's egress gate holds every dial it makes to the
// same rule, and a person's own machines
// are reached through their approved lanes, never through egress. Only the
// wording differs between the two kinds of box.
//
// Resolution failures refuse too: a name this box cannot resolve is not one
// it can vouch for. // it can vouch for.
// //
// THE ADDRESS THE CHECK VETTED IS THE ADDRESS DIALLED. egressVettedAddr // THE ADDRESS THE CHECK VETTED IS THE ADDRESS DIALLED. egressVettedAddrs
// returns "ip:port" on an anchored box, and the dial takes that, never the // returns "ip:port" forms, and the dial takes those, never the name: a name
// name: a name resolved once for the check and again for the dial is a // resolved once for the check and again for the dial is a rebinding hole (a
// rebinding hole (a public answer for the check, 127.0.0.1 for the dial). // public answer for the check, 127.0.0.1 for the dial).
func egressInternalRefusal(ctx context.Context, hostport string) string { func egressInternalRefusal(ctx context.Context, hostport string) string {
_, why := egressVettedAddr(ctx, hostport) _, why := egressVettedAddrs(ctx, hostport)
return why return why
} }
// egressResolve is how the anchored lane resolves a name. A seam so a test can // egressResolve is how the lane resolves a name. A seam so a test can answer
// answer for a name without DNS. // for a name without DNS.
var egressResolve = net.DefaultResolver.LookupIPAddr var egressResolve = net.DefaultResolver.LookupIPAddr
// egressVettedAddr: (the address to dial, "") or ("", the refusal). On a // egressVettedAddr: (the first address to dial, "") or ("", the refusal).
// local box the input is returned as it came (the OS resolves at the dial,
// as before); on an anchored box every address the name resolves to is
// checked and the FIRST one is what the dial gets, as "ip:port".
func egressVettedAddr(ctx context.Context, hostport string) (string, string) { func egressVettedAddr(ctx context.Context, hostport string) (string, string) {
if currentAnchor() == nil { all, why := egressVettedAddrs(ctx, hostport)
return hostport, "" if why != "" {
return "", why
} }
return all[0], ""
}
// egressVettedAddrs: every address the name resolved to, as "ip:port" in the
// resolver's order, or (nil, the refusal). Any one refused address refuses the
// name. The dial tries them in order (dialEach), so a dual-stack site still
// answers from a machine whose IPv6 goes nowhere, as it did when the OS dialled
// the name.
func egressVettedAddrs(ctx context.Context, hostport string) ([]string, string) {
anchored := currentAnchor() != nil
host, port, err := net.SplitHostPort(hostport) host, port, err := net.SplitHostPort(hostport)
if err != nil { if err != nil {
host, port = hostport, "" host, port = hostport, ""
} }
host = strings.Trim(strings.TrimSpace(host), "[]") host = strings.Trim(strings.TrimSpace(host), "[]")
if strings.EqualFold(host, "localhost") || strings.HasSuffix(strings.ToLower(host), ".localhost") { if strings.EqualFold(host, "localhost") || strings.HasSuffix(strings.ToLower(host), ".localhost") {
return "", "refused: egress to " + host + " — this lane reaches the public internet only on an anchored machine" if anchored {
return nil, "refused: egress to " + host + " — this lane reaches the public internet only on an anchored machine"
}
return nil, "refused: egress to " + host + " — this lane never reaches this machine"
} }
var addrs []net.IP var addrs []net.IP
if ip := net.ParseIP(host); ip != nil { if ip := net.ParseIP(host); ip != nil {
@@ -193,50 +232,163 @@ func egressVettedAddr(ctx context.Context, hostport string) (string, string) {
} else { } else {
resolved, err := egressResolve(ctx, host) resolved, err := egressResolve(ctx, host)
if err != nil || len(resolved) == 0 { if err != nil || len(resolved) == 0 {
return "", "refused: egress to " + host + " — could not resolve it on this machine" return nil, "refused: egress to " + host + " — could not resolve it on this machine"
} }
for _, r := range resolved { for _, r := range resolved {
addrs = append(addrs, r.IP) addrs = append(addrs, r.IP)
} }
} }
for _, ip := range addrs { for _, ip := range addrs {
if internalIP(ip) { if anchored && internalIP(ip) {
return "", "refused: egress to " + host + " (" + ip.String() + ") — this lane reaches the public " + return nil, "refused: egress to " + host + " (" + ip.String() + ") — this lane reaches the public " +
"internet only on an anchored machine, never this machine, its network or the mesh" "internet only on an anchored machine, never this machine, its network or the mesh"
} }
if !anchored && thisMachineOrLinkIP(ip) {
return nil, "refused: egress to " + host + " (" + ip.String() + ") — this lane never reaches " +
"this machine or a link-local address"
} }
// This machine is not only loopback: a public address on its NIC, or a }
// range the kernel delivers locally through a `local` route, reaches its // This machine is not only loopback: a public address on its NIC, a LAN
// 0.0.0.0 services too (ownaddr.go). This lane has no allow list, so the // address on it, or a range the kernel delivers locally through a `local`
// refusal has no exception; a local box returned above and is unchanged. // route, reaches its 0.0.0.0 services too, and a container's or VM's
// address on a network made here reaches a service on it (ownaddr.go). This
// lane has no allow list, so the refusal has no exception on either kind of
// box.
own, err := ownDestination(addrs) own, err := ownDestination(addrs)
if err != nil { if err != nil {
return "", "refused: egress to " + host + " — this machine could not tell whether that is itself, so it cannot be vetted" return nil, "refused: egress to " + host + " — this machine could not tell whether that is itself, so it cannot be vetted"
} }
if own { if own {
return "", "refused: egress to " + host + " — that is this machine; this lane reaches the public " + if anchored {
return nil, "refused: egress to " + host + " — that is this machine; this lane reaches the public " +
"internet only on an anchored machine, never this machine, its network or the mesh" "internet only on an anchored machine, never this machine, its network or the mesh"
} }
return net.JoinHostPort(addrs[0].String(), port), "" return nil, "refused: egress to " + host + " — that is this machine; this lane never reaches this machine"
}
// The network around this machine, on a local box as on an anchored one
// (above): any one address of the name that is not the public internet
// refuses it. Checked after this machine's own addresses, so those keep
// saying "that is this machine".
if !anchored {
for _, ip := range addrs {
if internalIP(ip) {
return nil, "refused: egress to " + host + " (" + ip.String() + ") — this lane reaches the " +
"public internet only, never this machine's network or the mesh; reach your own " +
"machines through their approved lanes"
}
}
}
out := make([]string, 0, len(addrs))
for _, ip := range addrs {
out = append(out, net.JoinHostPort(ip.String(), port))
}
return out, ""
} }
// internalIP: loopback, link-local, private (RFC 1918, ULA), CGNAT, unspecified. // dialEachStagger is how long an attempt has to answer before the next
func internalIP(ip net.IP) bool { // address is tried beside it: the connection attempt delay of Happy Eyeballs
if ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || // (RFC 8305 recommends 250 ms), which the OS dialler used when it was given
ip.IsPrivate() || ip.IsUnspecified() || ip.IsMulticast() { // the name.
return true var dialEachStagger = 300 * time.Millisecond
// dialEach dials vetted addresses, in order, and returns the first that
// answers. The next address starts as soon as an attempt fails, or when the
// attempts in flight have not answered within dialEachStagger, so an address
// whose packets go nowhere costs a fraction of a second, not the dial budget.
// Only these addresses are ever dialled. The attempts still in flight when one
// answers are cancelled, and any that connects anyway is closed.
func dialEach(ctx context.Context, dialOut egressDialer, addrs []string) (net.Conn, error) {
if len(addrs) == 0 {
return nil, errors.New("no address")
}
ctx, cancel := context.WithCancel(ctx)
defer cancel()
type attempt struct {
c net.Conn
err error
}
results := make(chan attempt, len(addrs))
next, inFlight := 0, 0
start := func() {
a := addrs[next]
next++
inFlight++
go func() {
c, err := dialOut(ctx, "tcp", a)
results <- attempt{c, err}
}()
}
stagger := time.NewTimer(dialEachStagger)
defer stagger.Stop()
start()
var err error
for inFlight > 0 {
var tick <-chan time.Time
if next < len(addrs) {
tick = stagger.C
}
select {
case r := <-results:
inFlight--
if r.err == nil {
go func(n int) {
for ; n > 0; n-- {
if late := <-results; late.c != nil {
_ = late.c.Close()
}
}
}(inFlight)
return r.c, nil
}
err = r.err
if next < len(addrs) {
start()
stagger.Reset(dialEachStagger)
}
case <-tick:
start()
stagger.Reset(dialEachStagger)
}
}
return nil, err
}
// ipClass is what an address reaches, read by every lane that dials for
// someone else. The address rules live here once: internalIP is "anything but
// public", and the local egress lane refuses the classes that are this machine
// or its link.
type ipClass int
const (
ipPublic ipClass = iota // the public internet
ipThisMachine // loopback, unspecified, 0.0.0.0/8
ipLink // link-local unicast and multicast (the cloud metadata address)
ipNetwork // private (RFC 1918, ULA), CGNAT, multicast, benchmarking, reserved
ipUnreadable // a translated form whose IPv4 address this box cannot read
ipObsolete // IPv4-compatible ::/96, IPv4-translated ::ffff:0:0/96, site-local fec0::/10
)
func classifyIP(ip net.IP) ipClass {
switch {
case ip.IsLoopback(), ip.IsUnspecified():
return ipThisMachine
case ip.IsLinkLocalUnicast(), ip.IsLinkLocalMulticast():
return ipLink
case ip.IsPrivate(), ip.IsMulticast():
return ipNetwork
} }
if v4 := ip.To4(); v4 != nil { if v4 := ip.To4(); v4 != nil {
// 100.64.0.0/10 (carrier NAT) is as internal as 10/8 for this purpose. // 0.0.0.0/8 is "this host on this network".
if v4[0] == 100 && v4[1]&0xc0 == 64 { if v4[0] == 0 {
return true return ipThisMachine
} }
// 0.0.0.0/8 ("this network"), 198.18.0.0/15 (benchmarking) and // 100.64.0.0/10 (carrier NAT) is as internal as 10/8 for this purpose;
// 240.0.0.0/4 (reserved) are never a public host. // 198.18.0.0/15 (benchmarking) and 240.0.0.0/4 (reserved) are never a
if v4[0] == 0 || (v4[0] == 198 && v4[1]&0xfe == 18) || v4[0] >= 240 { // public host.
return true if (v4[0] == 100 && v4[1]&0xc0 == 64) || (v4[0] == 198 && v4[1]&0xfe == 18) || v4[0] >= 240 {
return ipNetwork
} }
return false return ipPublic
} }
// IPv6 forms that carry an IPv4 address inside them reach that address, so // IPv6 forms that carry an IPv4 address inside them reach that address, so
// they are judged by it: NAT64 64:ff9b::/96 (the last four bytes — how an // they are judged by it: NAT64 64:ff9b::/96 (the last four bytes — how an
@@ -245,15 +397,51 @@ func internalIP(ip net.IP) bool {
if len(ip) == net.IPv6len { if len(ip) == net.IPv6len {
if ip[0] == 0x00 && ip[1] == 0x64 && ip[2] == 0xff && ip[3] == 0x9b { if ip[0] == 0x00 && ip[1] == 0x64 && ip[2] == 0xff && ip[3] == 0x9b {
if ip[4]|ip[5]|ip[6]|ip[7]|ip[8]|ip[9]|ip[10]|ip[11] == 0 { if ip[4]|ip[5]|ip[6]|ip[7]|ip[8]|ip[9]|ip[10]|ip[11] == 0 {
return internalIP(net.IPv4(ip[12], ip[13], ip[14], ip[15])) return classifyIP(net.IPv4(ip[12], ip[13], ip[14], ip[15]))
} }
// The rest of 64:ff9b::/32 (local-use translators, RFC 8215) maps // The rest of 64:ff9b::/32 (local-use translators, RFC 8215) maps
// to IPv4 in ways this box cannot read: refused. // to IPv4 in ways this box cannot read.
return true return ipUnreadable
} }
if ip[0] == 0x20 && ip[1] == 0x02 { if ip[0] == 0x20 && ip[1] == 0x02 {
return internalIP(net.IPv4(ip[2], ip[3], ip[4], ip[5])) return classifyIP(net.IPv4(ip[2], ip[3], ip[4], ip[5]))
} }
// Deprecated forms that are no destination of their own: IPv4-compatible
// ::a.b.c.d (RFC 4291 2.5.5.1), IPv4-translated ::ffff:0:a.b.c.d (SIIT)
// and site-local fec0::/10 (RFC 3879). Where each is delivered depends on
// the platform, so no lane dials them. (:: and ::1 are matched above.)
if isZeros(ip[0:12]) ||
(isZeros(ip[0:8]) && ip[8] == 0xff && ip[9] == 0xff && ip[10] == 0 && ip[11] == 0) ||
(ip[0] == 0xfe && ip[1]&0xc0 == 0xc0) {
return ipObsolete
}
}
return ipPublic
}
func isZeros(b []byte) bool {
for _, c := range b {
if c != 0 {
return false
}
}
return true
}
// internalIP: loopback, link-local, private (RFC 1918, ULA), CGNAT,
// unspecified, multicast and reserved, also inside a NAT64 or 6to4 address,
// and the deprecated IPv4-compatible, IPv4-translated and site-local forms.
func internalIP(ip net.IP) bool {
return classifyIP(ip) != ipPublic
}
// thisMachineOrLinkIP: what the local egress lane refuses by address class.
// An address it cannot read is refused, since it could be either, and so is a
// deprecated form, which is no destination of its own.
func thisMachineOrLinkIP(ip net.IP) bool {
switch classifyIP(ip) {
case ipThisMachine, ipLink, ipUnreadable, ipObsolete:
return true
} }
return false return false
} }
+122
View File
@@ -0,0 +1,122 @@
package main
// dialEach, which both egress lanes dial through: an address whose packets go
// nowhere must not hold up the next one for its share of the dial budget, the
// way the OS dialler's Happy Eyeballs never did when it was given the name.
import (
"context"
"errors"
"net"
"reflect"
"sync"
"sync/atomic"
"testing"
"time"
)
type closeRecordingConn struct {
net.Conn
closed atomic.Bool
}
func (c *closeRecordingConn) Close() error {
c.closed.Store(true)
return c.Conn.Close()
}
// MUTATION: dial the addresses one after another, each with an even share of
// what is left of the budget.
// RESULT: red — the second address waits out the first one's share (2 s here).
func TestDialEachDoesNotWaitOutAnAddressThatGoesNowhere(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 4*time.Second)
defer cancel()
var mu sync.Mutex
var dialled []string
abandoned := make(chan struct{})
dial := func(ctx context.Context, _, addr string) (net.Conn, error) {
mu.Lock()
dialled = append(dialled, addr)
mu.Unlock()
if addr == "[2001:db8::30]:443" {
<-ctx.Done() // no answer and no error: the packets go nowhere
close(abandoned)
return nil, ctx.Err()
}
a, b := net.Pipe()
go func() { _ = b.Close() }()
return a, nil
}
began := time.Now()
c, err := dialEach(ctx, dial, []string{"[2001:db8::30]:443", "198.51.100.30:443"})
took := time.Since(began)
if err != nil {
t.Fatalf("the second address answers: %v", err)
}
_ = c.Close()
if took > 1500*time.Millisecond {
t.Fatalf("the second address waited %v for the first", took)
}
select {
case <-abandoned:
case <-time.After(2 * time.Second):
t.Fatal("the attempt left in flight must be cancelled once another answers")
}
mu.Lock()
defer mu.Unlock()
if want := []string{"[2001:db8::30]:443", "198.51.100.30:443"}; !reflect.DeepEqual(dialled, want) {
t.Fatalf("only the vetted addresses, in order: got %v, want %v", dialled, want)
}
}
// A connection that answers after another already won is closed, not leaked;
// an address that fails at once hands over to the next without waiting; and
// when every address fails the last error is returned.
func TestDialEachClosesALateWinnerAndReturnsTheLastError(t *testing.T) {
late := make(chan *closeRecordingConn, 1)
dial := func(ctx context.Context, _, addr string) (net.Conn, error) {
switch addr {
case "198.51.100.31:443":
time.Sleep(2 * dialEachStagger) // answers, but after the second address did
a, b := net.Pipe()
go func() { _ = b.Close() }()
rc := &closeRecordingConn{Conn: a}
late <- rc
return rc, nil
case "198.51.100.32:443":
a, b := net.Pipe()
go func() { _ = b.Close() }()
return a, nil
}
return nil, errors.New("refused: " + addr)
}
c, err := dialEach(context.Background(), dial, []string{"198.51.100.31:443", "198.51.100.32:443"})
if err != nil {
t.Fatal(err)
}
_ = c.Close()
select {
case rc := <-late:
deadline := time.Now().Add(2 * time.Second)
for !rc.closed.Load() && time.Now().Before(deadline) {
time.Sleep(10 * time.Millisecond)
}
if !rc.closed.Load() {
t.Fatal("a connection that answered after the winner must be closed")
}
case <-time.After(3 * time.Second):
t.Fatal("the slow attempt never answered")
}
began := time.Now()
_, err = dialEach(context.Background(), dial, []string{"198.51.100.33:443", "198.51.100.34:443"})
if err == nil || err.Error() != "refused: 198.51.100.34:443" {
t.Fatalf("every address failed, so the last error: %v", err)
}
if took := time.Since(began); took >= dialEachStagger {
t.Fatalf("a failed address hands over at once, not after the stagger: %v", took)
}
if _, err := dialEach(context.Background(), dial, nil); err == nil {
t.Fatal("no address is an error")
}
}
+420
View File
@@ -0,0 +1,420 @@
package main
import (
"bufio"
"errors"
"io"
"net"
"net/http/httputil"
"strconv"
"strings"
)
// ONE plain-HTTP request per egress connection, read the way the Host's egress
// gate reads it (its strict request parse and its authority parse: the same
// refusals, the same normalised host, the same rebuilt head).
//
// Everything the lane writes to a destination is what it checked, and nothing
// more:
// - the head is parsed strictly and REBUILT, so the destination reads the same
// host, port and body length the lane vetted: never a second reading of the
// Host's bytes (a Host header naming another site, a query or userinfo glued
// to the host, a padded Transfer-Encoding);
// - the request goes out with `Connection: close`, its body is framed by
// Content-Length or chunked encoding, and a byte the Host sends after the
// body closes the connection with nothing of it forwarded. Before this the
// lane wrote the first head and then spliced the whole connection, so every
// later request on a keep-alive connection reached the destination unread.
//
// An honest client sends its next request only after the response, which the
// destination closes behind `Connection: close`, so it opens a new connection
// and meets the vetting again.
// egressHostChars are the characters a host name may have here (lower-cased).
const egressHostChars = "abcdefghijklmnopqrstuvwxyz0123456789-._"
// egressTokenChars are RFC 9110 token characters: methods and header names.
const egressTokenChars = "!#$%&'*+-.^_`|~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
// egressHopByHop are headers that belong to one hop. The lane speaks for itself
// on the next one, and a proxy credential never goes on to the destination.
var egressHopByHop = map[string]bool{
"connection": true, "proxy-connection": true, "keep-alive": true, "upgrade": true,
"te": true, "trailer": true, "proxy-authorization": true, "proxy-authenticate": true,
}
// egressTrailerLimit bounds the trailer section of a chunked body, which is
// read to its end and dropped (the gate's limit).
const egressTrailerLimit = 8192
// egressRefusal is a request the lane refuses before it dials anything.
type egressRefusal struct{ status, body string }
func egressBad(body string) *egressRefusal {
return &egressRefusal{"400 Bad Request", body}
}
const egressNoHost = "The bastion could not read where this request is going (no host in the " +
"request line or in a Host header), so it sent nothing."
// egressUnreachable is the body of a 502: the destination passed the vetting,
// and no address of it answered from this machine.
const egressUnreachable = "The bastion could not reach that destination from this machine's own " +
"connection, so nothing was sent."
// egressHostPort is host and port as the vetting takes them.
func egressHostPort(host string, port int) string {
return net.JoinHostPort(host, strconv.Itoa(port))
}
// writeEgressRefusal answers the Host with a status and a short plain-text
// body saying why, and closes the exchange. why, when set, also goes in
// X-Privatae-Refused, the header the Host already reads.
func writeEgressRefusal(w io.Writer, status, body, why string) {
body += "\n"
var b strings.Builder
b.WriteString("HTTP/1.1 " + status + "\r\n")
b.WriteString("Content-Type: text/plain; charset=utf-8\r\n")
b.WriteString("Content-Length: " + strconv.Itoa(len(body)) + "\r\n")
b.WriteString("Connection: close\r\n")
if why != "" {
b.WriteString("X-Privatae-Refused: " + strings.NewReplacer("\r", " ", "\n", " ").Replace(why) + "\r\n")
}
b.WriteString("\r\n")
b.WriteString(body)
_, _ = io.WriteString(w, b.String())
}
// parseEgressAuthority: (host, port, true) from "host", "host:port", "[v6]" or
// "[v6]:port", or false when the lane can't read it exactly as the destination
// would. Refused rather than guessed: userinfo ("a@b"), an unbracketed IPv6
// literal, characters no host name has, a non-numeric or out-of-range port.
// The host is lower-cased and loses one trailing dot.
func parseEgressAuthority(s string, defaultPort int) (string, int, bool) {
s = strings.TrimSpace(s)
if s == "" || strings.Contains(s, "@") {
return "", 0, false
}
var host string
port := defaultPort
digits := func(p string) bool {
if len(p) == 0 || len(p) > 5 {
return false
}
for _, c := range p {
if c < '0' || c > '9' {
return false
}
}
return true
}
if strings.HasPrefix(s, "[") {
end := strings.IndexByte(s, ']')
if end < 0 {
return "", 0, false
}
// An IPv6 literal, IPv4-mapped forms included (the vetting judges the
// address they map to); a bare IPv4 address in brackets or a zone is
// not one.
inner := s[1:end]
ip := net.ParseIP(inner)
if ip == nil || !strings.Contains(inner, ":") || strings.Contains(inner, "%") {
return "", 0, false
}
host = ip.String()
rest := s[end+1:]
switch {
case rest == "":
case strings.HasPrefix(rest, ":") && digits(rest[1:]):
port, _ = strconv.Atoi(rest[1:])
default:
return "", 0, false
}
} else {
if strings.Count(s, ":") > 1 {
return "", 0, false
}
h, p, hasPort := strings.Cut(s, ":")
if hasPort {
if !digits(p) {
return "", 0, false
}
port, _ = strconv.Atoi(p)
}
h = strings.ToLower(h)
h = strings.TrimSuffix(h, ".")
if h == "" || strings.HasPrefix(h, ".") || strings.Contains(h, "..") {
return "", 0, false
}
for _, c := range h {
if !strings.ContainsRune(egressHostChars, c) {
return "", 0, false
}
}
host = h
}
if port <= 0 || port >= 65536 {
return "", 0, false
}
return host, port, true
}
// fmtEgressAuthority writes host[:port] back, the port left out when it is the
// scheme's default.
func fmtEgressAuthority(host string, port, defaultPort int) string {
h := host
if strings.Contains(host, ":") {
h = "[" + host + "]"
}
if port == defaultPort {
return h
}
return h + ":" + strconv.Itoa(port)
}
// egressRequest is the one plain-HTTP request the lane forwards.
type egressRequest struct {
host string
port int
head []byte // rebuilt, CRLF, ends at its blank line
chunked bool
length int64
}
func egressTokenOK(s string) bool {
if s == "" {
return false
}
for _, c := range s {
if !strings.ContainsRune(egressTokenChars, c) {
return false
}
}
return true
}
// parseEgressRequest reads the one request in head (the request line and
// headers through their blank line, as handleEgressConn read them) and
// rebuilds it, or refuses it. CONNECT never comes here.
func parseEgressRequest(head string) (egressRequest, *egressRefusal) {
lines := strings.Split(head, "\n")
// The read stops at the first blank line, so the head ends "\n", blank, "\n":
// drop the empty tail after the last "\n" and the blank line itself.
if len(lines) < 3 {
return egressRequest{}, egressBad("The request ended before its headers did, so the bastion sent nothing.")
}
lines = lines[:len(lines)-2]
for i, ln := range lines {
lines[i] = strings.TrimSuffix(ln, "\r")
if strings.ContainsAny(lines[i], "\r\x00") {
return egressRequest{}, egressBad("The request's headers carry a stray carriage return or NUL, so the bastion sent nothing.")
}
}
parts := strings.Split(lines[0], " ")
if len(parts) != 3 || parts[0] == "" || parts[1] == "" || parts[2] == "" || !egressTokenOK(parts[0]) {
return egressRequest{}, egressBad("The request line is not 'METHOD target HTTP/1.x', so the bastion sent nothing.")
}
method, rawTarget, version := strings.ToUpper(parts[0]), parts[1], parts[2]
if version != "HTTP/1.1" && version != "HTTP/1.0" {
return egressRequest{}, egressBad("The bastion forwards HTTP/1.0 and HTTP/1.1 only, so it sent nothing.")
}
type header struct{ name, value string }
var headers []header
for _, ln := range lines[1:] {
if strings.HasPrefix(ln, " ") || strings.HasPrefix(ln, "\t") {
return egressRequest{}, egressBad("A header is folded onto a second line (obsolete line folding), so the bastion sent nothing.")
}
name, value, ok := strings.Cut(ln, ":")
if !ok || !egressTokenOK(name) {
return egressRequest{}, egressBad("A header line is not 'Name: value' (no space may come before the colon), so the bastion sent nothing.")
}
headers = append(headers, header{name, strings.Trim(value, " \t")})
}
values := func(n string) []string {
var out []string
for _, h := range headers {
if strings.EqualFold(h.name, n) {
out = append(out, h.value)
}
}
return out
}
hosts := values("host")
if len(hosts) > 1 {
return egressRequest{}, egressBad("The request carries more than one Host header, so the bastion sent nothing.")
}
var host, target, authority string
var port int
if scheme, after, abs := strings.Cut(rawTarget, "://"); abs {
if strings.ToLower(scheme) != "http" {
// https:// in absolute form asks the lane to make the TLS connection,
// which it does not do: it used to send the request in plain text to
// port 80. Every client sends https as a CONNECT.
return egressRequest{}, egressBad("The bastion forwards plain http:// requests only; send https as a CONNECT. Nothing was sent.")
}
cut := len(after)
for _, sep := range []string{"/", "?", "#"} {
if i := strings.Index(after, sep); i >= 0 && i < cut {
cut = i
}
}
var ok bool
if host, port, ok = parseEgressAuthority(after[:cut], 80); !ok {
return egressRequest{}, egressBad(egressNoHost)
}
path, _, _ := strings.Cut(after[cut:], "#")
if !strings.HasPrefix(path, "/") {
path = "/" + path
}
authority = fmtEgressAuthority(host, port, 80)
target = "http://" + authority + path
} else {
if !strings.HasPrefix(rawTarget, "/") && rawTarget != "*" {
return egressRequest{}, egressBad(egressNoHost)
}
hostHdr := ""
if len(hosts) == 1 {
hostHdr = hosts[0]
}
var ok bool
if host, port, ok = parseEgressAuthority(hostHdr, 80); !ok {
return egressRequest{}, egressBad(egressNoHost)
}
authority = fmtEgressAuthority(host, port, 80)
target = rawTarget
}
// Body framing (RFC 9112 section 6). Every reading two parsers could disagree
// on is refused: both length headers, an unknown or doubled coding, a coding
// on HTTP/1.0, Content-Length values that differ.
req := egressRequest{host: host, port: port}
te, cl := values("transfer-encoding"), values("content-length")
if len(te) > 0 {
var codings []string
for _, v := range te {
for _, c := range strings.Split(v, ",") {
codings = append(codings, strings.ToLower(strings.TrimSpace(c)))
}
}
if len(cl) > 0 || version == "HTTP/1.0" || len(codings) != 1 || codings[0] != "chunked" {
return egressRequest{}, egressBad("The request's body length is ambiguous (Transfer-Encoding with " +
"Content-Length, on HTTP/1.0, or a coding other than chunked), so the bastion sent nothing.")
}
req.chunked = true
} else if len(cl) > 0 {
seen := map[string]bool{}
for _, v := range cl {
for _, c := range strings.Split(v, ",") {
seen[strings.TrimSpace(c)] = true
}
}
var one string
for k := range seen {
one = k
}
n, err := strconv.ParseInt(one, 10, 64)
if len(seen) != 1 || one == "" || len(one) > 18 || err != nil || n < 0 || strings.TrimLeft(one, "0123456789") != "" {
return egressRequest{}, egressBad("The request's Content-Length is not one number, so the bastion sent nothing.")
}
req.length = n
}
named := map[string]bool{}
for _, v := range values("connection") {
for _, t := range strings.Split(v, ",") {
named[strings.ToLower(strings.TrimSpace(t))] = true
}
}
var b strings.Builder
b.WriteString(method + " " + target + " " + version + "\r\n")
b.WriteString("Host: " + authority + "\r\n")
for _, h := range headers {
lk := strings.ToLower(h.name)
if egressHopByHop[lk] || named[lk] || lk == "host" || lk == "content-length" || lk == "transfer-encoding" {
continue
}
b.WriteString(h.name + ": " + h.value + "\r\n")
}
if req.chunked {
b.WriteString("Transfer-Encoding: chunked\r\n")
} else if len(cl) > 0 {
b.WriteString("Content-Length: " + strconv.FormatInt(req.length, 10) + "\r\n")
}
b.WriteString("Connection: close\r\n\r\n")
req.head = []byte(b.String())
return req, nil
}
// forwardEgressBody copies the one request body from src to dst, framed as the
// rebuilt head says. A chunked body goes out re-encoded (plain sizes, no
// extensions, trailers read and dropped), so the destination parses exactly the
// body this parser found.
func forwardEgressBody(dst io.Writer, src *bufio.Reader, req egressRequest) error {
if !req.chunked {
if req.length == 0 {
return nil
}
_, err := io.CopyN(dst, src, req.length)
return err
}
cw := httputil.NewChunkedWriter(dst)
if _, err := io.Copy(cw, httputil.NewChunkedReader(src)); err != nil {
return err
}
// The trailer section, read to its blank line within a bound, and dropped.
read := 0
for {
line, err := src.ReadSlice('\n')
read += len(line)
if err != nil || read > egressTrailerLimit {
return errors.New("chunked body: bad trailer section")
}
if string(line) == "\r\n" || string(line) == "\n" {
break
}
}
if err := cw.Close(); err != nil {
return err
}
_, err := io.WriteString(dst, "\r\n")
return err
}
// oneEgressExchange writes the one rebuilt request and its body to upstream,
// and carries the destination's answer back until the destination closes.
// Nothing the Host sends after the body is forwarded: its first byte closes
// both sides.
func oneEgressExchange(clientReader *bufio.Reader, client, upstream net.Conn, req egressRequest) {
answered := make(chan struct{})
go func() {
_, _ = io.Copy(client, upstream)
close(answered)
_ = client.Close()
}()
defer func() {
_ = upstream.Close()
<-answered
}()
if _, err := upstream.Write(req.head); err != nil {
return
}
if err := forwardEgressBody(upstream, clientReader, req); err != nil {
return
}
// The request is complete. A further byte is a second request on this
// connection: never forwarded, and the connection closes. End of stream
// from the Host is passed on, as the splice did.
go func() {
var one [1]byte
if n, _ := clientReader.Read(one[:]); n > 0 {
_ = upstream.Close()
_ = client.Close()
return
}
if cw, ok := upstream.(interface{ CloseWrite() error }); ok {
_ = cw.CloseWrite()
}
}()
<-answered
}
+229
View File
@@ -0,0 +1,229 @@
//go:build !windows
// Tagged !windows: it borrows setupBox (anchor_test.go) and stubOwnAddrs
// (ownaddr_test.go).
package main
// The egress lane on a local (unanchored) box. It refuses this machine and
// its link-local addresses, so a host cannot reach this machine's services
// (ollama above all) around the switches that govern them, and, since
// 2026-10-26, the network around it as well: the LAN and the mesh
// (egress_network_test.go). An anchored box is unchanged: the public internet
// only.
import (
"bufio"
"context"
"errors"
"fmt"
"io"
"net"
"reflect"
"strings"
"sync/atomic"
"testing"
"time"
)
// answerEgressNames answers the egress lane's name lookups from m.
func answerEgressNames(t *testing.T, m map[string][]string) {
t.Helper()
old := egressResolve
t.Cleanup(func() { egressResolve = old })
egressResolve = func(_ context.Context, host string) ([]net.IPAddr, error) {
ips, ok := m[strings.ToLower(host)]
if !ok {
return nil, errors.New("no such host")
}
var out []net.IPAddr
for _, s := range ips {
out = append(out, net.IPAddr{IP: net.ParseIP(s)})
}
return out, nil
}
}
// A machine with a LAN address of its own and an AnyIP range, and names that
// answer with this machine, with the LAN and with the public internet.
func localEgressFixture(t *testing.T) {
t.Helper()
t.Setenv("XDG_CONFIG_HOME", t.TempDir())
if currentAnchor() != nil || boxMode() != modeLocal {
t.Fatal("the fixture must be a local, unanchored box")
}
stubOwnAddrs(t, []string{"127.0.0.1/8", "::1/128", "192.168.50.7/24", "fe80::7/64"}, localRange("203.0.113.0/24"))
answerEgressNames(t, map[string][]string{
"ollama.example.test": {"127.0.0.1"},
"mixed.example.test": {"198.51.100.8", "127.0.0.1"},
"self.example.test": {"192.168.50.7"},
"nas.example.test": {"192.168.50.30"},
"dual.example.test": {"2001:db8::8", "198.51.100.8"},
"site.example.test": {"198.51.100.9"},
})
}
// MUTATION: restore the early return for an unanchored box in egressVettedAddrs.
// RESULT: red — 127.0.0.1:11434 is handed to the dial, and a name is dialled
// as a name instead of the address that was vetted.
func TestLocalEgressRefusesThisMachineAndTheLAN(t *testing.T) {
localEgressFixture(t)
ctx := context.Background()
for _, hp := range []string{
"127.0.0.1:11434", "127.9.9.9:80", "localhost:11434", "ollama.localhost:80", "[::1]:11434",
"0.0.0.0:11434", "0.1.2.3:80", "[::]:80",
"[::ffff:127.0.0.1]:11434", "[64:ff9b::7f00:1]:11434", "[2002:7f00:1::]:80", "[64:ff9b:1::5]:80",
"169.254.169.254:80", "[fe80::1]:80",
"192.168.50.7:11434", // this machine's own LAN address
"203.0.113.5:80", // a range a `local` route delivers here
"ollama.example.test:11434", "mixed.example.test:443", "self.example.test:11434",
"unknown.example.test:443",
// the network around this machine (egress_network_test.go has the rest)
"192.168.50.20:8080", "10.1.2.3:443", "172.16.4.4:22", "[fd12:3456::9]:80", "100.64.0.9:80",
"nas.example.test:443",
} {
if addr, why := egressVettedAddr(ctx, hp); why == "" || addr != "" {
t.Errorf("%s must be refused on a local box, got dial %q", hp, addr)
}
}
for hp, want := range map[string][]string{
"198.51.100.8:443": {"198.51.100.8:443"},
"site.example.test:443": {"198.51.100.9:443"},
"dual.example.test:443": {"[2001:db8::8]:443", "198.51.100.8:443"},
} {
got, why := egressVettedAddrs(ctx, hp)
if why != "" || !reflect.DeepEqual(got, want) {
t.Errorf("%s is the internet, so a local box dials %v: got %v %q", hp, want, got, why)
}
}
}
// The anchored lane keeps its own wording and still refuses the LAN that a
// local box reaches.
func TestAnchoredEgressStillRefusesTheLAN(t *testing.T) {
setupBox(t)
stubOwnAddrs(t, []string{"127.0.0.1/8", "::1/128", "192.168.50.7/24"}, localRange("203.0.113.0/24"))
ctx := context.Background()
for _, hp := range []string{"192.168.50.20:8080", "10.1.2.3:443", "[fd12:3456::9]:80", "100.64.0.9:80", "127.0.0.1:11434"} {
if addr, why := egressVettedAddr(ctx, hp); addr != "" || !strings.Contains(why, "only on an anchored machine") {
t.Errorf("%s must be refused on an anchored box as before, got %q %q", hp, addr, why)
}
}
if addr, why := egressVettedAddr(ctx, "198.51.100.8:443"); why != "" || addr != "198.51.100.8:443" {
t.Fatalf("the public internet passes: %q %q", addr, why)
}
}
// Through the lane itself, with a real dialer: a request for this machine is
// answered 403 and never reaches the listener; a public name is dialled at
// its vetted address; a dual-stack name falls back to its second address.
func TestLocalEgressLaneNeverDialsThisMachine(t *testing.T) {
localEgressFixture(t)
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer ln.Close()
var hits atomic.Int32
go func() {
for {
c, err := ln.Accept()
if err != nil {
return
}
hits.Add(1)
c.Close()
}
}()
ask := func(dial egressDialer, req string) string {
srv, cli := net.Pipe()
go handleEgressConn(srv, dial, "home.host.example")
_ = cli.SetDeadline(time.Now().Add(5 * time.Second))
_, _ = io.WriteString(cli, req)
line, _ := bufio.NewReader(cli).ReadString('\n')
cli.Close()
return line
}
netDial := (&net.Dialer{Timeout: 2 * time.Second}).DialContext
target := ln.Addr().String()
for _, req := range []string{
"CONNECT " + target + " HTTP/1.1\r\n\r\n",
"GET http://" + target + "/api/tags HTTP/1.1\r\nHost: " + target + "\r\n\r\n",
"CONNECT localhost:" + strings.Split(target, ":")[1] + " HTTP/1.1\r\n\r\n",
} {
if line := ask(netDial, req); !strings.HasPrefix(line, "HTTP/1.1 403") {
t.Errorf("a local box must refuse %q, answered %q", req, line)
}
}
time.Sleep(50 * time.Millisecond)
if n := hits.Load(); n != 0 {
t.Fatalf("the lane reached this machine's listener %d times", n)
}
var asked []string
recording := func(_ context.Context, _, addr string) (net.Conn, error) {
asked = append(asked, addr)
return nil, errors.New("stop here")
}
if line := ask(recording, "CONNECT site.example.test:443 HTTP/1.1\r\n\r\n"); !strings.HasPrefix(line, "HTTP/1.1 502") {
t.Fatalf("a public destination is dialled (and this dial fails): %q", line)
}
if !reflect.DeepEqual(asked, []string{"198.51.100.9:443"}) {
t.Fatalf("the vetted address, never the name: %v", asked)
}
asked = nil
fallback := func(_ context.Context, _, addr string) (net.Conn, error) {
asked = append(asked, addr)
if strings.HasPrefix(addr, "[") {
return nil, errors.New("network is unreachable")
}
a, b := net.Pipe()
go func() { _, _ = io.Copy(b, b) }()
return a, nil
}
if line := ask(fallback, "CONNECT dual.example.test:443 HTTP/1.1\r\n\r\n"); !strings.HasPrefix(line, "HTTP/1.1 200") {
t.Fatalf("a dual-stack name answers from its second address: %q", line)
}
if want := []string{"[2001:db8::8]:443", "198.51.100.8:443"}; !reflect.DeepEqual(asked, want) {
t.Fatalf("each vetted address in turn: got %v, want %v", asked, want)
}
}
// What docs/models.md tells a person with a local box to do: switch `models`
// off for the host. The forward then filters store-changing requests, and the
// egress lane is no way around it, since it never reaches this machine's
// ollama, on loopback or on the machine's own LAN address.
func TestLocalModelsSwitchLeavesNoEgressRouteToOllama(t *testing.T) {
localEgressFixture(t)
const host = "home.host.example"
writePairingState(t, host, pairingActive)
if inferenceFilterNeeded(host, laneOllama, pairingModelScope(host)) {
t.Fatal("an unscoped local pairing is a raw splice before the switch, so this test would prove nothing")
}
if err := localDenySet(host, "models", true); err != nil {
t.Fatal(err)
}
if !inferenceFilterNeeded(host, laneOllama, pairingModelScope(host)) {
t.Fatal("with `models` off the ollama forward must filter /api/pull")
}
var dialled []string
dial := func(_ context.Context, _, addr string) (net.Conn, error) {
dialled = append(dialled, addr)
return nil, errors.New("stop here")
}
for _, dest := range []string{"127.0.0.1:11434", "localhost:11434", "192.168.50.7:11434", "[::1]:11434"} {
srv, cli := net.Pipe()
go handleEgressConn(srv, dial, host)
_ = cli.SetDeadline(time.Now().Add(5 * time.Second))
fmt.Fprintf(cli, "POST http://%s/api/pull HTTP/1.1\r\nHost: %s\r\n\r\n", dest, dest)
line, _ := bufio.NewReader(cli).ReadString('\n')
cli.Close()
if !strings.HasPrefix(line, "HTTP/1.1 403") {
t.Errorf("egress to %s must be refused: %q", dest, line)
}
}
if len(dialled) != 0 {
t.Fatalf("nothing for this machine may be dialled: %v", dialled)
}
}
+76
View File
@@ -0,0 +1,76 @@
//go:build !windows
// Tagged !windows: it borrows localEgressFixture and answerEgressNames
// (egress_local_test.go).
package main
// The egress lane on a local (unanchored) box refuses the network around the
// machine as well as the machine itself: the LAN, the mesh and every other
// address that is not the public internet, by address and by any address a
// name resolves to. The Host's egress gate holds every dial to the same rule;
// a person's own machines are reached through their approved lanes.
import (
"bufio"
"context"
"errors"
"io"
"net"
"reflect"
"strings"
"testing"
"time"
)
// MUTATION: drop the local-box internalIP loop at the end of egressVettedAddrs.
// RESULT: red — 192.168.50.20, nas.example.test and the rest are handed to the
// dial, and the lane dials 192.168.50.30.
func TestLocalEgressRefusesTheLANAndTheMesh(t *testing.T) {
localEgressFixture(t)
answerEgressNames(t, map[string][]string{
"nas.example.test": {"192.168.50.30"},
"half.example.test": {"198.51.100.8", "10.1.2.3"},
"mesh.example.test": {"fd12:3456::9"},
"nat.example.test": {"64:ff9b::a01:203"}, // NAT64 for 10.1.2.3
"pub.example.test": {"198.51.100.8"},
})
ctx := context.Background()
for _, hp := range []string{
"192.168.50.20:8080", "10.1.2.3:443", "172.16.4.4:22", "[fd12:3456::9]:80", "100.64.0.9:80",
"198.18.0.1:80", "[64:ff9b::a01:203]:443",
"nas.example.test:443", "half.example.test:443", "mesh.example.test:80", "nat.example.test:443",
} {
if addrs, why := egressVettedAddrs(ctx, hp); why == "" || addrs != nil || !strings.Contains(why, "approved lanes") {
t.Errorf("%s is not the public internet and must be refused on a local box, got dial %v %q", hp, addrs, why)
}
}
if got, why := egressVettedAddrs(ctx, "pub.example.test:443"); why != "" || !reflect.DeepEqual(got, []string{"198.51.100.8:443"}) {
t.Fatalf("the public internet passes at its vetted address: %v %q", got, why)
}
// Through the lane itself: 403, and nothing is dialled.
var asked []string
recording := func(_ context.Context, _, addr string) (net.Conn, error) {
asked = append(asked, addr)
return nil, errors.New("stop here")
}
for _, req := range []string{
"CONNECT nas.example.test:443 HTTP/1.1\r\n\r\n",
"GET http://10.1.2.3/ HTTP/1.1\r\nHost: 10.1.2.3\r\n\r\n",
"CONNECT half.example.test:443 HTTP/1.1\r\n\r\n",
} {
srv, cli := net.Pipe()
go handleEgressConn(srv, recording, "home.host.example")
_ = cli.SetDeadline(time.Now().Add(5 * time.Second))
_, _ = io.WriteString(cli, req)
line, _ := bufio.NewReader(cli).ReadString('\n')
cli.Close()
if !strings.HasPrefix(line, "HTTP/1.1 403") {
t.Errorf("a local box must refuse %q, answered %q", req, line)
}
}
if len(asked) != 0 {
t.Fatalf("nothing on the network around this machine is dialled: %v", asked)
}
}
+80
View File
@@ -0,0 +1,80 @@
//go:build !windows
// Tagged !windows: it borrows setupBox (anchor_test.go) and stubOwnAddrs
// (ownaddr_test.go).
package main
// The deprecated IPv6 forms are no destination of their own, and where each
// is delivered depends on the platform, so every lane refuses them: the
// anchored egress lane, the local one, and a coder run's proxy, which refused
// them first.
import (
"context"
"testing"
)
var obsoleteForms = []string{
"[::c633:6408]:443", // IPv4-compatible ::198.51.100.8
"[::7f00:1]:11434", // IPv4-compatible ::127.0.0.1
"[::c0a8:3214]:80", // IPv4-compatible ::192.168.50.20, a LAN address
"[::ffff:0:c633:6408]:443", // IPv4-translated ::ffff:0:198.51.100.8
"[::ffff:0:7f00:1]:80", // IPv4-translated ::ffff:0:127.0.0.1
"[fec0::5]:80", // site-local
"[feff:1::9]:443", // the top of fec0::/10
"compat.example.test:443", // a name with one such answer among public ones
"sitelocal.example.test:80",
}
func answerObsoleteNames(t *testing.T) {
answerEgressNames(t, map[string][]string{
"compat.example.test": {"198.51.100.8", "::c633:6408"},
"sitelocal.example.test": {"fec0::5"},
"public.example.test": {"198.51.100.8", "2001:db8::8"},
})
}
// MUTATION: classify the three forms as public in classifyIP.
// RESULT: red — both egress lanes hand them to the dial.
func TestEveryLaneRefusesTheDeprecatedIPv6Forms(t *testing.T) {
ctx := context.Background()
refusedOn := func(t *testing.T, lane string) {
t.Helper()
for _, hp := range obsoleteForms {
if addrs, why := egressVettedAddrs(ctx, hp); why == "" || addrs != nil {
t.Errorf("%s must be refused on the %s egress lane, got dial %v", hp, lane, addrs)
}
}
for _, hp := range []string{"198.51.100.8:443", "public.example.test:443"} {
if _, why := egressVettedAddrs(ctx, hp); why != "" {
t.Errorf("%s is the public internet and passes the %s egress lane: %s", hp, lane, why)
}
}
}
t.Run("anchored", func(t *testing.T) {
setupBox(t)
stubOwnAddrs(t, loopbackOnly, localRange())
answerObsoleteNames(t)
refusedOn(t, "anchored")
})
t.Run("local", func(t *testing.T) {
localEgressFixture(t)
answerObsoleteNames(t)
refusedOn(t, "local")
})
t.Run("a coder run's proxy agrees", func(t *testing.T) {
stubOwnAddrs(t, loopbackOnly, localRange())
for _, hp := range obsoleteForms[:7] {
if addrs, why := coderVettedAddrs(ctx, hp, nil); why == "" || addrs != nil {
t.Errorf("%s must be refused on a coder run's proxy, got dial %v", hp, addrs)
}
}
if _, why := coderVettedAddrs(ctx, "198.51.100.8:443", nil); why != "" {
t.Errorf("the public internet passes a coder run's proxy: %s", why)
}
})
}
+393
View File
@@ -0,0 +1,393 @@
//go:build !windows
package main
import (
"bufio"
"bytes"
"context"
"errors"
"fmt"
"io"
"net"
"net/http"
"strings"
"sync"
"testing"
"time"
)
// The bastion's egress lanes carry ONE plain-HTTP request per connection, read
// the way the Host's egress gate reads it, and every refusal says
// why in its body.
//
// MUTATIONS, each red:
// - handleEgressConn writes head.String() to upstream and splices, as before:
// TestEgressPlainHTTPCarriesOneRequestPerConnection, ...SendsTheHostItChecked;
// - oneEgressExchange's watcher forwards instead of closing: ...OneRequest...;
// - parseEgressAuthority accepts "@" or cuts the authority at "/" only:
// TestEgressPlainHTTPReadsTheHostAsTheGateDoes;
// - writeEgressRefusal writes Content-Length: 0: TestEgressRefusalsCarryABody.
// recordingOrigin stands in for a destination that keeps a connection open
// after answering (it ignores Connection: close, as a careless server may),
// and records every byte one connection carried to it.
type recordingOrigin struct {
ln net.Listener
conns chan string
// silent: never answer, so the lane never holds an unread answer when it
// closes (a close with unread data is a reset, which can cost the origin
// bytes it has not read yet).
silent bool
}
func newSilentOrigin(t *testing.T) *recordingOrigin {
o := newRecordingOrigin(t)
o.silent = true
return o
}
func newRecordingOrigin(t *testing.T) *recordingOrigin {
t.Helper()
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
o := &recordingOrigin{ln: ln, conns: make(chan string, 16)}
t.Cleanup(func() { _ = ln.Close() })
go func() {
for {
c, err := ln.Accept()
if err != nil {
return
}
go o.serve(c)
}
}()
return o
}
func (o *recordingOrigin) serve(c net.Conn) {
defer c.Close()
var mu sync.Mutex
var got bytes.Buffer
answered := false
buf := make([]byte, 4096)
for {
_ = c.SetReadDeadline(time.Now().Add(800 * time.Millisecond))
n, err := c.Read(buf)
mu.Lock()
got.Write(buf[:n])
if !o.silent && !answered && bytes.Contains(got.Bytes(), []byte("\r\n\r\n")) {
answered = true
_, _ = io.WriteString(c, "HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
}
mu.Unlock()
if err != nil {
break
}
}
o.conns <- got.String()
}
// next is everything the origin's next connection carried, once it ended.
func (o *recordingOrigin) next(t *testing.T) string {
t.Helper()
s, ok := o.nextWithin(8 * time.Second)
if !ok {
t.Fatal("the destination never saw its connection end")
}
return s
}
func (o *recordingOrigin) nextWithin(d time.Duration) (string, bool) {
select {
case s := <-o.conns:
return s, true
case <-time.After(d):
return "", false
}
}
func (o *recordingOrigin) addr() string { return o.ln.Addr().String() }
// A keep-alive connection carries one request to the destination, never two:
// neither a request sent after the first answer (a client reusing the
// connection) nor one pipelined behind the first.
func TestEgressPlainHTTPCarriesOneRequestPerConnection(t *testing.T) {
first := "GET http://198.51.100.30/first HTTP/1.1\r\nHost: 198.51.100.30\r\n\r\n"
second := "GET http://198.51.100.30/second HTTP/1.1\r\nHost: 198.51.100.30\r\n\r\n"
t.Run("reused after the answer", func(t *testing.T) {
o := newRecordingOrigin(t)
var asked []string
cli, cleanup := dialHandler(t, redirectDial(o.addr(), &asked))
defer cleanup()
_, _ = io.WriteString(cli, first)
br := bufio.NewReader(cli)
resp, err := http.ReadResponse(br, nil)
if err != nil {
t.Fatalf("the first request got no answer: %v", err)
}
body, _ := io.ReadAll(resp.Body)
if string(body) != "ok" {
t.Fatalf("the first answer was %q", body)
}
_, _ = io.WriteString(cli, second)
got := o.next(t)
if strings.Contains(got, "/second") {
t.Fatalf("a second request on one connection reached the destination unvetted:\n%s", got)
}
if !strings.Contains(got, "\r\nConnection: close\r\n") {
t.Fatalf("the request went out without Connection: close:\n%s", got)
}
_ = cli.SetReadDeadline(time.Now().Add(3 * time.Second))
if rest, _ := io.ReadAll(br); len(rest) != 0 {
t.Fatalf("the lane answered the second request: %q", rest)
}
if len(asked) != 1 {
t.Fatalf("one connection, one dial: %v", asked)
}
})
t.Run("pipelined", func(t *testing.T) {
o := newRecordingOrigin(t)
var asked []string
cli, cleanup := dialHandler(t, redirectDial(o.addr(), &asked))
defer cleanup()
_, _ = io.WriteString(cli, first+second)
got := o.next(t)
if !strings.Contains(got, "GET http://198.51.100.30/first HTTP/1.1\r\n") {
t.Fatalf("the first request did not reach the destination:\n%s", got)
}
if strings.Contains(got, "/second") {
t.Fatalf("a pipelined second request reached the destination unvetted:\n%s", got)
}
})
}
// The destination reads the host the lane vetted, in the one Host header; hop
// headers, those Connection names and a proxy credential stay on this hop.
func TestEgressPlainHTTPSendsTheHostItChecked(t *testing.T) {
o := newRecordingOrigin(t)
var asked []string
cli, cleanup := dialHandler(t, redirectDial(o.addr(), &asked))
defer cleanup()
_, _ = io.WriteString(cli, "GET http://198.51.100.31/p?q=1 HTTP/1.1\r\n"+
"Host: inside.example.test\r\nProxy-Authorization: Basic eDp5\r\n"+
"Connection: keep-alive, X-Hop\r\nX-Hop: 1\r\nX-Keep: 2\r\n\r\n")
got := o.next(t)
head, _, _ := strings.Cut(got, "\r\n\r\n")
want := "GET http://198.51.100.31/p?q=1 HTTP/1.1\r\nHost: 198.51.100.31\r\nX-Keep: 2\r\nConnection: close"
if head != want {
t.Fatalf("the destination got\n%q\nwant\n%q", head, want)
}
if len(asked) != 1 || asked[0] != "198.51.100.31:80" {
t.Fatalf("the lane must dial the address it vetted: %v", asked)
}
}
// stubEgressNames answers names without DNS and records each name exactly as
// the lane asked for it.
func stubEgressNames(t *testing.T, m map[string]string) *[]string {
t.Helper()
var mu sync.Mutex
var asked []string
old := egressResolve
t.Cleanup(func() { egressResolve = old })
egressResolve = func(_ context.Context, host string) ([]net.IPAddr, error) {
mu.Lock()
asked = append(asked, host)
mu.Unlock()
ip, ok := m[host]
if !ok {
return nil, errors.New("no such host")
}
return []net.IPAddr{{IP: net.ParseIP(ip)}}, nil
}
return &asked
}
// The host is read as the gate reads it: the authority ends at the first "/",
// "?" or "#", is lower-cased and loses one trailing dot; userinfo, an
// unbracketed IPv6 literal, a bad port, https in absolute form, two Host
// headers and an ambiguous body length are refused with 400 and a body, before
// anything is resolved or dialled.
func TestEgressPlainHTTPReadsTheHostAsTheGateDoes(t *testing.T) {
for _, c := range []struct {
req, wantName, wantLine string
}{
{"GET http://public.example.test?@10.0.0.1/ HTTP/1.1\r\n\r\n", "public.example.test",
"GET http://public.example.test/?@10.0.0.1/ HTTP/1.1"},
{"GET http://public.example.test#@10.0.0.1/ HTTP/1.1\r\n\r\n", "public.example.test",
"GET http://public.example.test/ HTTP/1.1"},
{"GET http://Public.Example.Test./x HTTP/1.1\r\n\r\n", "public.example.test",
"GET http://public.example.test/x HTTP/1.1"},
{"GET /x HTTP/1.1\r\nHost: Public.Example.Test:8080\r\n\r\n", "public.example.test",
"GET /x HTTP/1.1"},
} {
o := newRecordingOrigin(t)
names := stubEgressNames(t, map[string]string{"public.example.test": "198.51.100.40"})
var asked []string
cli, cleanup := dialHandler(t, redirectDial(o.addr(), &asked))
_, _ = io.WriteString(cli, c.req)
got, _ := o.nextWithin(4 * time.Second)
cleanup()
if len(*names) != 1 || (*names)[0] != c.wantName {
t.Errorf("%q: the lane resolved %q; want only %q", c.req, *names, c.wantName)
}
if line, _, _ := strings.Cut(got, "\r\n"); line != c.wantLine {
t.Errorf("%q: the destination got the request line %q; want %q", c.req, line, c.wantLine)
}
}
for _, req := range []string{
"GET http://user:pw@public.example.test/ HTTP/1.1\r\n\r\n",
"GET http://198.51.100.41:80@public.example.test/ HTTP/1.1\r\n\r\n",
"GET http://2001:db8::1/ HTTP/1.1\r\n\r\n",
"GET http://public.example.test:80x/ HTTP/1.1\r\n\r\n",
"GET https://public.example.test/ HTTP/1.1\r\n\r\n",
"GET / HTTP/1.1\r\nHost: public.example.test\r\nHost: 198.51.100.41\r\n\r\n",
"GET / HTTP/1.1\r\n\r\n",
"POST http://public.example.test/ HTTP/1.1\r\nContent-Length: 3\r\nTransfer-Encoding: chunked\r\n\r\n",
"POST http://public.example.test/ HTTP/1.1\r\nContent-Length: 3\r\nContent-Length: 4\r\n\r\n",
"POST http://public.example.test/ HTTP/1.1\r\nTransfer-Encoding: gzip, chunked\r\n\r\n",
"GET http://public.example.test/ HTTP/1.1\r\n X-Folded: 1\r\n\r\n",
"CONNECT user@public.example.test:443 HTTP/1.1\r\n\r\n",
"CONNECT 2001:db8::1:443 HTTP/1.1\r\n\r\n",
"CONNECT [198.51.100.41]:443 HTTP/1.1\r\n\r\n",
} {
names := stubEgressNames(t, map[string]string{"public.example.test": "198.51.100.40"})
var asked []string
cli, cleanup := dialHandler(t, redirectDial("127.0.0.1:9", &asked))
_, _ = io.WriteString(cli, req)
resp, err := http.ReadResponse(bufio.NewReader(cli), nil)
if err != nil {
t.Errorf("%q: no answer: %v", req, err)
cleanup()
continue
}
body, _ := io.ReadAll(resp.Body)
cleanup()
if resp.StatusCode != 400 || len(body) == 0 || int64(len(body)) != resp.ContentLength {
t.Errorf("%q: answered %d with body %q; want 400 with a body saying why", req, resp.StatusCode, body)
}
if len(*names) != 0 || len(asked) != 0 {
t.Errorf("%q: refused requests resolve and dial nothing; resolved %q, dialled %q", req, *names, asked)
}
}
}
// A request body goes out framed as its head says, chunked bodies re-encoded,
// and nothing after it.
func TestEgressPlainHTTPBodyIsFramed(t *testing.T) {
for _, c := range []struct{ req, wantBody string }{
{"POST http://198.51.100.42/u HTTP/1.1\r\nHost: 198.51.100.42\r\nContent-Length: 5\r\n\r\nhello" +
"GET http://198.51.100.42/smuggled HTTP/1.1\r\n\r\n", "hello"},
{"POST http://198.51.100.42/u HTTP/1.1\r\nHost: 198.51.100.42\r\nTransfer-Encoding: chunked\r\n\r\n" +
"5;ext=1\r\nhello\r\n0\r\nX-Trailer: 1\r\n\r\nGET /smuggled HTTP/1.1\r\n\r\n", "5\r\nhello\r\n0\r\n\r\n"},
} {
o := newSilentOrigin(t)
var asked []string
cli, cleanup := dialHandler(t, redirectDial(o.addr(), &asked))
_, _ = io.WriteString(cli, c.req)
got := o.next(t)
cleanup()
_, body, _ := strings.Cut(got, "\r\n\r\n")
if body != c.wantBody {
t.Errorf("the destination got the body %q; want %q", body, c.wantBody)
}
}
}
// A refusal says why in its body, not only in a header: a 403 from the vetting
// and a 502 from a dial that found no answer, on both request forms, and the
// coder-run proxy's refusals too.
func TestEgressRefusalsCarryABody(t *testing.T) {
failDial := func(ctx context.Context, network, addr string) (net.Conn, error) {
return nil, fmt.Errorf("no route")
}
okDial := func(ctx context.Context, network, addr string) (net.Conn, error) {
return nil, fmt.Errorf("must not dial %s", addr)
}
check := func(name string, resp *http.Response, err error, want int) {
t.Helper()
if err != nil {
t.Errorf("%s: no answer: %v", name, err)
return
}
body, _ := io.ReadAll(resp.Body)
if resp.StatusCode != want || len(body) < 10 || int64(len(body)) != resp.ContentLength {
t.Errorf("%s: answered %d with body %q (Content-Length %d); want %d with a short body saying why",
name, resp.StatusCode, body, resp.ContentLength, want)
}
}
for _, c := range []struct {
name string
dial egressDialer
req string
want int
}{
{"plain HTTP to this machine", okDial, "GET http://127.0.0.1:9/ HTTP/1.1\r\n\r\n", 403},
{"CONNECT to this machine", okDial, "CONNECT 127.0.0.1:9 HTTP/1.1\r\n\r\n", 403},
{"plain HTTP, no answer", failDial, "GET http://198.51.100.43/ HTTP/1.1\r\n\r\n", 502},
{"CONNECT, no answer", failDial, "CONNECT 198.51.100.43:443 HTTP/1.1\r\n\r\n", 502},
} {
cli, cleanup := dialHandler(t, c.dial)
_, _ = io.WriteString(cli, c.req)
resp, err := http.ReadResponse(bufio.NewReader(cli), nil)
check(c.name, resp, err, c.want)
cleanup()
}
oldDial := coderDialOut
t.Cleanup(func() { coderDialOut = oldDial })
coderDialOut = failDial
for _, c := range []struct {
name, req string
want int
}{
{"coder run to a private address", "GET http://10.0.0.1/ HTTP/1.1\r\nHost: 10.0.0.1\r\n\r\n", 403},
{"coder run, no answer", "GET http://93.184.216.34/ HTTP/1.1\r\nHost: 93.184.216.34\r\n\r\n", 502},
} {
boxEnd, runEnd := net.Pipe()
var refused int64
go handleCoderEgressConn(boxEnd, nil, &refused)
_ = runEnd.SetDeadline(time.Now().Add(5 * time.Second))
_, _ = io.WriteString(runEnd, c.req)
resp, err := http.ReadResponse(bufio.NewReader(runEnd), nil)
check(c.name, resp, err, c.want)
_ = runEnd.Close()
}
}
// The coder-run proxy carries one request per connection too: the run's own
// processes are the sender there.
func TestCoderEgressCarriesOneRequestPerConnection(t *testing.T) {
o := newRecordingOrigin(t)
oldDial := coderDialOut
t.Cleanup(func() { coderDialOut = oldDial })
coderDialOut = func(ctx context.Context, network, addr string) (net.Conn, error) {
return (&net.Dialer{}).DialContext(ctx, network, o.addr())
}
boxEnd, runEnd := net.Pipe()
var refused int64
go handleCoderEgressConn(boxEnd, nil, &refused)
_ = runEnd.SetDeadline(time.Now().Add(8 * time.Second))
_, _ = io.WriteString(runEnd, "GET http://93.184.216.34/first HTTP/1.1\r\nHost: 93.184.216.34\r\n\r\n")
br := bufio.NewReader(runEnd)
resp, err := http.ReadResponse(br, nil)
if err != nil {
t.Fatalf("the first request got no answer: %v", err)
}
_, _ = io.ReadAll(resp.Body)
_, _ = io.WriteString(runEnd, "GET http://93.184.216.34/second HTTP/1.1\r\nHost: inside.example.test\r\n\r\n")
got := o.next(t)
_ = runEnd.Close()
if strings.Contains(got, "/second") || strings.Contains(got, "inside.example.test") {
t.Fatalf("a second request on one connection reached the destination unvetted:\n%s", got)
}
if !strings.Contains(got, "\r\nConnection: close\r\n") {
t.Fatalf("the request went out without Connection: close:\n%s", got)
}
}
+47 -12
View File
@@ -14,35 +14,60 @@ import (
const egressWorkerCount = 4 const egressWorkerCount = 4
// egressRefusedWait is how long a worker waits before it parks again after a
// job it did not take (malformed, refused) or after its WebSocket closed while
// parked. A job it took parks the worker again at once, so the person's own
// traffic is never slowed by it; a Host that answers every worker with a job
// this machine refuses costs it at most egressWorkerCount dials, handshakes
// and refusals a second, not as many as the Host can send. The serve lane's
// workers wait the same (serveRefusedWait).
var egressRefusedWait = time.Second
// wsConn adapts a worker WebSocket into a net.Conn so the existing handleEgressConn drives // wsConn adapts a worker WebSocket into a net.Conn so the existing handleEgressConn drives
// it unchanged. The host frames the proxied client stream as binary messages: 'D'+payload = // it unchanged. The host frames the proxied client stream as binary messages: 'D'+payload =
// data, 'E' = end-of-stream. Reads surface the host's bytes; writes go back as 'D' frames; // data, 'E' = end-of-stream. Reads surface the host's bytes; writes go back as 'D' frames;
// Close/CloseWrite send 'E'. // Close/CloseWrite send 'E'.
//
// A 'D' frame is STREAMED into the caller's buffer, never read whole: the host picks a
// frame's length, and one frame of any length must cost this machine no more memory
// than the read it is asked for.
type wsConn struct { type wsConn struct {
ws *websocket.Conn ws *websocket.Conn
rbuf []byte cur io.Reader // the rest of the 'D' frame being read, or nil
wmu sync.Mutex wmu sync.Mutex
} }
func (c *wsConn) Read(p []byte) (int, error) { func (c *wsConn) Read(p []byte) (int, error) {
for len(c.rbuf) == 0 { for {
mt, data, err := c.ws.ReadMessage() if c.cur != nil {
n, err := c.cur.Read(p)
if err == io.EOF {
c.cur = nil
err = nil
}
if n > 0 || err != nil {
return n, err
}
continue
}
mt, r, err := c.ws.NextReader()
if err != nil { if err != nil {
return 0, err return 0, err
} }
if mt != websocket.BinaryMessage || len(data) == 0 { if mt != websocket.BinaryMessage {
continue continue // NextReader discards what is left of it
} }
switch data[0] { var tag [1]byte
if _, err := io.ReadFull(r, tag[:]); err != nil {
continue // an empty frame
}
switch tag[0] {
case 'D': case 'D':
c.rbuf = data[1:] c.cur = r
case 'E': case 'E':
return 0, io.EOF return 0, io.EOF
} }
} }
n := copy(p, c.rbuf)
c.rbuf = c.rbuf[n:]
return n, nil
} }
func (c *wsConn) Write(p []byte) (int, error) { func (c *wsConn) Write(p []byte) (int, error) {
@@ -80,19 +105,25 @@ func (c *wsConn) SetWriteDeadline(t time.Time) error { return c.ws.SetWriteDeadl
// bastion's tunnel IP) and serves one egress job per worker, reconnecting after each. Each // bastion's tunnel IP) and serves one egress job per worker, reconnecting after each. Each
// worker authenticates with the bastion's bearer, then blocks until the host dispatches a // worker authenticates with the bastion's bearer, then blocks until the host dispatches a
// request; handleEgressConn then dials it out the USER's connection (dialOut) and relays. // request; handleEgressConn then dials it out the USER's connection (dialOut) and relays.
// A worker reconnects at once after a job it took, and after egressRefusedWait after one
// it refused or a close while parked. The func returned waits until every worker has
// exited, which is once pctx is done and each one's job has ended.
// //
// pctx is the pairing's context (run.go): when the anchor revokes a pairing the // pctx is the pairing's context (run.go): when the anchor revokes a pairing the
// workers must exit, not re-dial a closed tunnel every 3s for the process's life. // workers must exit, not re-dial a closed tunnel every 3s for the process's life.
func runEgressWorkers(pctx context.Context, self string, wsURL string, tlsCfg *tls.Config, func runEgressWorkers(pctx context.Context, self string, wsURL string, tlsCfg *tls.Config,
dialHost func(ctx context.Context) (net.Conn, error), bearer string, dialHost func(ctx context.Context) (net.Conn, error), bearer string,
dialOut egressDialer, n int) { dialOut egressDialer, n int) (wait func()) {
dialer := &websocket.Dialer{ dialer := &websocket.Dialer{
NetDialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { return dialHost(ctx) }, NetDialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { return dialHost(ctx) },
TLSClientConfig: tlsCfg, TLSClientConfig: tlsCfg,
HandshakeTimeout: 20 * time.Second, HandshakeTimeout: 20 * time.Second,
} }
var wg sync.WaitGroup
for i := 0; i < n; i++ { for i := 0; i < n; i++ {
wg.Add(1)
go func() { go func() {
defer wg.Done()
for pctx.Err() == nil { for pctx.Err() == nil {
ctx, cancel := context.WithTimeout(pctx, 25*time.Second) ctx, cancel := context.WithTimeout(pctx, 25*time.Second)
ws, _, err := dialer.DialContext(ctx, wsURL, http.Header{}) ws, _, err := dialer.DialContext(ctx, wsURL, http.Header{})
@@ -108,11 +139,15 @@ func runEgressWorkers(pctx context.Context, self string, wsURL string, tlsCfg *t
} }
stopKA := armWSKeepalive(ws) // ping/read-deadline: a dead tunnel unblocks the park stopKA := armWSKeepalive(ws) // ping/read-deadline: a dead tunnel unblocks the park
unpark := context.AfterFunc(pctx, func() { _ = ws.Close() }) unpark := context.AfterFunc(pctx, func() { _ = ws.Close() })
handleEgressConn(&wsConn{ws: ws}, dialOut, self) // blocks until a job, serves it, then returns took := handleEgressConn(&wsConn{ws: ws}, dialOut, self) // blocks until a job, serves it, then returns
unpark() unpark()
stopKA() stopKA()
_ = ws.Close() _ = ws.Close()
if !took {
sleepCtx(pctx, egressRefusedWait)
}
} }
}() }()
} }
return wg.Wait
} }
+166
View File
@@ -0,0 +1,166 @@
//go:build !windows
// The egress worker pool (egress_relay.go: runEgressWorkers) against a
// stand-in Host's /bastion-egress: how fast workers park again after a job
// they took and after one they refused. Uses localEgressFixture
// (egress_local_test.go), so it carries the same build tag.
package main
import (
"context"
"io"
"log"
"net"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/gorilla/websocket"
)
// egressHost is a Host's /bastion-egress as the pool meets it: every worker
// that parks is counted and handed the job job(n) (n is its place in line),
// and what the bastion answered is read to its end.
type egressHost struct {
parked atomic.Int32
ok atomic.Int32 // answers that began "HTTP/1.1 200"
}
func startEgressHost(t *testing.T, job func(n int32) string) (*egressHost, string, func(context.Context) (net.Conn, error)) {
t.Helper()
h := &egressHost{}
up := websocket.Upgrader{}
srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
c, err := up.Upgrade(w, r, nil)
if err != nil {
return
}
defer c.Close()
if _, _, err := c.ReadMessage(); err != nil { // BEARER
return
}
n := h.parked.Add(1)
if err := c.WriteMessage(websocket.BinaryMessage, append([]byte{'D'}, job(n)...)); err != nil {
return
}
_ = c.SetReadDeadline(time.Now().Add(5 * time.Second))
var answer strings.Builder
for {
_, m, err := c.ReadMessage()
if err != nil || len(m) == 0 || m[0] == 'E' {
break
}
answer.Write(m[1:])
}
if strings.HasPrefix(answer.String(), "HTTP/1.1 200") {
h.ok.Add(1)
}
}))
srv.Config.ErrorLog = log.New(io.Discard, "", 0)
srv.Start()
t.Cleanup(srv.Close)
addr := srv.Listener.Addr().String()
dial := func(ctx context.Context) (net.Conn, error) {
return (&net.Dialer{}).DialContext(ctx, "tcp", addr)
}
return h, "ws://" + addr + "/bastion-egress", dial
}
// runEgressPool runs egressWorkerCount workers for the test's life, and at its
// end stops them and waits until each has exited: a worker still vetting a job
// would otherwise read the fixture's stubs while the test's cleanup puts them
// back.
func runEgressPool(t *testing.T, wsURL string, dialHost func(context.Context) (net.Conn, error), dialOut egressDialer) {
t.Helper()
ctx, cancel := context.WithCancel(context.Background())
wait := runEgressWorkers(ctx, "home.host.example", wsURL, nil, dialHost, "test-bearer", dialOut, egressWorkerCount)
t.Cleanup(func() {
cancel()
done := make(chan struct{})
go func() { wait(); close(done) }()
select {
case <-done:
case <-time.After(10 * time.Second):
t.Error("the egress workers did not exit 10 s after the pool stopped")
}
})
}
// SECURITY (this machine's resources): a Host that answers every parked egress
// worker with a job this machine refuses (a request for this machine, a line
// that is no request) cannot make it dial, handshake and refuse in a loop as
// fast as the Host answers. Each worker that refused waits before it parks
// again, so the pool costs at most egressWorkerCount dials a second.
//
// MUTATION: in runEgressWorkers, park again at once whatever handleEgressConn
// reports (as the pool did).
// RESULT: red — hundreds of workers park in the 2 s window.
func TestARefusingHostCannotSpinTheEgressWorkers(t *testing.T) {
localEgressFixture(t)
var dialled atomic.Int32
dialOut := func(ctx context.Context, network, addr string) (net.Conn, error) {
dialled.Add(1)
return nil, io.EOF
}
h, wsURL, dialHost := startEgressHost(t, func(n int32) string {
if n%2 == 0 {
return "NOT-A-REQUEST\r\n"
}
return "CONNECT localhost:443 HTTP/1.1\r\nHost: localhost:443\r\n\r\n"
})
runEgressPool(t, wsURL, dialHost, dialOut)
time.Sleep(2 * time.Second)
// egressWorkerCount park at the start, then each worker once a second:
// about 12 in 2 s. A pool that parks again at once makes hundreds.
n := h.parked.Load()
t.Logf("%d workers parked in 2 s", n)
if limit := int32(egressWorkerCount * 4); n > limit {
t.Fatalf("%d egress workers parked in 2 s on a Host that refuses every job (at most %d): the pool re-dials in a loop", n, limit)
}
if n < egressWorkerCount {
t.Fatalf("only %d workers parked: the pool did not run, so the count proves nothing", n)
}
if d := dialled.Load(); d != 0 {
t.Fatalf("a refused job was dialled out %d times", d)
}
}
// The person's own traffic is not slowed: a worker that took a job parks again
// at once, so 40 jobs through 4 workers finish in a fraction of the time one
// wait per job would take (40 / 4 x egressRefusedWait = 10 s).
//
// MUTATION: in runEgressWorkers, wait egressRefusedWait after every job.
// RESULT: red — 40 jobs need about 10 s.
func TestEgressWorkersParkAgainAtOnceAfterAJobTheyTook(t *testing.T) {
localEgressFixture(t)
dialOut := func(ctx context.Context, network, addr string) (net.Conn, error) {
if addr != "198.51.100.9:443" {
t.Errorf("dialled %q, not the address the request named", addr)
}
ours, theirs := net.Pipe()
_ = theirs.Close() // the destination accepts and ends at once
return ours, nil
}
h, wsURL, dialHost := startEgressHost(t, func(int32) string {
return "CONNECT 198.51.100.9:443 HTTP/1.1\r\nHost: 198.51.100.9:443\r\n\r\n"
})
start := time.Now()
runEgressPool(t, wsURL, dialHost, dialOut)
const jobs = 40
deadline := start.Add(10 * time.Second)
for h.ok.Load() < jobs && time.Now().Before(deadline) {
time.Sleep(10 * time.Millisecond)
}
took := time.Since(start)
t.Logf("%d accepted jobs answered in %v", h.ok.Load(), took)
if got := h.ok.Load(); got < jobs {
t.Fatalf("only %d of %d accepted jobs were answered 200 in %v", got, jobs, took)
}
if limit := 3 * egressRefusedWait; took > limit {
t.Fatalf("%d accepted jobs took %v (over %v): a worker that took a job waited before it parked again", jobs, took, limit)
}
}
+23 -6
View File
@@ -38,6 +38,16 @@ func dialHandler(t *testing.T, dialOut egressDialer) (net.Conn, func()) {
return cli, func() { cli.Close(); ln.Close() } return cli, func() { cli.Close(); ln.Close() }
} }
// redirectDial stands in for the user's internet: the lane asks for a public
// destination, and this dials the loopback backend instead, recording what the
// lane asked for. The lane never reaches loopback itself.
func redirectDial(backend string, asked *[]string) egressDialer {
return func(ctx context.Context, network, addr string) (net.Conn, error) {
*asked = append(*asked, addr)
return (&net.Dialer{}).DialContext(ctx, network, backend)
}
}
// CONNECT establishes a raw tunnel; bytes flow both ways through it. // CONNECT establishes a raw tunnel; bytes flow both ways through it.
func TestEgressConnConnectTunnel(t *testing.T) { func TestEgressConnConnectTunnel(t *testing.T) {
backend, err := net.Listen("tcp", "127.0.0.1:0") backend, err := net.Listen("tcp", "127.0.0.1:0")
@@ -54,9 +64,10 @@ func TestEgressConnConnectTunnel(t *testing.T) {
_, _ = io.Copy(c, c) // echo _, _ = io.Copy(c, c) // echo
}() }()
cli, cleanup := dialHandler(t, (&net.Dialer{}).DialContext) var asked []string
cli, cleanup := dialHandler(t, redirectDial(backend.Addr().String(), &asked))
defer cleanup() defer cleanup()
fmt.Fprintf(cli, "CONNECT %s HTTP/1.1\r\nHost: %s\r\n\r\n", backend.Addr(), backend.Addr()) fmt.Fprint(cli, "CONNECT 198.51.100.20:443 HTTP/1.1\r\nHost: 198.51.100.20:443\r\n\r\n")
br := bufio.NewReader(cli) br := bufio.NewReader(cli)
status, err := br.ReadString('\n') status, err := br.ReadString('\n')
if err != nil || !strings.Contains(status, "200") { if err != nil || !strings.Contains(status, "200") {
@@ -82,6 +93,9 @@ func TestEgressConnConnectTunnel(t *testing.T) {
if string(buf) != want { if string(buf) != want {
t.Fatalf("tunnel echo mismatch: got %q", string(buf)) t.Fatalf("tunnel echo mismatch: got %q", string(buf))
} }
if len(asked) != 1 || asked[0] != "198.51.100.20:443" {
t.Fatalf("the lane must dial the address it vetted: %v", asked)
}
} }
func TestEgressConnPlainHTTP(t *testing.T) { func TestEgressConnPlainHTTP(t *testing.T) {
@@ -89,15 +103,18 @@ func TestEgressConnPlainHTTP(t *testing.T) {
fmt.Fprint(w, "plain-http-ok") fmt.Fprint(w, "plain-http-ok")
})) }))
defer backend.Close() defer backend.Close()
host := strings.TrimPrefix(backend.URL, "http://")
cli, cleanup := dialHandler(t, (&net.Dialer{}).DialContext) var asked []string
cli, cleanup := dialHandler(t, redirectDial(strings.TrimPrefix(backend.URL, "http://"), &asked))
defer cleanup() defer cleanup()
fmt.Fprintf(cli, "GET http://%s/ HTTP/1.1\r\nHost: %s\r\nConnection: close\r\n\r\n", host, host) fmt.Fprint(cli, "GET http://198.51.100.21/ HTTP/1.1\r\nHost: 198.51.100.21\r\nConnection: close\r\n\r\n")
body, _ := io.ReadAll(cli) body, _ := io.ReadAll(cli)
if !strings.Contains(string(body), "plain-http-ok") { if !strings.Contains(string(body), "plain-http-ok") {
t.Fatalf("plain HTTP forward: got %q", string(body)) t.Fatalf("plain HTTP forward: got %q", string(body))
} }
if len(asked) != 1 || asked[0] != "198.51.100.21:80" {
t.Fatalf("the lane must dial the address it vetted: %v", asked)
}
} }
func TestEgressConnDialFailure(t *testing.T) { func TestEgressConnDialFailure(t *testing.T) {
@@ -106,7 +123,7 @@ func TestEgressConnDialFailure(t *testing.T) {
} }
cli, cleanup := dialHandler(t, failDial) cli, cleanup := dialHandler(t, failDial)
defer cleanup() defer cleanup()
fmt.Fprint(cli, "CONNECT unreachable.example:443 HTTP/1.1\r\n\r\n") fmt.Fprint(cli, "CONNECT 198.51.100.22:443 HTTP/1.1\r\n\r\n")
status, _ := bufio.NewReader(cli).ReadString('\n') status, _ := bufio.NewReader(cli).ReadString('\n')
if !strings.Contains(status, "502") { if !strings.Contains(status, "502") {
t.Fatalf("expected 502 on dial failure, got %q", status) t.Fatalf("expected 502 on dial failure, got %q", status)
+230
View File
@@ -0,0 +1,230 @@
//go:build !windows
// Uses fakeToken from pairing_redeem_test.go and setupBox/driveOp from
// anchor_test.go, both unix-only.
package main
// A pairing whose file holds an endpoint this build refuses loads as invalid
// and is never served (endpoint_test.go). These are the paths that could
// still act on it: anchoring the box to it, an approval from the anchor host,
// what the hosts are told about it, and the two ways a person clears it.
import (
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
// unservedEndpoint is one the endpoint check refuses (badTunnelEndpoints).
const unservedEndpoint = "203.0.113.7:51820\nallowed_ip=::/0"
// writeUnservedPairing writes a pairing file in the given state whose
// endpoint this build refuses, and returns its path.
func writeUnservedPairing(t *testing.T, host, state string) string {
t.Helper()
pd, err := pairingsDir()
if err != nil {
t.Fatal(err)
}
b, _ := json.Marshal(Config{HostName: host, WGPrivateKey: "k", BastionTunnelIP: "fd12:9::a:1",
State: state, Endpoint: unservedEndpoint})
p := filepath.Join(pd, host+".json")
if err := os.WriteFile(p, b, 0o600); err != nil {
t.Fatal(err)
}
return p
}
// fileState is the state field as written in host's pairing file.
func fileState(t *testing.T, host string) string {
t.Helper()
pd, _ := pairingsDir()
raw, err := os.ReadFile(filepath.Join(pd, host+".json"))
if err != nil {
t.Fatal(err)
}
var c Config
if err := json.Unmarshal(raw, &c); err != nil {
t.Fatal(err)
}
return c.State
}
// claim-anchor names the reason and writes nothing: the pairing keeps its
// state, and the box is neither anchored nor claimed. As the sole pairing and
// named with --host among others, in the legacy active state and pending.
func TestClaimAnchorRefusesAPairingThatIsNotServed(t *testing.T) {
if os.Geteuid() != 0 {
t.Skip("claim-anchor needs machine authority, which needs root")
}
for _, c := range []struct {
name string
others []string
host string
}{{"sole pairing", nil, ""}, {"named with --host", []string{tAnchor}, tMember}} {
for _, st := range []string{"", pairingPending} {
t.Run(c.name+"/"+st, func(t *testing.T) {
t.Setenv("XDG_CONFIG_HOME", t.TempDir())
withMarkerAt(t, filepath.Join(t.TempDir(), "machine.json"))
if err := setBoxMode(modeNetworked); err != nil {
t.Fatal(err)
}
for _, o := range c.others {
writePairing(t, o)
}
p := writeUnservedPairing(t, tMember, st)
err := cmdClaimAnchor("", false, "", c.host)
if err == nil || !contains(err.Error(), "not a host:port") || !contains(err.Error(), p) {
t.Fatalf("claim-anchor must refuse the pairing, naming its file and why: %v", err)
}
if a := currentAnchor(); a != nil {
t.Fatalf("a refused claim anchored the box: %+v", a)
}
if oc := currentOwnerClaim(); oc != nil {
t.Fatalf("a refused claim wrote the owner claim: %+v", oc)
}
if got := fileState(t, tMember); got != st {
t.Fatalf("a refused claim changed the pairing's state from %q to %q", st, got)
}
})
}
}
}
// The anchor host's approval of such a pairing is refused: it is not made
// active, no grant is kept, and the refusal names no file of this machine.
// The copy updatePairingOnDisk hands back carries the same mark, so a caller
// that starts a pairing from it starts nothing.
func TestAnApprovalOfAPairingThatIsNotServedIsRefused(t *testing.T) {
priv := setupBox(t)
writeUnservedPairing(t, tMember, pairingPending)
future := time.Now().Add(time.Hour).UTC().Format(time.RFC3339)
terms, sig := signTerms(t, priv, grantTerms(tMember, []string{"observe"}, future))
_, errOut, code := driveOp(t, tAnchor, "pairing_grant_sync", []string{terms, sig})
if code != 126 || !contains(errOut, "not served") {
t.Fatalf("the approval must be refused (126, not served): %d %s", code, errOut)
}
pd, _ := pairingsDir()
if contains(errOut, pd) {
t.Fatalf("the refusal sent to the host names a file of this machine: %s", errOut)
}
if got := fileState(t, tMember); got != pairingPending {
t.Fatalf("a refused approval changed the state to %q", got)
}
if _, err := loadPairingGrant(tMember); err == nil {
t.Fatal("a refused approval kept the grant")
}
c, err := updatePairingOnDisk(tMember, func(d *Config) { d.State = pairingActive })
if err != nil {
t.Fatal(err)
}
if c.invalid == "" || c.active() {
t.Fatalf("the copy written back must still be invalid and never active: invalid=%q active=%v", c.invalid, c.active())
}
if startPairing(c) {
t.Fatal("a pairing that is not served was started")
}
}
// What the hosts hear: neither the peer list every host gets, nor the
// pairing list the anchor host gets (in the heartbeat and from pairing_list),
// names such a pairing, and every state they do carry is one a host knows.
// The machine's own status still shows it invalid (endpoint_test.go).
func TestNoHostIsToldOfAPairingThatIsNotServed(t *testing.T) {
setupBox(t)
const beta, pend = "beta.host.privatae.ai", "pend.host.privatae.ai"
writePairing(t, beta)
writeUnservedPairing(t, tMember, "")
writeUnservedPairing(t, pend, pairingPending)
known := map[string]bool{pairingActive: true, pairingPending: true, pairingRevoked: true}
peers := peersFor(tAnchor)
if len(peers) != 1 || peers[0].HostName != beta {
t.Fatalf("the peers must be only %s, got %+v", beta, peers)
}
for _, p := range peers {
if !known[p.State] {
t.Fatalf("a peer was sent with a state a host does not know: %+v", p)
}
}
list := pairingsReport()
var names []string
for _, e := range list {
names = append(names, e.HostName)
if !known[e.State] {
t.Fatalf("a pairing was sent with a state a host does not know: %+v", e)
}
}
if got := strings.Join(names, ","); got != tAnchor+","+beta {
t.Fatalf("the anchor host's list must be %s,%s, got %s", tAnchor, beta, got)
}
out, errOut, code := driveOp(t, tAnchor, "pairing_list", nil)
if code != 0 || contains(out, tMember) || contains(out, pend) || contains(out, pairingInvalid) {
t.Fatalf("pairing_list must leave the pairing out: %d %s %s", code, out, errOut)
}
if u := unapprovedAttempts(time.Now().Add(24 * time.Hour)); len(u) != 0 {
t.Fatalf("a pairing that is not served is no unapproved attempt: %+v", u)
}
cfgs, err := loadPairings()
if err != nil {
t.Fatal(err)
}
if c := pairingFor(cfgs, tMember); c == nil || c.stateName() != pairingInvalid {
t.Fatalf("this machine still lists it invalid: %+v", c)
}
}
// pair --replace over such a pairing pairs it again, served; unpair removes
// one. Both through the commands a person runs.
func TestPairReplaceAndUnpairClearAPairingThatIsNotServed(t *testing.T) {
uiTestConfigDir(t)
withMarkerAt(t, filepath.Join(t.TempDir(), "machine.json"))
prev := unpairNeedsRoot
unpairNeedsRoot = func() bool { return false }
t.Cleanup(func() { unpairNeedsRoot = prev })
const host, good = "acme.host.privatae.ai", "203.0.113.7:51820"
redeem := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_ = json.NewEncoder(w).Encode(map[string]any{
"operator_crt": "CERT", "ca_crt": "CA", "bearer": "b",
"endpoint": good, "endpoints": []string{good},
"endpoint_pubkey": testWGKey(t), "bastion_tunnel_ip": "2001:db8::a:1",
"host_tunnel_ip": "2001:db8::e:3", "host_name": host,
})
}))
defer redeem.Close()
old := redeemURLBase
redeemURLBase = redeem.URL
t.Cleanup(func() { redeemURLBase = old })
writeUnservedPairing(t, host, "")
if err := cmdPair(fakeToken("acme"), "", "", true); err != nil {
t.Fatalf("pair --replace over a pairing that is not served: %v", err)
}
cfgs, err := loadPairings()
if err != nil || len(cfgs) != 1 {
t.Fatalf("after pair --replace one pairing loads: %v %v", cfgs, err)
}
if c := cfgs[0]; c.HostName != host || c.invalid != "" || !c.active() || c.Endpoint != good {
t.Fatalf("pair --replace must leave a served pairing: %q invalid=%q active=%v endpoint=%q",
c.HostName, c.invalid, c.active(), c.Endpoint)
}
writeUnservedPairing(t, host, "")
if err := cmdUnpair(host); err != nil {
t.Fatalf("unpair of a pairing that is not served: %v", err)
}
if cfgs, err := loadPairings(); err != nil || len(cfgs) != 0 {
t.Fatalf("after unpair nothing is left: %v %v", cfgs, err)
}
pd, _ := pairingsDir()
if _, err := os.Stat(filepath.Join(pd, host+".json")); !os.IsNotExist(err) {
t.Fatalf("the pairing file is still there: %v", err)
}
}
+207
View File
@@ -0,0 +1,207 @@
//go:build !windows
// Uses fakeToken from pairing_redeem_test.go, unix-only.
package main
// A tunnel endpoint arrives from the host three ways: in the pairing
// response, in the rotate directive on a heartbeat answer, and from the
// pairing file those were saved to. Each one refuses an endpoint that is
// not one host:port, the file by never serving that pairing (the
// device-level checks are in claims_tunnel_test.go).
import (
"encoding/json"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
)
// A pairing response whose endpoint, or any entry of its endpoints list, is
// not one host:port is refused and nothing is saved. An IPv4, an IPv6 and a
// host name endpoint pair.
func TestAPairingResponseWithAnUnsafeEndpointSavesNothing(t *testing.T) {
cfgHome := uiTestConfigDir(t)
withMarkerAt(t, filepath.Join(t.TempDir(), "machine.json"))
var endpoint, hostName string
var endpoints []string
redeem := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_ = json.NewEncoder(w).Encode(map[string]any{
"operator_crt": "CERT", "ca_crt": "CA", "bearer": "b",
"endpoint": endpoint, "endpoints": endpoints,
"endpoint_pubkey": testWGKey(t), "bastion_tunnel_ip": "2001:db8::a:1",
"host_tunnel_ip": "2001:db8::e:3", "host_name": hostName,
})
}))
defer redeem.Close()
old := redeemURLBase
redeemURLBase = redeem.URL
t.Cleanup(func() { redeemURLBase = old })
hostName = "acme.host.privatae.ai"
for _, bad := range badTunnelEndpoints {
if bad == "" {
continue // a missing endpoint has its own refusal
}
for _, c := range []struct {
endpoint string
list []string
}{{bad, nil}, {"203.0.113.7:51820", []string{"[2001:db8::7]:51820", bad}}} {
endpoint, endpoints = c.endpoint, c.list
if _, _, err := pairWithToken(fakeToken("acme"), false, false); err == nil || !strings.Contains(err.Error(), "not a host:port") {
t.Errorf("%q in %+v: the pairing must be refused for its endpoint, got %v", bad, c, err)
}
}
}
_ = filepath.Walk(cfgHome, func(path string, fi os.FileInfo, err error) error {
if err == nil && !fi.IsDir() {
t.Errorf("a refused pairing left a file behind: %s", path)
}
return nil
})
for _, good := range goodTunnelEndpoints {
uiTestConfigDir(t) // a local bastion serves one host: a fresh box each
withMarkerAt(t, filepath.Join(t.TempDir(), "machine.json"))
endpoint, endpoints = good, []string{good}
cfg, _, err := pairWithToken(fakeToken("acme"), false, false)
if err != nil {
t.Fatalf("%q must pair: %v", good, err)
}
if cfg.Endpoint != good || len(cfg.Endpoints) != 1 || cfg.Endpoints[0] != good {
t.Fatalf("%q: saved %q %q", good, cfg.Endpoint, cfg.Endpoints)
}
}
}
// A pairing file that already holds an unsafe endpoint (written by an older
// build, or by hand) loads as invalid, naming the file and why: it is never
// served, while every other pairing still loads, status shows it, and unpair
// removes it. A machine that upgrades is never stopped by one such file.
func TestAPairingFileWithAnUnsafeEndpointLoadsButIsNeverServed(t *testing.T) {
uiTestConfigDir(t)
withMarkerAt(t, filepath.Join(t.TempDir(), "machine.json"))
supervised(t)
prev := unpairNeedsRoot
unpairNeedsRoot = func() bool { return false }
t.Cleanup(func() { unpairNeedsRoot = prev })
pd, err := pairingsDir()
if err != nil {
t.Fatal(err)
}
write := func(c *Config) string {
t.Helper()
b, _ := json.Marshal(c)
p := filepath.Join(pd, c.HostName+".json")
if err := os.WriteFile(p, b, 0o600); err != nil {
t.Fatal(err)
}
return p
}
const host, other = "acme.host.privatae.ai", "other.host.privatae.ai"
write(&Config{HostName: other, Endpoint: goodTunnelEndpoints[0]})
for _, bad := range badTunnelEndpoints {
if bad == "" {
continue // a pairing over a socket has no endpoint
}
for _, c := range []*Config{
{HostName: host, Endpoint: bad},
{HostName: host, Endpoint: "203.0.113.7:51820", Endpoints: []string{bad}},
} {
p := write(c)
cfgs, err := loadPairings()
if err != nil || len(cfgs) != 2 {
t.Fatalf("%q: one bad file must not fail the load: %v %v", bad, cfgs, err)
}
got := pairingFor(cfgs, host)
if got.active() || got.stateName() != "invalid" {
t.Errorf("%q: the pairing must load invalid: active=%v %q", bad, got.active(), got.stateName())
}
if out := statusOutput(t); !strings.Contains(out, host+" [invalid]") ||
!strings.Contains(out, "not a host:port") || !strings.Contains(out, p) {
t.Errorf("%q: status must show the pairing invalid, naming its file and why:\n%s", bad, out)
}
if !pairingFor(cfgs, other).active() {
t.Errorf("%q: the other pairing must still be served", bad)
}
if startPairing(got) {
t.Errorf("%q: an invalid pairing was started", bad)
}
}
}
// The supervisor starts nothing for it either (the other pairing is
// removed first: starting it would raise a real tunnel).
if err := os.Remove(filepath.Join(pd, other+".json")); err != nil {
t.Fatal(err)
}
superviseOnce()
supervisor.Lock()
running := len(supervisor.runs)
supervisor.Unlock()
if running != 0 {
t.Fatalf("the supervisor started %d run(s) for an invalid pairing", running)
}
if h, err := unpairHost(host); err != nil || h != host {
t.Fatalf("unpair must remove an invalid pairing: %q %v", h, err)
}
if cfgs, err := loadPairings(); err != nil || len(cfgs) != 0 {
t.Fatalf("after unpair nothing is left: %v %v", cfgs, err)
}
for _, good := range goodTunnelEndpoints {
write(&Config{HostName: host, Endpoint: good, Endpoints: goodTunnelEndpoints})
if cfgs, err := loadPairings(); err != nil || len(cfgs) != 1 || cfgs[0].Endpoint != good || !cfgs[0].active() {
t.Fatalf("%q must load and be served: %v %v", good, cfgs, err)
}
}
write(&Config{HostName: host, Transport: transportUnix})
if cfgs, err := loadPairings(); err != nil || len(cfgs) != 1 || !cfgs[0].active() {
t.Fatalf("a pairing with no endpoint must load and be served: %v %v", cfgs, err)
}
}
// statusOutput is what `status` prints.
func statusOutput(t *testing.T) string {
t.Helper()
r, w, err := os.Pipe()
if err != nil {
t.Fatal(err)
}
stdout := os.Stdout
os.Stdout = w
serr := cmdStatus()
os.Stdout = stdout
_ = w.Close()
out, _ := io.ReadAll(r)
if serr != nil {
t.Fatalf("status: %v", serr)
}
return string(out)
}
// A rotate directive that names any unsafe endpoint is not acted on; one
// that names only host:port entries is.
func TestARotateDirectiveWithAnUnsafeEndpointIsIgnored(t *testing.T) {
for _, bad := range badTunnelEndpoints {
if bad == "" {
continue // an empty entry is no entry
}
for _, d := range [][]byte{
directive(bad, nil, "PK="),
directive("203.0.113.7:51820", []string{bad, "[2001:db8::7]:51820"}, "PK="),
} {
if c, pk := parseRotateTarget(d); len(c) != 0 || pk != "" {
t.Errorf("%q: the directive must be ignored, got %q %q", bad, c, pk)
}
}
}
c, pk := parseRotateTarget(directive("203.0.113.7:51820", goodTunnelEndpoints, "PK="))
if pk != "PK=" || !sameList(c, goodTunnelEndpoints) {
t.Fatalf("a directive of host:port entries must be taken: %q %q", c, pk)
}
}
+60 -6
View File
@@ -60,6 +60,27 @@ type freeagentConfig struct {
Cert string // PEM: the client identity this box presents Cert string // PEM: the client identity this box presents
Key string Key string
CA string // PEM: the root the local agent's server cert must chain to CA string // PEM: the root the local agent's server cert must chain to
// Mode is the sanitize mode this bastion wrote into the agent's manifest
// (faManagedSanitizeMode for every agent it deploys), or "" for an agent
// someone runs by hand, whose manifest this bastion did not write.
Mode string
}
// faManagedSanitizeMode is the sanitize mode the bastion writes into every
// manifest it deploys (faManifestText). Flag mode is the agent's rules pass
// only: it replaces e-mail addresses, phone numbers, card, bank and ID
// numbers, IP and MAC addresses and keys, and does not find names, street
// addresses or dates of birth.
const faManagedSanitizeMode = "flag"
// faSanitizeNotes says, for each mode a reply can carry, what personal data
// may still be in the text, in the words the host relays. A mode with no entry
// here is reported as not known to remove personal data.
var faSanitizeNotes = map[string]string{
"flag": "rules only: e-mail addresses, phone numbers, card, bank and ID numbers, IP and MAC addresses " +
"and keys were replaced; names, street addresses and dates of birth were not",
"sterilize": "",
"none": "the agent's manifest has no sanitize stage, so nothing in the text was replaced",
} }
func freeagentEnv() (freeagentConfig, bool) { func freeagentEnv() (freeagentConfig, bool) {
@@ -252,6 +273,16 @@ func freeagentProbe() (bool, bool) {
// local agent's own 403 (grant/mask) and 503 (manifest not // local agent's own 403 (grant/mask) and 503 (manifest not
// serving) are refusals with the reason it gave; 404 on read_range is gone. // serving) are refusals with the reason it gave; 404 on read_range is gone.
func freeagentMap(op string, status int, body []byte, err error) (map[string]any, string, bool) { func freeagentMap(op string, status int, body []byte, err error) (map[string]any, string, bool) {
return freeagentMapMode(op, "", status, body, err)
}
// freeagentMapMode is freeagentMap for an agent whose manifest this bastion
// wrote with sanitize mode `mode` ("" when it did not write it). A successful
// reply says how its text was sanitized: `sanitize_mode` is the mode this
// bastion wrote, else the one the agent reports, else "unknown"; `sanitized`
// is true only for "sterilize", the mode that looks for names; and
// `sanitize_note` says what the mode leaves in.
func freeagentMapMode(op, mode string, status int, body []byte, err error) (map[string]any, string, bool) {
if err != nil { if err != nil {
return map[string]any{"state": "offline", "detail": "freeagent: " + err.Error()}, "", true return map[string]any{"state": "offline", "detail": "freeagent: " + err.Error()}, "", true
} }
@@ -267,7 +298,7 @@ func freeagentMap(op string, status int, body []byte, err error) (map[string]any
return nil, "freeagent: malformed reply", false return nil, "freeagent: malformed reply", false
case status == http.StatusOK: case status == http.StatusOK:
payload["state"] = "ok" payload["state"] = "ok"
payload["sanitized"] = true faStampSanitize(payload, mode)
return payload, "", true return payload, "", true
case status == http.StatusNotFound && op == "read_range": case status == http.StatusNotFound && op == "read_range":
return map[string]any{"state": "gone", "detail": reason}, "", true return map[string]any{"state": "gone", "detail": reason}, "", true
@@ -286,10 +317,33 @@ func freeagentMap(op string, status int, body []byte, err error) (map[string]any
} }
} }
// faStampSanitize writes the three sanitize fields onto a successful reply.
// The mode this bastion wrote wins over the one the reply carries: a reply
// cannot make text it never sterilized read as sterilized.
func faStampSanitize(payload map[string]any, mode string) {
if mode == "" {
mode, _ = payload["sanitize_mode"].(string)
}
if mode == "" {
mode = "unknown"
}
note, known := faSanitizeNotes[mode]
if !known {
note = "the local agent did not say how this text was sanitized; it may contain personal data"
}
payload["sanitize_mode"] = mode
payload["sanitized"] = mode == "sterilize"
delete(payload, "sanitize_note")
if note != "" {
payload["sanitize_note"] = note
}
}
// handleDcnSearch — op `dcn_search`: args = <store_id> <json {query, k}>. Gated // handleDcnSearch — op `dcn_search`: args = <store_id> <json {query, k}>. Gated
// by the synced grant with the 'search' mask, then forwarded to the // by the synced grant with the 'search' mask, then forwarded to the
// local agent's /search. Hits are sanitized windows with document ids the host // local agent's /search. Hits are windows of the agent's sanitized text, with
// feeds back to read_range. // document ids the host feeds back to read_range; faStampSanitize says which
// sanitize mode produced them.
func handleDcnSearch(ws *websocket.Conn, dir string, args []string) { func handleDcnSearch(ws *websocket.Conn, dir string, args []string) {
storeID, a, ok := dcnParseArgs(ws, args) storeID, a, ok := dcnParseArgs(ws, args)
if !ok { if !ok {
@@ -319,7 +373,7 @@ func handleDcnSearch(ws *websocket.Conn, dir string, args []string) {
} }
status, body, err := freeagentCall(c, http.MethodPost, "/search", status, body, err := freeagentCall(c, http.MethodPost, "/search",
map[string]any{"query": q, "k": k}) map[string]any{"query": q, "k": k})
payload, why, ok := freeagentMap("search", status, body, err) payload, why, ok := freeagentMapMode("search", c.Mode, status, body, err)
if !ok { if !ok {
dcnRefuse(ws, dir, "search", storeID, "", why) dcnRefuse(ws, dir, "search", storeID, "", why)
return return
@@ -394,7 +448,7 @@ func handleDcnCatalog(ws *websocket.Conn, dir string, args []string) {
"state": "offline", "detail": "no catalog yet — the local agent's first sweep has not finished; try again later"}) "state": "offline", "detail": "no catalog yet — the local agent's first sweep has not finished; try again later"})
return return
} }
payload, why, ok := freeagentMap("catalog", status, body, err) payload, why, ok := freeagentMapMode("catalog", c.Mode, status, body, err)
if !ok { if !ok {
dcnRefuse(ws, dir, "catalog", storeID, "kind:"+kind, why) dcnRefuse(ws, dir, "catalog", storeID, "kind:"+kind, why)
return return
@@ -427,7 +481,7 @@ func handleDcnReadRangeDoc(ws *websocket.Conn, dir string, storeID int64, a *dcn
} }
status, body, err := freeagentCall(c, http.MethodPost, "/read_range", status, body, err := freeagentCall(c, http.MethodPost, "/read_range",
map[string]any{"doc": a.Doc, "start": start, "end": end}) map[string]any{"doc": a.Doc, "start": start, "end": end})
payload, why, ok := freeagentMap("read_range", status, body, err) payload, why, ok := freeagentMapMode("read_range", c.Mode, status, body, err)
if !ok { if !ok {
dcnRefuse(ws, dir, "read_range", storeID, "doc:"+a.Doc, why) dcnRefuse(ws, dir, "read_range", storeID, "doc:"+a.Doc, why)
return return
+9 -1
View File
@@ -81,11 +81,19 @@ type faOwner struct {
gid uint32 gid uint32
name string name string
home string home string
// plat is what this platform needs beyond the above: nothing on unix, the // plat is what this platform needs beyond the above: nothing on unix (an
// owner there is fully described by its uid, gid, name and home), the
// owner's SID on Windows (freeagent_drop_windows.go). // owner's SID on Windows (freeagent_drop_windows.go).
plat faOwnerPlat plat faOwnerPlat
} }
// faOwnerPlat holds the owner's SID, as text, and their bare user name. Only
// Windows fills it; it stays zero on unix.
type faOwnerPlat struct {
sid string
user string
}
// faResolveOwner decides the identity for a store's agent. Where this // faResolveOwner decides the identity for a store's agent. Where this
// platform drops and the daemon holds machine authority it is the granted // platform drops and the daemon holds machine authority it is the granted
// folder's owner, refused for the machine's own accounts and for anyone who // folder's owner, refused for the machine's own accounts and for anyone who
-3
View File
@@ -20,9 +20,6 @@ func faDropSupported() bool { return true }
// own 0700 leaf. // own 0700 leaf.
func faRootStateBase() string { return "/var/lib/privatae/freeagent" } func faRootStateBase() string { return "/var/lib/privatae/freeagent" }
// faOwnerPlat: a unix owner is fully described by its uid, gid, name and home.
type faOwnerPlat struct{}
// faHandsOverByChown: on unix every file root writes for the child is handed // faHandsOverByChown: on unix every file root writes for the child is handed
// over by changing its owner. // over by changing its owner.
const faHandsOverByChown = true const faHandsOverByChown = true
-13
View File
@@ -71,12 +71,6 @@ func faRootStateBase() string {
return filepath.Join(d, "freeagent-state") return filepath.Join(d, "freeagent-state")
} }
// faOwnerPlat holds the owner's SID, as text, and their bare user name.
type faOwnerPlat struct {
sid string
user string
}
// faHandsOverByChown: on Windows the hand-over is a permission entry on the // faHandsOverByChown: on Windows the hand-over is a permission entry on the
// state leaf, inherited by everything later created in it. // state leaf, inherited by everything later created in it.
const faHandsOverByChown = false const faHandsOverByChown = false
@@ -430,12 +424,6 @@ func (c *winChild) Wait() error {
return nil return nil
} }
func faPathOwner(string) (uint32, uint32, error) {
return 0, 0, errors.New("ownership is not read as a uid on windows")
}
func faChown(string, uint32, uint32) error { return nil }
// sysProcAttr: the Windows drop does not go through a credential; faStartChild // sysProcAttr: the Windows drop does not go through a credential; faStartChild
// starts the child under the owner's token itself. // starts the child under the owner's token itself.
func (o *faOwner) sysProcAttr() *syscall.SysProcAttr { return nil } func (o *faOwner) sysProcAttr() *syscall.SysProcAttr { return nil }
@@ -457,5 +445,4 @@ func faFdSane(f *os.File) error {
return nil return nil
} }
func faInfoOwner(os.FileInfo) (uint32, uint32, bool) { return 0, 0, false }
func faInfoLinks(os.FileInfo) (uint64, bool) { return 0, false } func faInfoLinks(os.FileInfo) (uint64, bool) { return 0, false }
+3 -2
View File
@@ -500,7 +500,7 @@ func faManifestText(id, cn string, terms *dcnTerms, sweep, recipient, embedModel
fmt.Fprintf(&b, " exclude: [%s]\n", strings.Join(ex, ", ")) fmt.Fprintf(&b, " exclude: [%s]\n", strings.Join(ex, ", "))
} }
fmt.Fprintf(&b, "mask: [%s]\nstages: [%s]\n", strings.Join(mask, ", "), strings.Join(stages, ", ")) fmt.Fprintf(&b, "mask: [%s]\nstages: [%s]\n", strings.Join(mask, ", "), strings.Join(stages, ", "))
b.WriteString("sanitize:\n mode: flag\n") fmt.Fprintf(&b, "sanitize:\n mode: %s\n", faManagedSanitizeMode)
if containsStr(stages, "embed") { if containsStr(stages, "embed") {
fmt.Fprintf(&b, "embed:\n model: %s\n dim: %d\n chunk: {size_tokens: 512, overlap_tokens: 64}\n", yamlStr(embedModel), embedDim) fmt.Fprintf(&b, "embed:\n model: %s\n dim: %d\n chunk: {size_tokens: 512, overlap_tokens: 64}\n", yamlStr(embedModel), embedDim)
} }
@@ -677,7 +677,8 @@ func faManagedConfig(storeID int64) (freeagentConfig, bool) {
if err != nil || port <= 0 { if err != nil || port <= 0 {
return freeagentConfig{}, false return freeagentConfig{}, false
} }
return freeagentConfig{Addr: "127.0.0.1:" + strconv.Itoa(port), Cert: inst.Cert, Key: inst.Key, CA: inst.Cert}, true return freeagentConfig{Addr: "127.0.0.1:" + strconv.Itoa(port), Cert: inst.Cert, Key: inst.Key, CA: inst.Cert,
Mode: faManagedSanitizeMode}, true
} }
// ── ensure / reconcile ────────────────────────────────────────────────────── // ── ensure / reconcile ──────────────────────────────────────────────────────
+47 -2
View File
@@ -190,8 +190,11 @@ func TestFreeagentProbeAndEnv(t *testing.T) {
func TestFreeagentMap(t *testing.T) { func TestFreeagentMap(t *testing.T) {
m, why, ok := freeagentMap("search", 200, []byte(`{"hits":[]}`), nil) m, why, ok := freeagentMap("search", 200, []byte(`{"hits":[]}`), nil)
if !ok || m["state"] != "ok" || m["sanitized"] != true || why != "" { if !ok || m["state"] != "ok" || why != "" {
t.Fatalf("200 → ok/sanitized, got %v %q", m, why) t.Fatalf("200 → ok, got %v %q", m, why)
}
if m["sanitized"] != false || m["sanitize_mode"] != "unknown" || m["sanitize_note"] == nil {
t.Fatalf("a reply that names no mode is not called sanitized, got %v", m)
} }
m, _, ok = freeagentMap("read_range", 404, []byte(`{"error":"no such document"}`), nil) m, _, ok = freeagentMap("read_range", 404, []byte(`{"error":"no such document"}`), nil)
if !ok || m["state"] != "gone" { if !ok || m["state"] != "gone" {
@@ -298,3 +301,45 @@ func TestDcnOpArgsCarryFreeagentFields(t *testing.T) {
t.Fatalf("args not carried: %+v", a) t.Fatalf("args not carried: %+v", a)
} }
} }
// The sanitize fields say what was done to the text. An agent this bastion
// deployed runs in flag mode, which leaves names in: its replies are never
// called sanitized, whatever the reply itself claims. An agent run by hand is
// reported in the mode it names, and only sterilize counts as sanitized.
func TestASanitizeStampNamesTheModeThatRan(t *testing.T) {
for _, c := range []struct {
name, mode, body, wantMode string
wantSanitized, wantNote bool
}{
{"deployed, reply silent", faManagedSanitizeMode, `{"hits":[]}`, "flag", false, true},
{"deployed, reply claims sterilize", faManagedSanitizeMode, `{"hits":[],"sanitize_mode":"sterilize","sanitized":true}`, "flag", false, true},
{"by hand, flag", "", `{"hits":[],"sanitize_mode":"flag"}`, "flag", false, true},
{"by hand, sterilize", "", `{"hits":[],"sanitize_mode":"sterilize","sanitize_note":"x"}`, "sterilize", true, false},
{"by hand, silent", "", `{"hits":[],"sanitized":true}`, "unknown", false, true},
{"by hand, no sanitize stage", "", `{"hits":[],"sanitize_mode":"none"}`, "none", false, true},
{"by hand, a mode this build does not know", "", `{"hits":[],"sanitize_mode":"other"}`, "other", false, true},
} {
for _, op := range []string{"search", "read_range", "catalog"} {
m, why, ok := freeagentMapMode(op, c.mode, 200, []byte(c.body), nil)
if !ok || why != "" || m["state"] != "ok" {
t.Fatalf("%s/%s: %v %q", c.name, op, m, why)
}
note, hasNote := m["sanitize_note"].(string)
if m["sanitize_mode"] != c.wantMode || m["sanitized"] != c.wantSanitized || hasNote != c.wantNote || (hasNote && note == "") {
t.Errorf("%s/%s: got mode %v sanitized %v note %q", c.name, op, m["sanitize_mode"], m["sanitized"], note)
}
}
}
if !strings.Contains(faSanitizeNotes["flag"], "names") {
t.Fatal("the flag note must say names are left in")
}
}
// The mode the stamp reports for a deployed agent is the mode the bastion
// writes into its manifest.
func TestTheDeployedManifestUsesTheStampedMode(t *testing.T) {
text := faManifestText("s1", "cn", &dcnTerms{Root: "/tmp/x", Mask: []string{"search"}}, "", "", "m", 8)
if !strings.Contains(text, "sanitize:\n mode: "+faManagedSanitizeMode+"\n") {
t.Fatalf("manifest sanitize block:\n%s", text)
}
}
+4 -4
View File
@@ -1,14 +1,14 @@
module git.privatae.ai/privatae/privatae-bastion module git.privatae.ai/privatae/privatae-bastion
go 1.25.0 go 1.26.0
require ( require (
filippo.io/age v1.3.2 filippo.io/age v1.3.2
fyne.io/systray v1.12.2 fyne.io/systray v1.12.2
github.com/creack/pty v1.1.24 github.com/creack/pty v1.1.24
github.com/gorilla/websocket v1.5.3 github.com/gorilla/websocket v1.5.3
golang.org/x/crypto v0.55.0 golang.org/x/crypto v0.58.0
golang.org/x/sys v0.47.0 golang.org/x/sys v0.49.0
golang.zx2c4.com/wireguard v0.0.0-20260522210424-ecfc5a8d5446 golang.zx2c4.com/wireguard v0.0.0-20260522210424-ecfc5a8d5446
gopkg.in/yaml.v3 v3.0.1 gopkg.in/yaml.v3 v3.0.1
rsc.io/qr v0.2.0 rsc.io/qr v0.2.0
@@ -18,7 +18,7 @@ require (
filippo.io/hpke v0.4.0 // indirect filippo.io/hpke v0.4.0 // indirect
github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect
github.com/google/btree v1.1.2 // indirect github.com/google/btree v1.1.2 // indirect
golang.org/x/net v0.57.0 // indirect golang.org/x/net v0.61.0 // indirect
golang.org/x/time v0.7.0 // indirect golang.org/x/time v0.7.0 // indirect
golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 // indirect golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 // indirect
gvisor.dev/gvisor v0.0.0-20250503011706-39ed1f5ac29c // indirect gvisor.dev/gvisor v0.0.0-20250503011706-39ed1f5ac29c // indirect
+8 -8
View File
@@ -14,14 +14,14 @@ github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU=
github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4=
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54=
golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w=
golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= golang.org/x/net v0.61.0 h1:RR+6j/BTBCrggOLugxa+66R8FvoN6MHlzDlZIbN9YFw=
golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= golang.org/x/net v0.61.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI=
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA=
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= golang.org/x/term v0.47.0 h1:cpyOwQkr2Nzxe2p6KnrTG7Ei2wtzjsuSjhpXXepZ/BM=
golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= golang.org/x/term v0.47.0/go.mod h1:efAfAuG6RKABvg1ArGebRkvX6LXul14SM67o2e4mUCY=
golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ= golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ=
golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 h1:B82qJJgjvYKsXS9jeunTOisW56dUokqW/FOteYJJ/yg= golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 h1:B82qJJgjvYKsXS9jeunTOisW56dUokqW/FOteYJJ/yg=
+886 -104
View File
File diff suppressed because it is too large Load Diff
+44
View File
@@ -0,0 +1,44 @@
package main
// The agent's credential for a harness run is a file in a folder mounted into
// the container. No other local user may reach either: as root the bastion
// hands both to the container's user, owner-only; as anyone else it opens them
// for that user, inside a folder only the bastion can enter.
import (
"os"
"path/filepath"
"strconv"
"strings"
"testing"
)
func TestTheHarnessCredentialIsReadableOnlyByTheContainersUser(t *testing.T) {
// A stand-in runtime: on `run` it records the mode and owner of the folder
// the credential folder sits in, of that folder and of the file, then
// fails, so the run stops there.
bin := t.TempDir()
record := filepath.Join(bin, "record")
rt := filepath.Join(bin, "runtime")
script := "#!/bin/sh\n[ \"$1\" = run ] || exit 1\nfor a; do case \"$a\" in *:/home/privatae/.claude) d=${a%:/home/privatae/.claude};; esac; done\n" +
"[ -n \"$d\" ] && stat -c '%a %u %g' \"$(dirname \"$d\")\" \"$d\" \"$d/.credentials.json\" > " + record + "\nexit 1\n"
if err := os.WriteFile(rt, []byte(script), 0o755); err != nil {
t.Fatal(err)
}
spec := harnessSpec{HarnessID: "h1", Image: "harness:test", AgentCreds: `{"claudeAiOauth":{}}`}
if _, err := driveHarnessAgent(rt, t.TempDir(), spec, 1); err == nil {
t.Fatal("the stand-in runtime fails the start; the run must report it")
}
got, err := os.ReadFile(record)
if err != nil {
t.Fatalf("the credential folder was not mounted: %v", err)
}
self := strconv.Itoa(os.Geteuid()) + " " + strconv.Itoa(os.Getegid())
want := "700 " + self + "\n777 " + self + "\n666 " + self
if os.Geteuid() == 0 {
want = "700 " + self + "\n700 10005 10005\n600 10005 10005" // the harness image's user
}
if strings.TrimSpace(string(got)) != want {
t.Fatalf("enclosing folder, folder, then file: want %q, got %q", want, strings.TrimSpace(string(got)))
}
}
@@ -0,0 +1,154 @@
package main
// A git run's copy is bounded by how many entries a tree holds as well as by
// its bytes; it never hands git the run's token in an argument or a config
// file; and it uses nothing of the agent's tree while the agent's container
// may still be running.
import (
"encoding/base64"
"net/http"
"net/http/cgi"
"net/http/httptest"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
)
// Many empty files cost nothing in bytes; each still counts against the
// copy, so a tree of them is refused and nothing is pushed.
func TestAGitRunRefusesATreeOfTooManyEntries(t *testing.T) {
f := newGitRunFixture(t)
f.setAgent(t, `cd "$1"
i=0; while [ $i -lt 40 ]; do mkdir d$i; j=0; while [ $j -lt 100 ]; do : > d$i/f$j; j=$((j+1)); done; i=$((i+1)); done
`)
saved := maxCopiedTreeBytes
maxCopiedTreeBytes = 1 << 20 // 4000 entries at 512 bytes each are 2 MB
t.Cleanup(func() { maxCopiedTreeBytes = saved })
res, code := f.run(t)
if code == 0 || res["ok"] != false {
t.Fatalf("a tree of 4000 empty files was copied under a 1 MiB limit: %v", res)
}
if e, _ := res["error"].(string); !strings.Contains(e, "too large") {
t.Errorf("the error does not say the tree is too large: %q", e)
}
if branchPushed(f) {
t.Error("the branch was pushed")
}
}
// A container the runtime does not confirm removed fails the run before any
// of its tree is copied.
func TestAGitRunFailsWhenTheAgentsContainerIsNotRemoved(t *testing.T) {
f := newGitRunFixture(t)
f.setAgent(t, "cd \"$1\" && echo new > edit.txt\n")
docker := filepath.Join(filepath.Dir(f.agentFile), "docker")
b, err := os.ReadFile(docker)
if err != nil {
t.Fatal(err)
}
failing := strings.Replace(string(b), "ps|pull|rm) exit 0;;", "ps|pull) exit 0;;\nrm) echo 'Error: removal stuck' >&2; exit 1;;", 1)
if failing == string(b) {
t.Fatal("the stand-in runtime changed shape")
}
if err := os.WriteFile(docker, []byte(failing), 0o755); err != nil {
t.Fatal(err)
}
res, code := f.run(t)
if code == 0 || res["ok"] != false {
t.Fatalf("a run whose container was not removed reads ok: %v", res)
}
if e, _ := res["error"].(string); !strings.Contains(e, "could not be removed") || !strings.Contains(e, "removal stuck") {
t.Errorf("the error does not say the container was not removed: %q", e)
}
if branchPushed(f) {
t.Error("the branch was pushed")
}
}
// Against a server that needs the token, the run clones and pushes with the
// token in no git argument and in no clone's config.
func TestAGitRunKeepsTheTokenOutOfEveryArgumentAndConfig(t *testing.T) {
f := newGitRunFixture(t)
f.setAgent(t, "cd \"$1\" && echo new > edit.txt\n")
realGit, err := exec.LookPath("git")
if err != nil {
t.Skip("no git")
}
want := "Basic " + base64.StdEncoding.EncodeToString([]byte("harness:"+gitRunToken))
backend := &cgi.Handler{Path: realGit, Args: []string{"http-backend"}, Root: "/git",
Env: []string{"GIT_PROJECT_ROOT=" + filepath.Dir(f.origin), "GIT_HTTP_EXPORT_ALL=1",
"REMOTE_USER=harness", "GIT_CONFIG_NOSYSTEM=1", "HOME=" + t.TempDir()}}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("Authorization") != want {
w.Header().Set("WWW-Authenticate", `Basic realm="test"`)
http.Error(w, "auth", http.StatusUnauthorized)
return
}
backend.ServeHTTP(w, r)
}))
t.Cleanup(srv.Close)
logs := t.TempDir()
argvLog, cfgLog := filepath.Join(logs, "argv"), filepath.Join(logs, "config")
bin := t.TempDir()
wrapper := "#!/bin/sh\nprintf '%s\\n' \"$*\" >> " + argvLog + "\n" +
"prev=; for a; do if [ \"$prev\" = -C ] && [ -f \"$a/.git/config\" ]; then cat \"$a/.git/config\" >> " + cfgLog + "; fi; prev=$a; done\n" +
"exec " + realGit + " \"$@\"\n"
if err := os.WriteFile(filepath.Join(bin, "git"), []byte(wrapper), 0o755); err != nil {
t.Fatal(err)
}
t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH"))
repoURL := strings.Replace(srv.URL, "http://", "http://harness:"+gitRunToken+"@", 1) + "/git/" + filepath.Base(f.origin)
b, code := runGit(harnessSpec{Mode: "git", HarnessID: "h1", Image: "harness:test",
RepoURL: repoURL, Branch: "agent-work", MaxMinutes: 1})
if code != 0 {
t.Fatalf("the run failed: %s", b)
}
if !branchPushed(f) {
t.Fatal("the branch is not in the origin")
}
argv, _ := os.ReadFile(argvLog)
if !strings.Contains(string(argv), "push origin agent-work") {
t.Fatalf("the stand-in git did not see the push:\n%s", argv)
}
cfg, _ := os.ReadFile(cfgLog)
if len(cfg) == 0 {
t.Fatal("no clone's config was read")
}
for name, body := range map[string][]byte{"an argument": argv, "a clone's config": cfg} {
for _, secret := range []string{gitRunToken, strings.TrimPrefix(want, "Basic ")} {
if strings.Contains(string(body), secret) {
t.Errorf("the token is in %s:\n%s", name, body)
}
}
}
}
// Every git call carries the harness identity: nothing of this machine's user
// or host name reaches the clone the agent is handed, and the commit is the
// harness's.
func TestAGitRunsGitCarriesTheHarnessIdentity(t *testing.T) {
f := newGitRunFixture(t)
reflog := filepath.Join(t.TempDir(), "reflog")
f.setAgent(t, "cp \"$1/.git/logs/HEAD\" "+reflog+" && cd \"$1\" && echo new > edit.txt\n")
res, code := f.run(t)
if code != 0 {
t.Fatalf("the run failed: %v", res)
}
b, err := os.ReadFile(reflog)
if err != nil {
t.Fatal(err)
}
host, _ := os.Hostname()
if !strings.Contains(string(b), " harness <harness@privatae.ai> ") || (host != "" && strings.Contains(string(b), host)) {
t.Errorf("the agent's clone records another identity:\n%s", b)
}
who := strings.TrimSpace(testGit(t, f.origin, "log", "-1", "--format=%an <%ae> / %cn <%ce>", "agent-work"))
if who != "harness <harness@privatae.ai> / harness <harness@privatae.ai>" {
t.Errorf("the commit is by %q", who)
}
}
@@ -0,0 +1,360 @@
package main
// What a git run hands the agent, how it ends when its commit cannot be made,
// and the git-lfs filter it supplies itself.
import (
"bufio"
"bytes"
"fmt"
"io"
"io/fs"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"testing"
)
const gitRunToken = "tok3n0f7hisrun5a1e"
// The agent's clone names the repository without the run's token, and the
// token is in no file of the tree it is handed; the run still pushes.
func TestAGitRunHandsTheAgentNoToken(t *testing.T) {
f := newGitRunFixture(t)
snap := filepath.Join(t.TempDir(), "agent-saw")
f.setAgent(t, "cp -a \"$1\" "+snap+" && cd \"$1\" && echo new > edit.txt\n")
b, code := runGit(harnessSpec{Mode: "git", HarnessID: "h1", Image: "harness:test",
RepoURL: "file://harness:" + gitRunToken + "@" + f.origin, Branch: "agent-work", MaxMinutes: 1})
if code != 0 {
t.Fatalf("the run failed: %s", b)
}
if bytes.Contains(b, []byte(gitRunToken)) {
t.Errorf("the result carries the token: %s", b)
}
n := 0
err := filepath.WalkDir(snap, func(p string, d fs.DirEntry, err error) error {
if err != nil || !d.Type().IsRegular() {
return err
}
n++
body, err := os.ReadFile(p)
if err == nil && bytes.Contains(body, []byte(gitRunToken)) {
t.Errorf("the token is in the agent's tree, in %s", p[len(snap):])
}
return err
})
if err != nil || n == 0 {
t.Fatalf("the agent's tree was not read (%d files): %v", n, err)
}
if got := testGit(t, f.origin, "show", "agent-work:edit.txt"); got != "new\n" {
t.Errorf("edit.txt = %q", got)
}
}
func TestRepoURLPartsAndLFSPins(t *testing.T) {
bare, secret, err := repoURLParts("https://host-user:" + gitRunToken + "@git.example.invalid/o/r.git")
if err != nil || bare != "https://git.example.invalid/o/r.git" || secret != gitRunToken {
t.Errorf("parts = %q, %q, %v", bare, secret, err)
}
if bare, secret, err := repoURLParts("/srv/x/origin.git"); err != nil || bare != "/srv/x/origin.git" || secret != "" {
t.Errorf("a path = %q, %q, %v", bare, secret, err)
}
pins := strings.Join(lfsEndpointPins("https://u:"+gitRunToken+"@git.example.invalid/o/r"), " ")
for _, k := range []string{"lfs.url", "lfs.pushurl", "remote.origin.lfsurl", "remote.origin.lfspushurl"} {
if !strings.Contains(pins, k+"=https://u:"+gitRunToken+"@git.example.invalid/o/r.git/info/lfs") {
t.Errorf("%s is not pinned to the repository's own LFS address: %s", k, pins)
}
}
if p := lfsEndpointPins("/srv/x/origin.git"); p != nil {
t.Errorf("a path got pins: %v", p)
}
}
// standInGit puts a git on PATH that fails on the subcommand fail and is the
// real git otherwise.
func standInGit(t *testing.T, fail string) {
t.Helper()
real, err := exec.LookPath("git")
if err != nil {
t.Skip("no git")
}
bin := t.TempDir()
script := "#!/bin/sh\nfor a; do [ \"$a\" = " + fail + " ] && { echo 'stand-in: " + fail + " failed' >&2; exit 128; }; done\nexec " + real + " \"$@\"\n"
if err := os.WriteFile(filepath.Join(bin, "git"), []byte(script), 0o755); err != nil {
t.Fatal(err)
}
t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH"))
}
func branchPushed(f gitRunFixture) bool {
return exec.Command("git", "--git-dir", f.origin, "rev-parse", "--verify", "-q", "agent-work").Run() == nil
}
// A commit that fails is the run failing: nothing is pushed.
func TestAGitRunWhoseCommitFailsPushesNothing(t *testing.T) {
f := newGitRunFixture(t)
f.setAgent(t, "cd \"$1\" && echo new > edit.txt\n")
standInGit(t, "commit")
res, code := f.run(t)
if code == 0 || res["ok"] != false {
t.Errorf("a run whose commit failed reads ok: code %d, %v", code, res)
}
if e, _ := res["error"].(string); !strings.Contains(e, "commit") || !strings.Contains(e, "nothing was pushed") {
t.Errorf("the error does not say the commit failed: %q", e)
}
if branchPushed(f) {
t.Error("the branch was pushed")
}
}
// A run that changed nothing is ok, says so, and pushes no branch.
func TestAGitRunThatChangesNothingPushesNothing(t *testing.T) {
f := newGitRunFixture(t)
f.setAgent(t, "true\n")
res, code := f.run(t)
if code != 0 || res["ok"] != true || res["changed"] != false {
t.Errorf("want ok with changed false: code %d, %v", code, res)
}
if _, ok := res["branch"]; ok {
t.Errorf("the result names a branch: %v", res["branch"])
}
if branchPushed(f) {
t.Error("a branch was pushed for a run that changed nothing")
}
}
// Names a file system may open as .git are left out of the copy, and the
// rest of the run goes on.
func TestAGitRunLeavesOutEveryNameThatOpensAsDotGit(t *testing.T) {
f := newGitRunFixture(t)
names := []string{".git.", ".GIT ", "git~1", "GIT~1. ", ".g\u200cit", ".git::$INDEX_ALLOCATION"}
script := "cd \"$1\" && echo new > edit.txt\n"
for _, n := range names {
script += "mkdir '" + n + "' && echo '[core]' > '" + n + "/config'\n"
}
f.setAgent(t, script)
res, code := f.run(t)
if code != 0 || res["ok"] != true {
t.Fatalf("the run failed: code %d, %v", code, res)
}
files := testGit(t, f.origin, "ls-tree", "-r", "--name-only", "-z", "agent-work")
for _, n := range names {
if strings.Contains(files, n+"/") {
t.Errorf("%q is in the commit", n)
}
}
skipped, _ := res["worktree_skipped"].([]any)
listed := map[string]bool{}
for _, s := range skipped {
listed[fmt.Sprint(s)] = true
}
for _, n := range names {
if !listed[n] {
t.Errorf("%q is not listed as left out: %v", n, skipped)
}
}
for _, n := range []string{".git", ".Git", ".git.", ".git . .", "GiT~1", ".gi\ufefft", ".git:x"} {
if !dotGitName(n) {
t.Errorf("dotGitName(%q) = false", n)
}
}
for _, n := range []string{".gitignore", ".gitattributes", "git", ".github", "git~2", "x.git"} {
if dotGitName(n) {
t.Errorf("dotGitName(%q) = true", n)
}
}
}
// With git-lfs installed, the bastion's own git calls clean and smudge
// through it and push its objects; the agent's .git cannot replace it, and a
// filter of another name the agent's .gitattributes names runs nothing.
func TestAGitRunUsesTheLFSFilterOnlyFromThisProcess(t *testing.T) {
f := newGitRunFixture(t)
seed := filepath.Join(t.TempDir(), "s")
testGit(t, "", "clone", "-q", f.origin, seed)
if err := os.WriteFile(filepath.Join(seed, ".gitattributes"), []byte("*.bin filter=lfs -text\n"), 0o644); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(seed, "big.bin"), []byte("v1\n"), 0o644); err != nil {
t.Fatal(err)
}
testGit(t, seed, "add", "-A")
testGit(t, seed, "commit", "-q", "-m", "lfs")
testGit(t, seed, "push", "-q", "origin", "main")
bin := t.TempDir()
lfsLog := filepath.Join(t.TempDir(), "lfs.log")
wrapper := "#!/bin/sh\nGO_HELPER_GITLFS=1 GO_HELPER_GITLFS_LOG=" + lfsLog + " exec " + os.Args[0] +
" -test.run='^TestHelperGitLFS$' -- \"$@\"\n"
if err := os.WriteFile(filepath.Join(bin, "git-lfs"), []byte(wrapper), 0o755); err != nil {
t.Fatal(err)
}
t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH"))
mark := func(name string) string { return "touch " + filepath.Join(f.marks, name) }
f.setAgent(t, `set -e
cd "$1"
git config filter.lfs.process '`+mark("agent-lfs-process")+`; exit 1'
git config filter.lfs.clean '`+mark("agent-lfs-clean")+`; cat'
git config filter.evil.clean '`+mark("evil")+`; cat'
printf '*.bin filter=lfs -text\n*.txt filter=evil\n' > .gitattributes
echo v2 > big.bin
echo new > edit.txt
`)
res, code := f.run(t)
if code != 0 || res["ok"] != true {
t.Fatalf("the run failed: code %d, %v", code, res)
}
ents, _ := os.ReadDir(f.marks)
for _, e := range ents {
t.Errorf("the agent's %s ran as this process", e.Name())
}
logged, _ := os.ReadFile(lfsLog)
for _, want := range []string{"smudge big.bin", "clean big.bin", "push origin agent-work"} {
if !strings.Contains(string(logged), want+"\n") {
t.Errorf("git-lfs did not %s; it logged:\n%s", want, logged)
}
}
for _, line := range strings.Split(string(logged), "\n") {
if strings.HasSuffix(line, ".txt") {
t.Errorf("git-lfs ran for a file of another filter: %s", line)
}
}
if got := testGit(t, f.origin, "show", "agent-work:big.bin"); got != "v2\n" {
t.Errorf("big.bin = %q", got)
}
}
// TestHelperGitLFS is the stand-in git-lfs: run by the wrapper above, it is
// an identity filter that logs what it was asked to do.
func TestHelperGitLFS(t *testing.T) {
if os.Getenv("GO_HELPER_GITLFS") != "1" {
return
}
args := os.Args
for i, a := range args {
if a == "--" {
args = args[i+1:]
break
}
}
logf, err := os.OpenFile(os.Getenv("GO_HELPER_GITLFS_LOG"), os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0o644)
if err != nil {
os.Exit(2)
}
switch {
case len(args) > 0 && args[0] == "filter-process":
if err := helperFilterProcess(logf); err != nil {
fmt.Fprintln(os.Stderr, "stand-in git-lfs:", err)
os.Exit(1)
}
case len(args) > 0 && (args[0] == "clean" || args[0] == "smudge"):
fmt.Fprintf(logf, "%s %s\n", args[0], args[len(args)-1])
_, _ = io.Copy(os.Stdout, os.Stdin)
default:
fmt.Fprintln(logf, strings.Join(args, " "))
}
logf.Close()
os.Exit(0)
}
func pktRead(r *bufio.Reader) ([]byte, bool, error) {
var h [4]byte
if _, err := io.ReadFull(r, h[:]); err != nil {
return nil, false, err
}
n, err := strconv.ParseUint(string(h[:]), 16, 16)
if err != nil {
return nil, false, err
}
if n == 0 {
return nil, true, nil
}
b := make([]byte, n-4)
_, err = io.ReadFull(r, b)
return b, false, err
}
func pktList(r *bufio.Reader) ([]string, error) {
var out []string
for {
b, flush, err := pktRead(r)
if err != nil || flush {
return out, err
}
out = append(out, strings.TrimSuffix(string(b), "\n"))
}
}
func pktWrite(w io.Writer, lines ...string) {
for _, l := range lines {
fmt.Fprintf(w, "%04x%s", len(l)+4, l)
}
_, _ = w.Write([]byte("0000"))
}
// helperFilterProcess speaks git's long-running filter protocol, version 2,
// returning every file as it came.
func helperFilterProcess(logf io.Writer) error {
r, w := bufio.NewReader(os.Stdin), bufio.NewWriter(os.Stdout)
if _, err := pktList(r); err != nil {
return err
}
pktWrite(w, "git-filter-server\n", "version=2\n")
if err := w.Flush(); err != nil {
return err
}
if _, err := pktList(r); err != nil {
return err
}
pktWrite(w, "capability=clean\n", "capability=smudge\n")
if err := w.Flush(); err != nil {
return err
}
for {
head, err := pktList(r)
if err == io.EOF {
return nil
}
if err != nil {
return err
}
cmd, path := "", ""
for _, kv := range head {
k, v, _ := strings.Cut(kv, "=")
switch k {
case "command":
cmd = v
case "pathname":
path = v
}
}
var body bytes.Buffer
for {
b, flush, err := pktRead(r)
if err != nil {
return err
}
if flush {
break
}
body.Write(b)
}
fmt.Fprintf(logf, "%s %s\n", cmd, path)
pktWrite(w, "status=success\n")
data := body.Bytes()
for len(data) > 0 {
n := min(len(data), 65516)
fmt.Fprintf(w, "%04x", n+4)
_, _ = w.Write(data[:n])
data = data[n:]
}
_, _ = w.Write([]byte("0000"))
pktWrite(w)
if err := w.Flush(); err != nil {
return err
}
}
}
+355
View File
@@ -0,0 +1,355 @@
package main
// A git run's tree is the agent's, .git included. These tests stand in for an
// agent that uses that .git against the bastion: hooks, a file system
// monitor, a filter, a signing program, a changed remote and a link to a file
// outside the tree, besides ordinary edits. The container runtime is a
// stand-in script that does the agent's work on the mounted tree; the sidecar
// it would start is a local HTTP server on the sidecar's port.
import (
"encoding/json"
"net"
"net/http"
"os"
"os/exec"
"path/filepath"
"sort"
"strings"
"testing"
"time"
)
const gitRunOutsideSecret = "outside-the-tree-7c41"
// gitRunFixture is an origin with one commit, a decoy repository, a file
// outside the tree, a folder the agent's code marks when it runs, and a
// stand-in runtime that runs agent (a shell script, given the tree's path as
// $1) when the run starts its container.
type gitRunFixture struct {
origin, decoy, outside, marks, agentFile string
}
// setAgent writes the script the stand-in runtime runs as the agent.
func (f gitRunFixture) setAgent(t *testing.T, script string) {
t.Helper()
if err := os.WriteFile(f.agentFile, []byte(script), 0o755); err != nil {
t.Fatal(err)
}
}
func testGit(t *testing.T, dir string, args ...string) string {
t.Helper()
cmd := exec.Command("git", args...)
cmd.Dir = dir
out, err := cmd.CombinedOutput()
if err != nil {
t.Fatalf("git %v: %v: %s", args, err, out)
}
return string(out)
}
func newGitRunFixture(t *testing.T) gitRunFixture {
t.Helper()
if _, err := exec.LookPath("git"); err != nil {
t.Skip("no git")
}
tmp := t.TempDir()
// The test's own git, and git as the code before this change ran it, read
// no config of this machine's.
home := filepath.Join(tmp, "home")
if err := os.Mkdir(home, 0o700); err != nil {
t.Fatal(err)
}
t.Setenv("HOME", home)
t.Setenv("XDG_CONFIG_HOME", home)
t.Setenv("GIT_CONFIG_NOSYSTEM", "1")
t.Setenv("GIT_AUTHOR_NAME", "seed")
t.Setenv("GIT_AUTHOR_EMAIL", "seed@example.invalid")
t.Setenv("GIT_COMMITTER_NAME", "seed")
t.Setenv("GIT_COMMITTER_EMAIL", "seed@example.invalid")
f := gitRunFixture{
origin: filepath.Join(tmp, "origin.git"),
decoy: filepath.Join(tmp, "decoy.git"),
outside: filepath.Join(tmp, "outside", "secret"),
marks: filepath.Join(tmp, "marks"),
}
for _, d := range []string{filepath.Dir(f.outside), f.marks} {
if err := os.Mkdir(d, 0o755); err != nil {
t.Fatal(err)
}
}
if err := os.WriteFile(f.outside, []byte(gitRunOutsideSecret), 0o600); err != nil {
t.Fatal(err)
}
testGit(t, tmp, "init", "-q", "--bare", "-b", "main", f.origin)
testGit(t, tmp, "init", "-q", "--bare", "-b", "main", f.decoy)
seed := filepath.Join(tmp, "seed")
testGit(t, tmp, "init", "-q", "-b", "main", seed)
for name, body := range map[string]string{"keep.txt": "keep\n", "edit.txt": "old\n", "gone.txt": "bye\n"} {
if err := os.WriteFile(filepath.Join(seed, name), []byte(body), 0o644); err != nil {
t.Fatal(err)
}
}
if err := os.WriteFile(filepath.Join(seed, "tool.sh"), []byte("#!/bin/sh\n"), 0o755); err != nil {
t.Fatal(err)
}
testGit(t, seed, "add", "-A")
testGit(t, seed, "commit", "-q", "-m", "seed")
testGit(t, seed, "push", "-q", f.origin, "main")
bin := t.TempDir()
f.agentFile = filepath.Join(bin, "agent.sh")
agentFile := f.agentFile
runtime := "#!/bin/sh\ncase \"$1\" in\nps|pull|rm) exit 0;;\ninspect) echo 127.0.0.1; exit 0;;\n" +
"run) for a; do case \"$a\" in *:/work) d=${a%:/work};; esac; done\n" +
" /bin/sh " + agentFile + " \"$d\" > " + filepath.Join(bin, "agent.log") + " 2>&1 || exit 1\n" +
" echo standin0123456789; exit 0;;\nesac\nexit 1\n"
if err := os.WriteFile(filepath.Join(bin, "docker"), []byte(runtime), 0o755); err != nil {
t.Fatal(err)
}
t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH"))
t.Cleanup(func() {
if t.Failed() {
if b, err := os.ReadFile(filepath.Join(bin, "agent.log")); err == nil {
t.Logf("agent log:\n%s", b)
}
}
})
// The sidecar the run drives: it answers, and /run reports success.
ln, err := net.Listen("tcp", "127.0.0.1:8844")
if err != nil {
t.Fatalf("the sidecar's port is taken: %v", err)
}
srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/run" {
_, _ = w.Write([]byte(`{"ok":true,"transcript":"done"}`))
}
})}
go func() { _ = srv.Serve(ln) }()
t.Cleanup(func() { _ = srv.Close() })
return f
}
func (f gitRunFixture) run(t *testing.T) (map[string]any, int) {
t.Helper()
b, code := runGit(harnessSpec{Mode: "git", HarnessID: "h1", Image: "harness:test",
RepoURL: f.origin, Branch: "agent-work", MaxMinutes: 1})
var res map[string]any
if err := json.Unmarshal(b, &res); err != nil {
t.Fatalf("result: %v: %s", err, b)
}
return res, code
}
// theAgent turns its tree's .git against whoever runs git in it next, and
// makes ordinary changes to the files.
func theAgent(f gitRunFixture) string {
mark := func(name string) string { return "touch " + filepath.Join(f.marks, name) }
return `set -e
d="$1"; cd "$d"
g="$d/.git"
printf '#!/bin/sh\n` + mark("pre-commit") + `\n' > "$g/hooks/pre-commit"
printf '#!/bin/sh\n` + mark("post-commit") + `\n' > "$g/hooks/post-commit"
chmod +x "$g/hooks/pre-commit" "$g/hooks/post-commit"
printf '#!/bin/sh\n` + mark("fsmonitor") + `\nexit 1\n' > "$g/fsmon.sh"; chmod +x "$g/fsmon.sh"
git config core.fsmonitor "$g/fsmon.sh"
printf '* filter=evil\n' > .gitattributes
git config filter.evil.clean '` + mark("filter") + `; cat ` + f.outside + `'
git config filter.evil.smudge cat
printf '#!/bin/sh\n` + mark("gpg") + `\necho >&2; echo "[GNUPG:] SIG_CREATED D 1 8 00 0 X" >&2\necho "-----BEGIN PGP SIGNATURE-----"; echo x; echo "-----END PGP SIGNATURE-----"\n' > "$g/gpg.sh"
chmod +x "$g/gpg.sh"
git config commit.gpgsign true
git config gpg.program "$g/gpg.sh"
git config remote.origin.url ` + f.decoy + `
ln -s ` + f.outside + ` link-to-outside
echo new > edit.txt
echo added > added.txt
rm gone.txt
mkdir sub; echo deep > sub/deep.txt
`
}
func TestAGitRunNeverRunsGitInTheAgentsTree(t *testing.T) {
f := newGitRunFixture(t)
f.setAgent(t, theAgent(f))
res, code := f.run(t)
if code != 0 || res["ok"] != true {
t.Errorf("the run failed: code %d, %v", code, res)
}
// Nothing the agent set in its .git ran.
ents, _ := os.ReadDir(f.marks)
for _, e := range ents {
t.Errorf("the agent's %s ran as this process", e.Name())
}
// The push went to the repository the run was given, not the one the
// agent named.
if refs := strings.TrimSpace(testGit(t, f.decoy, "for-each-ref")); refs != "" {
t.Errorf("the push went to the agent's remote: %s", refs)
}
out, err := exec.Command("git", "--git-dir", f.origin, "rev-parse", "--verify", "-q", "agent-work").Output()
if err != nil {
t.Fatalf("the branch is not in the origin: %v", err)
}
tip := strings.TrimSpace(string(out))
if n := strings.TrimSpace(testGit(t, f.origin, "rev-list", "--count", "main.."+tip)); n != "1" {
t.Errorf("the branch carries %s commits on main, want 1", n)
}
// Exactly the agent's file changes: the link is not there, and the outside
// file's content is in no file of the commit.
files := strings.Fields(testGit(t, f.origin, "ls-tree", "-r", "--format=%(objectmode) %(path)", tip))
got := map[string]string{}
for i := 0; i+1 < len(files); i += 2 {
got[files[i+1]] = files[i]
}
want := map[string]string{".gitattributes": "100644", "added.txt": "100644", "edit.txt": "100644",
"keep.txt": "100644", "sub/deep.txt": "100644", "tool.sh": "100755"}
if !sameMap(got, want) {
t.Errorf("the commit holds %v, want %v", got, want)
}
for path := range got {
body := testGit(t, f.origin, "show", tip+":"+path)
if strings.Contains(body, gitRunOutsideSecret) {
t.Errorf("%s in the commit holds the file outside the tree", path)
}
}
for path, body := range map[string]string{"edit.txt": "new\n", "added.txt": "added\n", "keep.txt": "keep\n"} {
if _, ok := got[path]; ok {
if b := testGit(t, f.origin, "show", tip+":"+path); b != body {
t.Errorf("%s = %q, want %q", path, b, body)
}
}
}
if s, _ := res["worktree_skipped"].([]any); len(s) != 1 || s[0] != "link-to-outside" {
t.Errorf("left out = %v, want [link-to-outside]", res["worktree_skipped"])
}
for _, k := range []string{"branch", "commit", "commit_out"} {
if _, ok := res[k]; !ok {
t.Errorf("the result has no %s", k)
}
}
if res["commit"] != tip {
t.Errorf("the result names commit %v, the branch is at %s", res["commit"], tip)
}
}
// A link the repository itself holds stays as it is when the agent leaves it
// alone: a run that cannot copy a link must not delete it.
func TestAGitRunKeepsALinkTheRepositoryHolds(t *testing.T) {
f := newGitRunFixture(t)
seed := filepath.Join(t.TempDir(), "s")
testGit(t, "", "clone", "-q", f.origin, seed)
if err := os.Symlink("keep.txt", filepath.Join(seed, "alias")); err != nil {
t.Fatal(err)
}
testGit(t, seed, "add", "-A")
testGit(t, seed, "commit", "-q", "-m", "a link")
testGit(t, seed, "push", "-q", "origin", "main")
f.setAgent(t, "cd \"$1\" && echo new > edit.txt\n")
res, code := f.run(t)
if code != 0 {
t.Fatalf("the run failed: %v", res)
}
if mode := strings.Fields(testGit(t, f.origin, "ls-tree", "agent-work", "alias")); len(mode) == 0 || mode[0] != "120000" {
t.Errorf("the repository's link is gone from the branch: %v", mode)
}
if diff := strings.Fields(testGit(t, f.origin, "diff", "--name-only", "main", "agent-work")); len(diff) != 1 || diff[0] != "edit.txt" {
t.Errorf("the branch changes %v, want [edit.txt]", diff)
}
}
// A file far larger than the copy may take (sparse, so it costs nothing to
// make) is refused from its size, before a byte of it is read: the run ends
// at once, pushes nothing, and says why.
func TestAGitRunRefusesATreeLargerThanItCopies(t *testing.T) {
f := newGitRunFixture(t)
f.setAgent(t, "cd \"$1\" && truncate -s 64G huge.bin && echo new > edit.txt\n")
type out struct {
res map[string]any
code int
}
ch := make(chan out, 1)
start := time.Now()
go func() { r, c := f.run(t); ch <- out{r, c} }()
var o out
select {
case o = <-ch:
case <-time.After(30 * time.Second):
t.Fatal("a run with a 64 GiB file was still going after 30 s")
}
t.Logf("returned in %v", time.Since(start))
if o.code == 0 || o.res["ok"] != false {
t.Fatalf("a tree over the cap was pushed: %v", o.res)
}
if e, _ := o.res["error"].(string); !strings.Contains(e, "huge.bin") || !strings.Contains(e, "too large") {
t.Errorf("the error does not say what was too large: %q", e)
}
if err := exec.Command("git", "--git-dir", f.origin, "rev-parse", "--verify", "-q", "agent-work").Run(); err == nil {
t.Error("the branch was pushed")
}
}
// The tree a worktree run returns is refused the same way, and no part of
// it is returned.
func TestAWorktreeRunRefusesATreeLargerThanTheLaneCarries(t *testing.T) {
f := newGitRunFixture(t)
f.setAgent(t, "cd \"$1\" && truncate -s 64G huge.bin\n")
ch := make(chan []byte, 1)
start := time.Now()
go func() {
b, _ := runWorktree(harnessSpec{Mode: "worktree", HarnessID: "h1", Image: "harness:test", MaxMinutes: 1})
ch <- b
}()
var b []byte
select {
case b = <-ch:
case <-time.After(30 * time.Second):
t.Fatal("a run with a 64 GiB file was still going after 30 s")
}
t.Logf("returned in %v", time.Since(start))
var res map[string]any
if err := json.Unmarshal(b, &res); err != nil {
t.Fatal(err)
}
if res["ok"] != false || res["worktree_b64"] != nil {
t.Fatalf("a tree over the cap was returned: ok=%v, %d bytes of tree", res["ok"], len(b))
}
if e, _ := res["error"].(string); !strings.Contains(e, "huge.bin") || !strings.Contains(e, "too large") {
t.Errorf("the error does not say what was too large: %q", e)
}
}
func sameMap(a, b map[string]string) bool {
if len(a) != len(b) {
return false
}
ka := make([]string, 0, len(a))
for k := range a {
ka = append(ka, k)
}
sort.Strings(ka)
for _, k := range ka {
if b[k] != a[k] {
return false
}
}
return true
}
// A repository with no commit yet gets the run's commit as its first.
func TestAGitRunIntoAnEmptyRepositoryPushesItsFirstCommit(t *testing.T) {
f := newGitRunFixture(t)
empty := filepath.Join(t.TempDir(), "empty.git")
testGit(t, "", "init", "-q", "--bare", "-b", "main", empty)
f.setAgent(t, "cd \"$1\" && echo first > first.txt\n")
b, code := runGit(harnessSpec{Mode: "git", HarnessID: "h1", Image: "harness:test",
RepoURL: empty, Branch: "agent-work", MaxMinutes: 1})
if code != 0 {
t.Fatalf("the run failed: %s", b)
}
if got := testGit(t, empty, "show", "agent-work:first.txt"); got != "first\n" {
t.Errorf("first.txt = %q", got)
}
}
@@ -0,0 +1,143 @@
package main
// A git run against a repository that keeps a file in Git LFS, with the real
// git-lfs. It runs only where git-lfs is installed; every server it talks to
// is in this test, on loopback.
import (
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"net/http"
"net/http/cgi"
"net/http/httptest"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"testing"
)
func lfsPointer(body string) (oid, pointer string) {
sum := sha256.Sum256([]byte(body))
oid = hex.EncodeToString(sum[:])
return oid, fmt.Sprintf("version https://git-lfs.github.com/spec/v1\noid sha256:%s\nsize %d\n", oid, len(body))
}
func TestAGitRunWithTheRealGitLFS(t *testing.T) {
if out, err := exec.Command("git-lfs", "version").Output(); err != nil || !strings.HasPrefix(string(out), "git-lfs/") {
t.Skip("git-lfs is not installed")
}
f := newGitRunFixture(t)
realGit, _ := exec.LookPath("git")
want := "Basic " + base64.StdEncoding.EncodeToString([]byte("harness:"+gitRunToken))
var mu sync.Mutex
store := map[string][]byte{}
v1oid, v1ptr := lfsPointer("v1\n")
store[v1oid] = []byte("v1\n")
var srvURL string
backend := &cgi.Handler{Path: realGit, Args: []string{"http-backend"}, Root: "/git",
Env: []string{"GIT_PROJECT_ROOT=" + filepath.Dir(f.origin), "GIT_HTTP_EXPORT_ALL=1",
"REMOTE_USER=harness", "GIT_CONFIG_NOSYSTEM=1", "HOME=" + t.TempDir()}}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if strings.HasPrefix(r.URL.Path, "/store/") { // the object hrefs the batch answer gives
oid := strings.TrimPrefix(r.URL.Path, "/store/")
mu.Lock()
defer mu.Unlock()
if r.Method == http.MethodPut {
b, _ := io.ReadAll(r.Body)
store[oid] = b
return
}
_, _ = w.Write(store[oid])
return
}
if r.Header.Get("Authorization") != want {
w.Header().Set("WWW-Authenticate", `Basic realm="test"`)
http.Error(w, "auth", http.StatusUnauthorized)
return
}
if !strings.Contains(r.URL.Path, "/info/lfs/") {
backend.ServeHTTP(w, r)
return
}
if !strings.HasSuffix(r.URL.Path, "/objects/batch") {
http.NotFound(w, r) // locks: not supported here
return
}
var req struct {
Operation string `json:"operation"`
Objects []struct {
Oid string `json:"oid"`
Size int64 `json:"size"`
} `json:"objects"`
}
_ = json.NewDecoder(r.Body).Decode(&req)
var objs []map[string]any
mu.Lock()
for _, o := range req.Objects {
obj := map[string]any{"oid": o.Oid, "size": o.Size, "authenticated": true}
_, have := store[o.Oid]
if req.Operation == "download" || !have {
obj["actions"] = map[string]any{req.Operation: map[string]any{"href": srvURL + "/store/" + o.Oid}}
}
objs = append(objs, obj)
}
mu.Unlock()
w.Header().Set("Content-Type", "application/vnd.git-lfs+json")
_ = json.NewEncoder(w).Encode(map[string]any{"transfer": "basic", "objects": objs})
}))
t.Cleanup(srv.Close)
srvURL = srv.URL
var foreignHits atomic.Int64
foreign := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
foreignHits.Add(1)
http.Error(w, "no", http.StatusForbidden)
}))
t.Cleanup(foreign.Close)
seed := filepath.Join(t.TempDir(), "s")
testGit(t, "", "clone", "-q", f.origin, seed)
for name, body := range map[string]string{".gitattributes": "*.bin filter=lfs diff=lfs merge=lfs -text\n", "big.bin": v1ptr} {
if err := os.WriteFile(filepath.Join(seed, name), []byte(body), 0o644); err != nil {
t.Fatal(err)
}
}
testGit(t, seed, "add", "-A")
testGit(t, seed, "commit", "-q", "-m", "an lfs file")
testGit(t, seed, "push", "-q", "origin", "main")
// The agent changes the large file and names a foreign LFS server.
f.setAgent(t, `set -e
cd "$1"
grep -qx v1 big.bin
echo v2 > big.bin
printf '[lfs]\n\turl = `+foreign.URL+`/x.git/info/lfs\n\tpushurl = `+foreign.URL+`/x.git/info/lfs\n[remote "origin"]\n\tlfsurl = `+foreign.URL+`/y\n' > .lfsconfig
`)
repoURL := strings.Replace(srv.URL, "http://", "http://harness:"+gitRunToken+"@", 1) + "/git/" + filepath.Base(f.origin)
b, code := runGit(harnessSpec{Mode: "git", HarnessID: "h1", Image: "harness:test",
RepoURL: repoURL, Branch: "agent-work", MaxMinutes: 1})
t.Logf("result: %s", b)
if code != 0 {
t.Fatalf("the run failed")
}
v2oid, v2ptr := lfsPointer("v2\n")
if got := testGit(t, f.origin, "show", "agent-work:big.bin"); got != v2ptr {
t.Errorf("big.bin on the branch = %q, want the pointer to v2", got)
}
mu.Lock()
got := string(store[v2oid])
mu.Unlock()
if got != "v2\n" {
t.Errorf("the LFS object for v2 was not sent to the repository's own LFS server (%q)", got)
}
if n := foreignHits.Load(); n != 0 {
t.Errorf("the foreign LFS server named in .lfsconfig got %d requests", n)
}
}
+43
View File
@@ -0,0 +1,43 @@
package main
import (
"errors"
"strconv"
"golang.org/x/sys/unix"
)
// openTreeEntry opens the entry name in the folder dfd with flags, for
// packFolder.
//
// A folder is opened directly: with O_DIRECTORY the kernel refuses anything
// that is not a folder before it opens it. Anything else is first opened as a
// path only (O_PATH), which runs no driver code, so a device, pipe or terminal
// swapped in since the look is never opened: opening some devices does
// something by itself (a watchdog arms, a tape rewinds on close). Only a plain
// file is then opened for reading, through /proc/self/fd, which reaches the
// file the path fd holds and no other.
func openTreeEntry(dfd int, name string, flags int) (int, error) {
if flags&unix.O_DIRECTORY != 0 {
return unix.Openat(dfd, name, flags, 0)
}
pfd, err := unix.Openat(dfd, name, unix.O_PATH|unix.O_NOFOLLOW|unix.O_CLOEXEC, 0)
if err != nil {
return -1, err
}
defer unix.Close(pfd)
var st unix.Stat_t
if err := unix.Fstat(pfd, &st); err != nil {
return -1, err
}
if st.Mode&unix.S_IFMT != unix.S_IFREG {
return -1, errNotPlainFile
}
fd, err := unix.Open("/proc/self/fd/"+strconv.Itoa(pfd), flags&^unix.O_NOFOLLOW, 0)
if errors.Is(err, unix.ENOENT) {
// No /proc mounted: open by name as other systems do. packOpened
// still checks that what was opened is the file that was looked at.
return unix.Openat(dfd, name, flags, 0)
}
return fd, err
}
+70
View File
@@ -0,0 +1,70 @@
package main
import (
"os"
"path/filepath"
"strings"
"testing"
"golang.org/x/sys/unix"
)
// A name swapped for a special file between the walk's look and its open must
// never be opened: opening a device runs its driver (a watchdog arms, a tape
// rewinds on close, a serial line raises DTR), whatever is read after. A named
// pipe stands in for any of them, since anyone can make one, and inotify
// reports every open of it except one made only as a path (O_PATH), which runs
// no driver code. Both a name looked at as a plain file and one looked at as
// a folder are swapped.
func TestMakeTarNeverOpensASpecialFileSwappedInBeforeTheOpen(t *testing.T) {
for _, swap := range []string{"a.txt", "sub"} {
t.Run(swap, func(t *testing.T) {
_, _, tree := outsideAndTree(t)
in, err := unix.InotifyInit1(unix.IN_NONBLOCK | unix.IN_CLOEXEC)
if err != nil {
t.Skipf("no inotify: %v", err)
}
defer unix.Close(in)
watched := false
packTreeBeforeOpen = func(rel string) {
if rel != swap {
return
}
p := filepath.Join(tree, swap)
if err := os.RemoveAll(p); err != nil {
t.Errorf("remove %s: %v", p, err)
return
}
if err := unix.Mkfifo(p, 0o644); err != nil {
t.Errorf("mkfifo %s: %v", p, err)
return
}
if _, err := unix.InotifyAddWatch(in, p, unix.IN_OPEN); err != nil {
t.Errorf("watch %s: %v", p, err)
return
}
watched = true
}
defer func() { packTreeBeforeOpen = nil }()
b, skipped, err := packWorktree(tree)
if err != nil {
t.Fatalf("packWorktree: %v", err)
}
if !watched {
t.Fatal("the swap never ran")
}
buf := make([]byte, 4096)
if n, err := unix.Read(in, buf); n > 0 {
t.Fatalf("the special file swapped in for %s was opened (%d bytes of inotify events)", swap, n)
} else if err != unix.EAGAIN {
t.Fatalf("read inotify: %d %v", n, err)
}
if _, ok := tarEntries(t, b)[swap]; ok {
t.Fatalf("%s was a named pipe when it was opened and must be left out", swap)
}
if !strings.Contains(","+strings.Join(skipped, ",")+",", ","+swap+",") {
t.Fatalf("%s must be named as left out, got %v", swap, skipped)
}
})
}
}
+13
View File
@@ -0,0 +1,13 @@
//go:build !windows && !linux
package main
import "golang.org/x/sys/unix"
// openTreeEntry opens the entry name in the folder dfd with flags, for
// packFolder. These systems have no open that runs no driver code, so a
// special file swapped in since the look is opened (O_NONBLOCK, O_NOCTTY),
// then closed unread by packOpened.
func openTreeEntry(dfd int, name string, flags int) (int, error) {
return unix.Openat(dfd, name, flags, 0)
}
+114
View File
@@ -0,0 +1,114 @@
//go:build !windows
package main
import (
"errors"
"fmt"
"os"
"golang.org/x/sys/unix"
)
// errNotPlainFile is openTreeEntry's refusal of a name that is no longer a
// plain file when it is opened.
var errNotPlainFile = errors.New("not a plain file")
// packTree hands the tree under root to sink, walking it by open folder
// (packWorktree says why). Each name is looked at with fstatat and opened
// relative to the folder above it (openTreeEntry, per system), never
// following a link, and it goes in only if the opened file is still the plain
// file or folder that was looked at (same type, device and inode). A pipe or
// device seen as one is never opened, so the walk cannot wait on one; on
// Linux one swapped in between is not opened either, and elsewhere O_NONBLOCK
// and O_NOCTTY cover it. Each left-out entry goes to sink.skip.
func packTree(sink treeSink, root string) error {
fd, err := unix.Open(root, unix.O_RDONLY|unix.O_DIRECTORY|unix.O_NOFOLLOW|unix.O_CLOEXEC, 0)
if err != nil {
return &os.PathError{Op: "open", Path: root, Err: err}
}
d := os.NewFile(uintptr(fd), root)
defer d.Close()
return packFolder(sink, d, fd, "", 0)
}
func packFolder(sink treeSink, d *os.File, dfd int, rel string, depth int) error {
names, err := readTreeNames(sink, d, rel)
if err != nil {
return err
}
if err := sink.folder(rel, names); err != nil {
return err
}
for _, name := range names {
p := name
if rel != "" {
p = rel + "/" + name
}
var st unix.Stat_t
if err := unix.Fstatat(dfd, name, &st, unix.AT_SYMLINK_NOFOLLOW); err != nil {
sink.skip(p) // gone since the folder was read
continue
}
kind := st.Mode & unix.S_IFMT
if kind == unix.S_IFDIR && name == ".git" {
continue
}
if (kind != unix.S_IFREG && kind != unix.S_IFDIR) || (kind == unix.S_IFDIR && depth >= maxTreeDepth) {
sink.skip(p)
continue
}
if packTreeBeforeOpen != nil {
packTreeBeforeOpen(p)
}
flags := unix.O_RDONLY | unix.O_NOFOLLOW | unix.O_CLOEXEC | unix.O_NONBLOCK | unix.O_NOCTTY
if kind == unix.S_IFDIR {
flags |= unix.O_DIRECTORY
}
cfd, err := openTreeEntry(dfd, name, flags)
if err != nil {
sink.skip(p) // a link or special file now, or gone
continue
}
err = packOpened(sink, cfd, name, st, p, depth)
if err != nil {
return err
}
}
return nil
}
// packOpened hands sink the entry opened as fd, and for a folder what is in it,
// if it is still the entry fstatat saw; it closes fd.
func packOpened(sink treeSink, fd int, name string, seen unix.Stat_t, p string, depth int) error {
var got unix.Stat_t
if err := unix.Fstat(fd, &got); err != nil {
unix.Close(fd)
return fmt.Errorf("stat %q: %w", p, err)
}
kind := got.Mode & unix.S_IFMT
if kind != seen.Mode&unix.S_IFMT || got.Dev != seen.Dev || got.Ino != seen.Ino {
unix.Close(fd)
sink.skip(p) // swapped between the look and the open
return nil
}
// A plain file or folder now: back to blocking reads, so os.File reads it
// as it reads any file it opened itself.
if err := unix.SetNonblock(fd, false); err != nil {
unix.Close(fd)
return fmt.Errorf("open %q: %w", p, err)
}
f := os.NewFile(uintptr(fd), name)
defer f.Close()
fi, err := f.Stat()
if err != nil {
return fmt.Errorf("stat %q: %w", p, err)
}
if err := sink.put(f, fi, p); err != nil {
return err
}
if kind == unix.S_IFDIR {
return packFolder(sink, f, fd, p, depth+1)
}
return nil
}
+204
View File
@@ -0,0 +1,204 @@
//go:build !windows
package main
import (
"archive/tar"
"bytes"
"io"
"os"
"path/filepath"
"strings"
"testing"
"time"
"golang.org/x/sys/unix"
)
const tarOutsideSecret = "outside-the-tree-3f9a"
// tarInsideA is as long as the outside file: a writer who knows a file's size
// can make the one it swaps out the same size, so the tar's header fits.
const tarInsideA = "inside-the-tree-a-000"
// tarEntries reads back what makeTar produced: name -> content.
func tarEntries(t *testing.T, b []byte) map[string]string {
t.Helper()
out := map[string]string{}
tr := tar.NewReader(bytes.NewReader(b))
for {
hdr, err := tr.Next()
if err == io.EOF {
return out
}
if err != nil {
t.Fatalf("read tar: %v", err)
}
data, err := io.ReadAll(tr)
if err != nil {
t.Fatalf("read %s: %v", hdr.Name, err)
}
out[hdr.Name] = string(data)
}
}
// outsideAndTree makes a file outside the tree that a run must never return,
// and a tree with one file and one folder.
func outsideAndTree(t *testing.T) (outsideDir, secret, tree string) {
t.Helper()
outsideDir = t.TempDir()
secret = filepath.Join(outsideDir, "secret")
if err := os.WriteFile(secret, []byte(tarOutsideSecret), 0o600); err != nil {
t.Fatal(err)
}
tree = t.TempDir()
if err := os.WriteFile(filepath.Join(tree, "a.txt"), []byte(tarInsideA), 0o644); err != nil {
t.Fatal(err)
}
if err := os.Mkdir(filepath.Join(tree, "sub"), 0o755); err != nil {
t.Fatal(err)
}
// Named as the outside file is: a writer who knows the names in a folder
// outside can fill a folder of the tree with the same names, then swap it.
if err := os.WriteFile(filepath.Join(tree, "sub", "secret"), []byte("inside b"), 0o644); err != nil {
t.Fatal(err)
}
return outsideDir, secret, tree
}
func assertNoOutside(t *testing.T, ents map[string]string) {
t.Helper()
for name, data := range ents {
if strings.Contains(data, tarOutsideSecret) {
t.Fatalf("the tar holds a file from outside the tree, as %q", name)
}
}
}
// The tree's writer swaps a file for a link, and a folder for a link, after
// the walk has seen each one as a plain file or folder and before it opens it.
// The hook stands exactly in that gap, so the swap lands there every time.
func TestMakeTarNeverReadsThroughALinkSwappedInBeforeTheOpen(t *testing.T) {
for _, swap := range []string{"a.txt", "sub"} {
t.Run(swap, func(t *testing.T) {
outsideDir, secret, tree := outsideAndTree(t)
packTreeBeforeOpen = func(rel string) {
if rel != swap {
return
}
p := filepath.Join(tree, swap)
_ = os.RemoveAll(p)
if swap == "sub" {
_ = os.Symlink(outsideDir, p)
} else {
_ = os.Symlink(secret, p)
}
}
defer func() { packTreeBeforeOpen = nil }()
b, err := makeTar(tree)
if err != nil {
t.Fatalf("makeTar: %v", err)
}
ents := tarEntries(t, b)
assertNoOutside(t, ents)
if swap == "a.txt" {
if _, ok := ents["a.txt"]; ok {
t.Fatalf("a.txt was a link when it was opened and must be left out, got it in the tar")
}
}
})
}
}
// No hook: a writer flips one name between a plain file and a link to a file
// outside the tree as fast as it can while the tree is packed again and again.
func TestMakeTarNeverReadsThroughALinkUnderAConcurrentWriter(t *testing.T) {
_, secret, tree := outsideAndTree(t)
name := filepath.Join(tree, "a.txt")
tmpLink, tmpFile := filepath.Join(tree, ".l"), filepath.Join(tree, ".f")
stop := make(chan struct{})
done := make(chan struct{})
go func() {
defer close(done)
for {
select {
case <-stop:
return
default:
}
_ = os.WriteFile(tmpFile, []byte(tarInsideA), 0o644)
_ = os.Rename(tmpFile, name)
_ = os.Symlink(secret, tmpLink)
_ = os.Rename(tmpLink, name)
}
}()
defer func() { close(stop); <-done }()
deadline := time.Now().Add(3 * time.Second)
passes := 0
defer func() { t.Logf("%d passes", passes) }()
for i := 0; time.Now().Before(deadline); i++ {
passes = i + 1
b, err := makeTar(tree)
if err != nil {
continue // a name that vanished mid-walk; the next pass tries again
}
for n, data := range tarEntries(t, b) {
if strings.Contains(data, tarOutsideSecret) {
t.Fatalf("pass %d: the tar holds the file outside the tree, as %q", i, n)
}
}
}
}
// A link, a named pipe and a socket in the tree are left out and named as
// left out; the pipe is never opened, so the walk does not wait on it.
func TestMakeTarLeavesOutAndNamesWhatIsNotAPlainFileOrFolder(t *testing.T) {
_, secret, tree := outsideAndTree(t)
if err := os.Symlink(secret, filepath.Join(tree, "link")); err != nil {
t.Fatal(err)
}
if err := unix.Mkfifo(filepath.Join(tree, "sub", "pipe"), 0o644); err != nil {
t.Fatal(err)
}
if err := os.Mkdir(filepath.Join(tree, ".git"), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(tree, ".git", "config"), []byte("x"), 0o644); err != nil {
t.Fatal(err)
}
type res struct {
b []byte
skipped []string
err error
}
ch := make(chan res, 1)
go func() { b, s, err := packWorktree(tree); ch <- res{b, s, err} }()
var r res
select {
case r = <-ch:
case <-time.After(10 * time.Second):
t.Fatal("packing the tree waited on the named pipe")
}
if r.err != nil {
t.Fatalf("packWorktree: %v", r.err)
}
ents := tarEntries(t, r.b)
assertNoOutside(t, ents)
for _, want := range []string{"a.txt", "sub", "sub/secret"} {
if _, ok := ents[want]; !ok {
t.Errorf("%s missing from the tar (got %v)", want, ents)
}
}
if ents["a.txt"] != tarInsideA || ents["sub/secret"] != "inside b" {
t.Errorf("file contents changed in the tar: %v", ents)
}
for _, gone := range []string{"link", "sub/pipe", ".git", ".git/config"} {
if _, ok := ents[gone]; ok {
t.Errorf("%s is in the tar and must not be", gone)
}
}
got := strings.Join(r.skipped, ",")
if got != "link,sub/pipe" {
t.Errorf("left out = %q, want link,sub/pipe", got)
}
}
+83
View File
@@ -0,0 +1,83 @@
//go:build windows
package main
import (
"fmt"
"os"
"path/filepath"
)
// packTree hands the tree under root to sink (packWorktree says why it is
// walked this way). Every name is opened through an os.Root at the tree, so
// no link, junction or name swapped in while it is packed can open a file
// outside the tree; an entry goes in only if the opened file is the plain
// file or folder that was looked at. Each left-out entry goes to sink.skip.
func packTree(sink treeSink, root string) error {
r, err := os.OpenRoot(root)
if err != nil {
return err
}
defer r.Close()
return packRootFolder(sink, r, "", 0)
}
func packRootFolder(sink treeSink, r *os.Root, rel string, depth int) error {
open := rel
if open == "" {
open = "."
}
d, err := r.Open(open)
if err != nil {
return fmt.Errorf("open folder %q: %w", rel, err)
}
names, err := readTreeNames(sink, d, filepath.ToSlash(rel))
d.Close()
if err != nil {
return err
}
if err := sink.folder(filepath.ToSlash(rel), names); err != nil {
return err
}
for _, name := range names {
p := filepath.Join(rel, name)
tp := filepath.ToSlash(p)
seen, err := r.Lstat(p)
if err != nil {
sink.skip(tp)
continue
}
if seen.IsDir() && name == ".git" {
continue
}
if (!seen.Mode().IsRegular() && !seen.IsDir()) || (seen.IsDir() && depth >= maxTreeDepth) {
sink.skip(tp)
continue
}
if packTreeBeforeOpen != nil {
packTreeBeforeOpen(tp)
}
f, err := r.Open(p)
if err != nil {
sink.skip(tp)
continue
}
fi, err := f.Stat()
if err != nil || fi.Mode().Type() != seen.Mode().Type() {
f.Close()
sink.skip(tp)
continue
}
err = sink.put(f, fi, tp)
f.Close()
if err != nil {
return err
}
if fi.IsDir() {
if err := packRootFolder(sink, r, p, depth+1); err != nil {
return err
}
}
}
return nil
}
+125
View File
@@ -0,0 +1,125 @@
package main
// A harness run's tree is bind-mounted into the container at /work and opened
// to every user, because the agent runs as a uid of its own. What keeps the
// other users of this machine out of it is the folder it sits in, which only
// this process can enter. Opening the tree follows no link in it.
import (
"archive/tar"
"bytes"
"encoding/base64"
"os"
"path/filepath"
"strconv"
"strings"
"testing"
)
// harnessStandIn writes a stand-in container runtime named name into a new
// folder. On `ps` and `pull` it succeeds; on `run` it records, into the
// returned file, the mode and owner of the folder the /work tree sits in, then
// fails, so the run stops there.
func harnessStandIn(t *testing.T, name string) (bin, record string) {
t.Helper()
bin = t.TempDir()
record = filepath.Join(bin, "record")
script := "#!/bin/sh\ncase \"$1\" in ps|pull) exit 0;; run) ;; *) exit 1;; esac\n" +
"for a; do case \"$a\" in *:/work) d=${a%:/work};; esac; done\n" +
"[ -n \"$d\" ] && stat -c '%a %u' \"$(dirname \"$d\")\" > " + record + "\nexit 1\n"
if err := os.WriteFile(filepath.Join(bin, name), []byte(script), 0o755); err != nil {
t.Fatal(err)
}
return bin, record
}
func TestAWorktreeRunsTreeSitsInAFolderOnlyTheBastionCanEnter(t *testing.T) {
bin, record := harnessStandIn(t, "docker")
t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH"))
// The temporary folder every user may write in, as /tmp is.
shared := t.TempDir()
if err := os.Chmod(shared, os.ModeSticky|0o777); err != nil {
t.Fatal(err)
}
t.Setenv("TMPDIR", shared)
var tb bytes.Buffer
tw := tar.NewWriter(&tb)
_ = tw.WriteHeader(&tar.Header{Name: "src/", Typeflag: tar.TypeDir, Mode: 0o755})
_ = tw.WriteHeader(&tar.Header{Name: "src/a.txt", Typeflag: tar.TypeReg, Mode: 0o644, Size: 2})
_, _ = tw.Write([]byte("a\n"))
_ = tw.Close()
spec := harnessSpec{Mode: "worktree", HarnessID: "h1", Image: "harness:test",
WorktreeB64: base64.StdEncoding.EncodeToString(tb.Bytes())}
if _, code := runWorktree(spec); code == 0 {
t.Fatal("the stand-in runtime fails the start; the run must report it")
}
got, err := os.ReadFile(record)
if err != nil {
t.Fatalf("the tree was not mounted: %v", err)
}
if want := "700 " + strconv.Itoa(os.Geteuid()); strings.TrimSpace(string(got)) != want {
t.Fatalf("the folder the tree sits in: want %q, got %q", want, strings.TrimSpace(string(got)))
}
}
func TestOpeningARunsTreeFollowsNoLinkAndKeepsWhatIsExecutable(t *testing.T) {
bin, _ := harnessStandIn(t, "runtime")
rt := filepath.Join(bin, "runtime")
outside := filepath.Join(t.TempDir(), "shadow")
if err := os.WriteFile(outside, []byte("x\n"), 0o600); err != nil {
t.Fatal(err)
}
tree := t.TempDir()
files := map[string]os.FileMode{"run.sh": 0o755, "notes.txt": 0o644}
for name, mode := range files {
if err := os.WriteFile(filepath.Join(tree, name), nil, mode); err != nil {
t.Fatal(err)
}
}
if err := os.Mkdir(filepath.Join(tree, "sub"), 0o755); err != nil {
t.Fatal(err)
}
if err := os.Symlink(outside, filepath.Join(tree, "sub", "link")); err != nil {
t.Fatal(err)
}
if _, err := driveHarnessAgent(rt, tree, harnessSpec{HarnessID: "h1", Image: "harness:test"}, 1); err == nil {
t.Fatal("the stand-in runtime fails the start; the run must report it")
}
for p, want := range map[string]os.FileMode{
tree: 0o777, filepath.Join(tree, "sub"): 0o777,
filepath.Join(tree, "run.sh"): 0o777, filepath.Join(tree, "notes.txt"): 0o666,
outside: 0o600,
} {
fi, err := os.Stat(p)
if err != nil {
t.Fatal(err)
}
if fi.Mode().Perm() != want {
t.Errorf("%s: want %04o, got %04o", p, want, fi.Mode().Perm())
}
}
}
func TestARunsTreeInAFolderOtherUsersCanEnterIsRefused(t *testing.T) {
bin, record := harnessStandIn(t, "runtime")
rt := filepath.Join(bin, "runtime")
open := t.TempDir()
if err := os.Chmod(open, 0o755); err != nil {
t.Fatal(err)
}
tree := filepath.Join(open, "tree")
if err := os.Mkdir(tree, 0o755); err != nil {
t.Fatal(err)
}
_, err := driveHarnessAgent(rt, tree, harnessSpec{HarnessID: "h1", Image: "harness:test"}, 1)
if err == nil || !strings.Contains(err.Error(), "open to other users") {
t.Fatalf("want the tree refused for its folder, got %v", err)
}
if _, err := os.Stat(record); err == nil {
t.Fatal("a refused tree must start no container")
}
if fi, _ := os.Stat(tree); fi.Mode().Perm() != 0o755 {
t.Fatalf("a refused tree must not be opened: %04o", fi.Mode().Perm())
}
}
+119
View File
@@ -0,0 +1,119 @@
//go:build !windows
package main
import (
"errors"
"os"
"path/filepath"
"runtime"
"strconv"
"strings"
"testing"
)
// A copy of a tree of very many empty files stops at its limit, and what it
// holds in memory does not grow with the number of files.
func TestACopyOfVeryManyEmptyFilesIsRefusedWithBoundedMemory(t *testing.T) {
src, dst := t.TempDir(), t.TempDir()
for i := 0; i < 50; i++ {
d := filepath.Join(src, "d"+strconv.Itoa(i))
if err := os.Mkdir(d, 0o755); err != nil {
t.Fatal(err)
}
for j := 0; j < 400; j++ {
if err := os.WriteFile(filepath.Join(d, "f"+strconv.Itoa(j)), nil, 0o644); err != nil {
t.Fatal(err)
}
}
}
var before, after runtime.MemStats
runtime.GC()
runtime.ReadMemStats(&before)
_, err := mirrorTree(src, dst, 1<<20)
runtime.ReadMemStats(&after)
if !errors.Is(err, errTreeTooLarge) {
t.Fatalf("20000 empty files under a 1 MiB limit: %v", err)
}
got := after.TotalAlloc - before.TotalAlloc
t.Logf("refused: %v; %d KiB allocated", err, got>>10)
if got > 16<<20 {
t.Errorf("the copy allocated %d MiB", got>>20)
}
n := 0
_ = filepath.WalkDir(dst, func(string, os.DirEntry, error) error { n++; return nil })
if n > (1<<20)/512+1 {
t.Errorf("%d entries were written under a limit of %d", n, (1<<20)/512)
}
}
// Left-out entries are counted in full, and only the first
// maxSkippedReported names are kept.
func TestLeftOutEntriesAreCountedAndTheirNamesBounded(t *testing.T) {
src, dst := t.TempDir(), t.TempDir()
for i := 0; i < 3*maxSkippedReported; i++ {
if err := os.Symlink("x", filepath.Join(src, "l"+strconv.Itoa(i))); err != nil {
t.Fatal(err)
}
}
skips, err := mirrorTree(src, dst, maxCopiedTreeBytes)
if err != nil {
t.Fatal(err)
}
if skips.count != 3*maxSkippedReported || len(skips.names) != maxSkippedReported {
t.Errorf("count %d, %d names kept", skips.count, len(skips.names))
}
res := map[string]interface{}{}
skips.report(res)
if res["worktree_skipped_count"] != 3*maxSkippedReported {
t.Errorf("reported %v", res)
}
}
func TestRepoGitAuthPutsTheTokenOnlyInTheEnvironment(t *testing.T) {
bare, env, err := repoGitAuth("https://host-user:tok3n0f7hisrun5a1e@git.example.invalid/o/r.git")
if err != nil || bare != "https://git.example.invalid/o/r.git" {
t.Fatalf("bare = %q, %v", bare, err)
}
joined := strings.Join(env, "\n")
if !strings.Contains(joined, "GIT_CONFIG_KEY_0=http.https://git.example.invalid/o/r.git.extraHeader") ||
!strings.Contains(joined, "GIT_CONFIG_VALUE_0=Authorization: Basic ") {
t.Errorf("env = %q", env)
}
if _, env, _ := repoGitAuth("/srv/x/origin.git"); env != nil {
t.Errorf("a path got credentials: %v", env)
}
}
// One folder holding very many names is refused once the names read pass
// the limit, without the whole folder being read first.
func TestAFolderOfVeryManyNamesIsRefusedWithoutReadingItWhole(t *testing.T) {
src := t.TempDir()
long := strings.Repeat("n", 190)
for i := 0; i < 30000; i++ {
if err := os.WriteFile(filepath.Join(src, long+strconv.Itoa(i)), nil, 0o644); err != nil {
t.Fatal(err)
}
}
saved := maxReturnedTreeBytes
maxReturnedTreeBytes = 1 << 20
t.Cleanup(func() { maxReturnedTreeBytes = saved })
for name, walk := range map[string]func() error{
"copy": func() error { _, err := mirrorTree(src, t.TempDir(), 1<<20); return err },
"tar": func() error { _, _, err := packWorktree(src); return err },
} {
var before, after runtime.MemStats
runtime.GC()
runtime.ReadMemStats(&before)
err := walk()
runtime.ReadMemStats(&after)
got := after.TotalAlloc - before.TotalAlloc
t.Logf("%s: %v; %d KiB allocated", name, err, got>>10)
if !errors.Is(err, errTreeTooLarge) {
t.Errorf("%s: a folder of 30000 names under a 1 MiB limit: %v", name, err)
}
if got > 4<<20 {
t.Errorf("%s: %d KiB allocated; the 30000 names alone are about 6 MiB", name, got>>10)
}
}
}
+71
View File
@@ -0,0 +1,71 @@
//go:build !windows
package main
import (
"os"
"path/filepath"
"runtime"
"strings"
"testing"
"time"
)
// A file far larger than the tree a run may return (sparse, so it costs
// nothing to make) is refused from its size: packing returns at once, with
// an error and no tar, having read none of it.
func TestPackingRefusesAFileLargerThanTheLaneCarriesWithoutReadingIt(t *testing.T) {
tree := t.TempDir()
if err := os.WriteFile(filepath.Join(tree, "a.txt"), []byte("a\n"), 0o644); err != nil {
t.Fatal(err)
}
f, err := os.Create(filepath.Join(tree, "huge.bin"))
if err != nil {
t.Fatal(err)
}
if err := f.Truncate(64 << 30); err != nil {
t.Skipf("no sparse file here: %v", err)
}
f.Close()
type res struct {
b []byte
err error
}
var before, after runtime.MemStats
runtime.ReadMemStats(&before)
ch := make(chan res, 1)
start := time.Now()
go func() { b, _, err := packWorktree(tree); ch <- res{b, err} }()
var r res
select {
case r = <-ch:
case <-time.After(20 * time.Second):
t.Fatal("packing a tree with a 64 GiB file was still going after 20 s")
}
runtime.ReadMemStats(&after)
t.Logf("returned in %v, %d MiB allocated", time.Since(start), (after.TotalAlloc-before.TotalAlloc)>>20)
if r.err == nil || !strings.Contains(r.err.Error(), "huge.bin") || !strings.Contains(r.err.Error(), "too large") {
t.Fatalf("want the tree refused for huge.bin, got %v", r.err)
}
if r.b != nil {
t.Fatal("a tar came back with the refusal")
}
if got := after.TotalAlloc - before.TotalAlloc; got > 64<<20 {
t.Fatalf("packing allocated %d MiB", got>>20)
}
}
// A result larger than one message on the lane is replaced by a refusal that
// says so, never sent to be dropped by the host; one that fits goes as it is.
func TestAResultTheLaneCannotCarryIsRefusedNotSent(t *testing.T) {
small := []byte(`{"ok":true,"mode":"worktree"}`)
if b, code := fitLane(small, 0); string(b) != string(small) || code != 0 {
t.Fatalf("a result that fits was changed: %s %d", b, code)
}
big := append([]byte(`{"ok":true,"mode":"worktree","worktree_b64":"`), make([]byte, uvicornWSMaxSize)...)
b, code := fitLane(big, 0)
if code == 0 || len(b) > 1024 || !strings.Contains(string(b), `"result_too_large":true`) ||
!strings.Contains(string(b), `"ok":false`) {
t.Fatalf("an oversized result was not refused: %d bytes, code %d: %.200s", len(b), code, b)
}
}
+9 -6
View File
@@ -484,7 +484,7 @@ func TestRepeatedRefusedWritesQuarantineThePairing(t *testing.T) {
for i := 0; i < quarantineStrikes; i++ { for i := 0; i < quarantineStrikes; i++ {
driveJob(t, tAnchor, putJob(""), ctx, "") driveJob(t, tAnchor, putJob(""), ctx, "")
} }
if u := quarantinedUntil(tAnchor, time.Now()); u.IsZero() || u.Sub(time.Now()) > quarantineBase+time.Minute { if u := quarantinedUntil(tAnchor, time.Now()); u.IsZero() || time.Until(u) > quarantineBase+time.Minute {
t.Fatalf("a cleared hold is forgiven — the next one is the base length again: %v", u) t.Fatalf("a cleared hold is forgiven — the next one is the base length again: %v", u)
} }
// Strikes double when the hold lapses on its own. // Strikes double when the hold lapses on its own.
@@ -494,7 +494,7 @@ func TestRepeatedRefusedWritesQuarantineThePairing(t *testing.T) {
for i := 0; i < quarantineStrikes; i++ { for i := 0; i < quarantineStrikes; i++ {
driveJob(t, tAnchor, putJob(""), ctx, "") driveJob(t, tAnchor, putJob(""), ctx, "")
} }
if u := quarantinedUntil(tAnchor, time.Now()); u.IsZero() || u.Sub(time.Now()) < 2*quarantineBase-time.Minute { if u := quarantinedUntil(tAnchor, time.Now()); u.IsZero() || time.Until(u) < 2*quarantineBase-time.Minute {
t.Fatalf("a repeat hold doubles: %v", u) t.Fatalf("a repeat hold doubles: %v", u)
} }
} }
@@ -1010,12 +1010,15 @@ func TestAnchoredEgressRefusesLoopbackAndPrivateDestinations(t *testing.T) {
if addr, why := egressVettedAddr(ctx, "localhost:80"); why == "" || addr != "" { if addr, why := egressVettedAddr(ctx, "localhost:80"); why == "" || addr != "" {
t.Fatalf("a refused name yields no address: %q %q", addr, why) t.Fatalf("a refused name yields no address: %q %q", addr, why)
} }
// A local box refuses this machine too, and the network around it
// (egress_local_test.go and egress_network_test.go have the rest).
t.Setenv("XDG_CONFIG_HOME", t.TempDir()) t.Setenv("XDG_CONFIG_HOME", t.TempDir())
if why := egressInternalRefusal(ctx, "127.0.0.1:11434"); why != "" { stubOwnAddrs(t, loopbackOnly, localRange())
t.Fatalf("a local box keeps its lane: %s", why) if why := egressInternalRefusal(ctx, "127.0.0.1:11434"); why == "" {
t.Fatal("a local box must refuse its own ollama on the egress lane")
} }
if addr, _ := egressVettedAddr(ctx, "example.com:443"); addr != "example.com:443" { if addr, why := egressVettedAddr(ctx, "192.168.1.1:80"); why == "" || addr != "" {
t.Fatalf("a local box dials the name it was given: %q", addr) t.Fatalf("a local box refuses its LAN on the egress lane: %q %q", addr, why)
} }
} }
+29 -4
View File
@@ -133,18 +133,39 @@ func inferenceFilterNeeded(self string, lane inferenceLane, scope modelScope) bo
// //
// With nothing to enforce it is the original raw splice, byte for byte. // With nothing to enforce it is the original raw splice, byte for byte.
func proxyInference(in net.Conn, upstream string, lane inferenceLane, self string) { func proxyInference(in net.Conn, upstream string, lane inferenceLane, self string) {
proxyInferenceVia(in, dialUpstream(upstream), lane, self, spliceConnVia)
}
// upstreamDial opens the connection to the service behind a forward.
type upstreamDial func() (net.Conn, error)
// dialUpstream is the forwards' dial: TCP to the address, 5 s.
func dialUpstream(upstream string) upstreamDial {
return func() (net.Conn, error) { return net.DialTimeout("tcp", upstream, 5*time.Second) }
}
// proxyInferenceVia is proxyInference with the dial, and the pass-through for
// a connection with nothing to enforce, in the caller's hands. The serve lane
// (servelane.go) uses it so its connections take the same decision and the
// same filter as the tunnel's, while the dial goes to the address it vetted
// and is watched by its idle bound.
func proxyInferenceVia(in net.Conn, dial upstreamDial, lane inferenceLane, self string, pass func(net.Conn, upstreamDial)) {
if inferenceFilterNeeded(self, lane, pairingModelScope(self)) { if inferenceFilterNeeded(self, lane, pairingModelScope(self)) {
serveInferenceFiltered(in, upstream, lane, self) serveInferenceFilteredVia(in, dial, lane, self)
return return
} }
spliceConn(in, upstream) pass(in, dial)
} }
// spliceConn is what the forwards did before this file existed, unchanged and // spliceConn is what the forwards did before this file existed, unchanged and
// in one place now that three call sites share it. // in one place now that three call sites share it.
func spliceConn(in net.Conn, upstream string) { func spliceConn(in net.Conn, upstream string) {
spliceConnVia(in, dialUpstream(upstream))
}
func spliceConnVia(in net.Conn, dial upstreamDial) {
defer in.Close() defer in.Close()
out, err := net.DialTimeout("tcp", upstream, 5*time.Second) out, err := dial()
if err != nil { if err != nil {
return return
} }
@@ -163,8 +184,12 @@ func spliceConn(in net.Conn, upstream string) {
// grant on every request below, so there is nowhere for a caller — including a // grant on every request below, so there is nowhere for a caller — including a
// test — to hand it one the grant does not say. // test — to hand it one the grant does not say.
func serveInferenceFiltered(in net.Conn, upstream string, lane inferenceLane, self string) { func serveInferenceFiltered(in net.Conn, upstream string, lane inferenceLane, self string) {
serveInferenceFilteredVia(in, dialUpstream(upstream), lane, self)
}
func serveInferenceFilteredVia(in net.Conn, dial upstreamDial, lane inferenceLane, self string) {
defer in.Close() defer in.Close()
out, err := net.DialTimeout("tcp", upstream, 5*time.Second) out, err := dial()
if err != nil { if err != nil {
writeFilterError(in, nil, http.StatusBadGateway, "the service behind this lane is not answering") writeFilterError(in, nil, http.StatusBadGateway, "the service behind this lane is not answering")
return return
+2 -2
View File
@@ -90,7 +90,7 @@ var procIdentity = osProcIdentity
// walked. The tree kill used to test only the root, so stopping a finding // walked. The tree kill used to test only the root, so stopping a finding
// took everything under it — a replaced sshd took every ssh session with it, // took everything under it — a replaced sshd took every ssh session with it,
// accounting entries included, which is exactly what the guards on the root // accounting entries included, which is exactly what the guards on the root
// exist to refuse (review of #100). Killing a parent does not kill its // exist to refuse. Killing a parent does not kill its
// children, so a spared subtree simply survives, re-parented. // children, so a spared subtree simply survives, re-parented.
// //
// A PID IS NOT A PROCESS. Each pid's identity is read before it is stopped and // A PID IS NOT A PROCESS. Each pid's identity is read before it is stopped and
@@ -111,7 +111,7 @@ func killTree(root int, spare func(int) bool) int { return killTreeExpect(root,
// killTreeExpect is killTree for a root whose identity was read when it was // killTreeExpect is killTree for a root whose identity was read when it was
// chosen (the finding walk): if the pid no longer carries that identity, the // chosen (the finding walk): if the pid no longer carries that identity, the
// number has changed hands since and nothing is signalled. That closes the gap // number has changed hands since and nothing is signalled. That closes the gap
// between choosing the root and the first read inside the kill (review of #100). // between choosing the root and the first read inside the kill.
func killTreeExpect(root int, spare func(int) bool, rootID string) int { func killTreeExpect(root int, spare func(int) bool, rootID string) int {
if spare(root) { if spare(root) {
return 0 return 0
+3 -3
View File
@@ -208,7 +208,7 @@ func identityRig(t *testing.T, script map[int][]string, stopped map[int]bool) {
} }
} }
// A PID IS NOT A PROCESS (review of #100). 202's number changes hands between // A PID IS NOT A PROCESS. 202's number changes hands between
// the read and the stop: the stranger is resumed and never killed. 203 dies // the read and the stop: the stranger is resumed and never killed. 203 dies
// while frozen and its number is reused before the kill: not killed, and not // while frozen and its number is reused before the kill: not killed, and not
// resumed either (the frozen process is gone; the new one was never stopped). // resumed either (the frozen process is gone; the new one was never stopped).
@@ -239,7 +239,7 @@ func TestKillTreeNeverSignalsAStrangerUnderARecycledPid(t *testing.T) {
} }
// A PROCESS SOMEONE ELSE HAD STOPPED is killed with the tree but never resumed // A PROCESS SOMEONE ELSE HAD STOPPED is killed with the tree but never resumed
// by us when the kill fails (review of #100, N4): its stopped state was not // by us when the kill fails: its stopped state was not
// ours to undo. // ours to undo.
// //
// MUTATION: resume on kill failure regardless of wasStopped. RESULT: red. // MUTATION: resume on kill failure regardless of wasStopped. RESULT: red.
@@ -311,7 +311,7 @@ func TestRecoverFrozenTreesThawsOnlyWhatItFroze(t *testing.T) {
} }
} }
// THE FROZEN SET IS JOURNALLED AS IT IS FROZEN (review of #100): a bastion that // THE FROZEN SET IS JOURNALLED AS IT IS FROZEN: a bastion that
// dies during the freeze rounds still leaves a record of what it stopped. // dies during the freeze rounds still leaves a record of what it stopped.
// //
// MUTATION: write the journal only after the freeze rounds. RESULT: red — the // MUTATION: write the journal only after the freeze rounds. RESULT: red — the
-849
View File
@@ -1,849 +0,0 @@
package main
// lab.go — mode=lab: the Lab Harness.
//
// A lab run builds a sealed lab of N hardened hosts on THIS box's own k3s cluster (the
// containment lab), puts the application under test into every host, places ONE coding
// harness pod inside the lab and drives the test runbook through it, collects the
// transcript + per-step verdicts + the runbook worktree, and tears everything down.
//
// THREE STRUCTURAL WALLS (the product claim "spun up in the lab, nowhere else"):
// 1. This mode only answers on a LAB bastion: the lab marker, the inert egress wall file
// and the lab operator on loopback must all be present (labAvailable). A user's home
// bastion returns exit 126 in the same shape as an ungranted op.
// 2. The host-side grant carries a `mode:lab` token, so a grant for a home
// box never covers a lab run and a lab grant never covers a home box.
// 3. The lab cluster has no path out (cloud firewall + labguard per run + tripwire).
//
// What runs where: the lab HOSTS do not think — they hold and run the app under test. The
// coding harness pod thinks, with the per-run credential the host delivered, and its
// tools reach the lab hosts in-cluster. Nothing here talks to the platform.
//
// Gate: only gate="auto" is supported today (typed deliberately, exactly like the old bench's
// --auto). gate="ask" is the App lane (per-call y/n from outside the lab) and is not built
// yet; until then it is refused, never silently downgraded to auto.
import (
"bytes"
"crypto/rand"
"encoding/base64"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"syscall"
"time"
)
const (
labMarkerPath = "/etc/privatae-lab/marker"
labGuardPath = "/etc/nftables-labguard.nft"
labOperatorURL = "http://127.0.0.1:8011/healthz"
labStorageRoot = "/var/lib/rancher/k3s/storage"
labWorkspace = "/var/lib/skema/projects" // the harness image's workspace root (same image, same root)
labHarnessPort = 8844
labHostIDFloor = 9001 // lab host entity ids start here; 900 is the bench's own test entity
labHostIDCeil = 9899
labTemplateEntity = 900 // the lab's bench entity; its registry row is the template for lab hosts
labDefaultHosts = 1
labMaxHostsCap = 8 // per-box cap — protects the LAB BOX, never a user's workflow; marker max_hosts= raises it
)
// labMarker holds the key=value lines of /etc/privatae-lab/marker. Written by the lab
// rebuild, never by this binary.
type labMarker struct {
present bool
hostImage string // host_image= — the host image lab hosts run (must already be on the node)
maxHosts int // max_hosts= — optional
}
func readLabMarker() labMarker {
m := labMarker{}
b, err := os.ReadFile(labMarkerPath)
if err != nil {
return m
}
m.present = true
for _, line := range strings.Split(string(b), "\n") {
k, v, ok := strings.Cut(strings.TrimSpace(line), "=")
if !ok {
continue
}
switch strings.TrimSpace(k) {
case "host_image":
m.hostImage = strings.TrimSpace(v)
case "max_hosts":
m.maxHosts, _ = strconv.Atoi(strings.TrimSpace(v))
}
}
return m
}
// labAvailable is wall #1. Every check is a thing only the lab box has.
func labAvailable() (labMarker, error) {
m := readLabMarker()
if !m.present {
return m, fmt.Errorf("no lab marker")
}
if _, err := os.Stat(labGuardPath); err != nil {
return m, fmt.Errorf("no egress wall file")
}
c := &http.Client{Timeout: 3 * time.Second}
resp, err := c.Get(labOperatorURL)
if err != nil {
return m, fmt.Errorf("lab operator not on loopback")
}
resp.Body.Close()
if resp.StatusCode != 200 {
return m, fmt.Errorf("lab operator unhealthy (%d)", resp.StatusCode)
}
if _, err := runCmd(10*time.Second, "k3s", "kubectl", "version", "--client=true"); err != nil {
return m, fmt.Errorf("no cluster tooling")
}
if m.hostImage == "" {
return m, fmt.Errorf("marker has no host_image=")
}
return m, nil
}
// kc runs `k3s kubectl …` with a timeout.
func kc(timeout time.Duration, args ...string) (string, error) {
return runCmd(timeout, "k3s", append([]string{"kubectl"}, args...)...)
}
// kcApply applies a manifest from a temp file (runCmd has no stdin).
func kcApply(manifest string) (string, error) {
f, err := os.CreateTemp("", "lab-manifest-*.yaml")
if err != nil {
return "", err
}
defer os.Remove(f.Name())
if _, err := f.WriteString(manifest); err != nil {
f.Close()
return "", err
}
f.Close()
return kc(60*time.Second, "apply", "-f", f.Name())
}
func labFail(res map[string]interface{}, code int, msg string) int {
res["ok"], res["error"] = false, msg
return code
}
// runLab is the mode=lab entry point (harness.go runHarness). The result is marshalled
// AFTER the inner run's deferred teardown has recorded its log, so the caller always sees
// what happened including the cleanup.
func runLab(spec harnessSpec) ([]byte, int) {
res := map[string]interface{}{"mode": "lab", "ran_on": orUnknown(hostnameSafe())}
code := runLabInner(spec, res)
b, _ := json.Marshal(res)
return b, code
}
func runLabInner(spec harnessSpec, res map[string]interface{}) int {
marker, err := labAvailable()
if err != nil {
// Same shape + exit as an ungranted op on the host side (exit 126).
return labFail(res, 126, "mode 'lab' not available on this server — "+err.Error())
}
// ── validate the spec ─────────────────────────────────────────────────────
if spec.Gate != "auto" {
return labFail(res, 1, "gate='ask' (per-call y/n from the App) is not available yet — pass gate='auto', typed deliberately")
}
if spec.Image == "" {
return labFail(res, 1, "image required for mode=lab (the coding harness that drives the tests)")
}
if len(spec.Tests) == 0 {
return labFail(res, 1, "tests required for mode=lab (the runbook: ordered steps)")
}
hosts := spec.Hosts
if hosts <= 0 {
hosts = labDefaultHosts
}
maxHosts := labMaxHostsCap
if marker.maxHosts > 0 {
maxHosts = marker.maxHosts
}
if hosts > maxHosts {
return labFail(res, 1, fmt.Sprintf("hosts=%d exceeds this lab box's max_hosts=%d", hosts, maxHosts))
}
app := spec.App
if app == "" {
app = "app"
}
if !validWSName(app) {
return labFail(res, 1, "app must be a plain directory name")
}
engine := spec.Engine
if engine == "" {
engine = spec.HarnessID
}
maxMin := spec.MaxMinutes
if maxMin <= 0 {
maxMin = 10
}
var rb [4]byte
_, _ = rand.Read(rb[:])
runID := hex.EncodeToString(rb[:])
res["run_id"] = runID
start := time.Now()
var log []string
note := func(f string, a ...interface{}) {
log = append(log, time.Now().UTC().Format("15:04:05 ")+fmt.Sprintf(f, a...))
}
// ── 0. one run at a time, and reap whatever a previous run left ──────────
// The teardown below is a defer, so it covers every ordinary exit — but NOT a
// SIGKILL or a power loss (measured: a killed self-test left its two
// entity_registry rows behind). "The lab keeps nothing" has to hold after a crash
// too, so a run starts by removing anything in the lab's own id range / name
// prefixes. The lock is what makes that safe: without it a second concurrent run
// would reap the first one's hosts.
unlock, err := labLock()
if err != nil {
return labFail(res, 1, err.Error())
}
defer unlock()
if reaped := labReapOrphans(); reaped != "" {
note("reaped a previous run's leftovers: %s", reaped)
res["reaped"] = reaped
}
// ── 1. lab hosts: N hardened Entity CRs on the lab cluster ────────────────
ids, err := labAllocateIDs(hosts)
if err != nil {
return labFail(res, 1, "allocate host ids: "+err.Error())
}
res["hosts"] = ids
// ALWAYS tear down, whatever happens below — the lab keeps nothing between runs.
defer func() {
note("teardown: %d hosts + namespace lab-%s", len(ids), runID)
labTeardown(runID, ids)
res["log"] = strings.Join(log, "\n")
}()
for i, id := range ids {
name := fmt.Sprintf("lab-%s-%d", runID, i+1)
if err := labRegistryRow(id, name); err != nil {
return labFail(res, 1, fmt.Sprintf("host %d registry row: %v", id, err))
}
manifest := fmt.Sprintf(`apiVersion: privatae.ai/v1
kind: Entity
metadata:
name: entity-%d
namespace: default
labels:
privatae.ai/lab-run: "%s"
spec:
entityId: %d
lifecycleTier: always_on
image: %s
hardened: true
`, id, runID, id, marker.hostImage)
if out, err := kcApply(manifest); err != nil {
return labFail(res, 1, fmt.Sprintf("host %d apply: %s", id, truncate(out, 200)))
}
note("host %d (%s) applied", id, name)
}
for _, id := range ids {
// The operator builds the namespace + StatefulSet asynchronously after the CR lands;
// `kubectl wait` on a pod that does not exist yet fails at once, so poll for the pod
// first, then wait for Ready.
ns, pod := fmt.Sprintf("entity-%d", id), fmt.Sprintf("pod/entity-%d-0", id)
appeared := false
for i := 0; i < 60; i++ {
if _, err := kc(15*time.Second, "-n", ns, "get", pod); err == nil {
appeared = true
break
}
time.Sleep(3 * time.Second)
}
if !appeared {
ev, _ := kc(20*time.Second, "-n", ns, "get", "events", "--sort-by=.lastTimestamp")
return labFail(res, 1, fmt.Sprintf("host %d: the operator never created its pod (3 min): %s", id, truncate(ev, 300)))
}
if out, err := kc(300*time.Second, "-n", ns, "wait", "--for=condition=Ready", pod, "--timeout=270s"); err != nil {
ev, _ := kc(20*time.Second, "-n", ns, "get", "events", "--sort-by=.lastTimestamp")
return labFail(res, 1, fmt.Sprintf("host %d never became Ready: %s | %s", id, truncate(out, 160), truncate(ev, 300)))
}
}
hostIPs := map[int]string{}
for _, id := range ids {
ip, _ := kc(20*time.Second, "-n", fmt.Sprintf("entity-%d", id), "get", "pod",
fmt.Sprintf("entity-%d-0", id), "-o", "jsonpath={.status.podIP}")
hostIPs[id] = strings.TrimSpace(ip)
}
res["host_ips"] = hostIPs
note("%d hosts Ready", len(ids))
// ── 2. the app under test + its settings into every host ─────────────────
if spec.WorktreeB64 != "" {
tarIn, err := base64.StdEncoding.DecodeString(spec.WorktreeB64)
if err != nil {
return labFail(res, 1, "bad worktree_b64: "+err.Error())
}
for _, id := range ids {
dir, err := labHostProjectDir(id, app)
if err != nil {
return labFail(res, 1, fmt.Sprintf("host %d projects dir: %v", id, err))
}
if err := extractTar(tarIn, dir); err != nil {
return labFail(res, 1, fmt.Sprintf("host %d app extract: %v", id, err))
}
if err := labWriteEnv(dir, spec.Envs); err != nil {
return labFail(res, 1, fmt.Sprintf("host %d envs: %v", id, err))
}
_ = filepath.Walk(dir, func(p string, _ os.FileInfo, werr error) error {
if werr == nil {
_ = os.Chown(p, 1000, 1000) // the entity uid — the host owns its own project
}
return nil
})
}
note("app %q + %d settings placed in every host", app, len(spec.Envs))
}
// ── 3. the ONE harness pod inside the lab (same image + env as the pool) ──
ns := "lab-" + runID
if out, err := kc(30*time.Second, "create", "namespace", ns); err != nil {
return labFail(res, 1, "namespace: "+truncate(out, 200))
}
if spec.AgentCreds != "" {
f, err := os.CreateTemp("", "lab-creds-*")
if err != nil {
return labFail(res, 1, "creds temp: "+err.Error())
}
_ = os.WriteFile(f.Name(), []byte(spec.AgentCreds), 0o600)
f.Close()
out, err := kc(30*time.Second, "-n", ns, "create", "secret", "generic", "harness-creds",
"--from-file=.credentials.json="+f.Name())
_ = os.Remove(f.Name()) // per-run: nothing durable on the box
if err != nil {
return labFail(res, 1, "creds secret: "+truncate(out, 200))
}
}
if out, err := kcApply(labHarnessPodManifest(ns, spec.Image, engine, spec.AgentCreds != "")); err != nil {
return labFail(res, 1, "harness pod: "+truncate(out, 200))
}
if out, err := kc(240*time.Second, "-n", ns, "wait", "--for=condition=Ready", "pod/harness", "--timeout=210s"); err != nil {
ev, _ := kc(20*time.Second, "-n", ns, "get", "events", "--sort-by=.lastTimestamp")
return labFail(res, 1, "harness pod never Ready: "+truncate(out, 160)+" | "+truncate(ev, 400))
}
hip, _ := kc(20*time.Second, "-n", ns, "get", "pod", "harness", "-o", "jsonpath={.status.podIP}")
hip = strings.TrimSpace(hip)
if hip == "" {
return labFail(res, 1, "harness pod has no IP")
}
base := fmt.Sprintf("http://[%s]:%d", hip, labHarnessPort)
if !strings.Contains(hip, ":") {
base = fmt.Sprintf("http://%s:%d", hip, labHarnessPort)
}
if ok, _ := waitReachable(base+"/", 60*time.Second); !ok {
return labFail(res, 1, "harness sidecar not reachable on :"+strconv.Itoa(labHarnessPort))
}
note("harness pod %s Ready at %s", engine, hip)
// ── 4. the runbook: one governed /run per step ────────────────────────────
// The workspace the harness works in carries the runbook + a hosts file, so the
// agent knows what it is testing and where each lab host answers.
wsName := "lab-run"
wsTar, err := labRunbookTar(app, spec.Tests, ids, hostIPs, spec.Envs)
if err != nil {
return labFail(res, 1, "runbook tar: "+err.Error())
}
if code, body := labPost(base+"/workspace/load?dir="+wsName, "application/x-tar", wsTar, 60*time.Second); code != 200 {
return labFail(res, 1, fmt.Sprintf("workspace/load %d: %s", code, truncate(body, 200)))
}
projectDir := labWorkspace + "/" + wsName
client := &http.Client{Timeout: time.Duration(maxMin+2) * time.Minute}
steps := make([]map[string]interface{}, 0, len(spec.Tests))
allOK := true
for i, step := range spec.Tests {
prompt := labStepPrompt(i+1, len(spec.Tests), step, app, ids, hostIPs)
reqBody, _ := json.Marshal(map[string]interface{}{
"prompt": prompt, "project_dir": projectDir, "max_minutes": maxMin,
})
t0 := time.Now()
resp, err := client.Post(base+"/run", "application/json", bytes.NewReader(reqBody))
st := map[string]interface{}{"step": i + 1, "test": step}
if err != nil {
st["ok"], st["error"] = false, "harness /run: "+err.Error()
steps = append(steps, st)
allOK = false
note("step %d: /run failed: %v", i+1, err)
break
}
rbody, _ := io.ReadAll(resp.Body)
resp.Body.Close()
var r map[string]interface{}
_ = json.Unmarshal(rbody, &r)
okv, _ := r["ok"].(bool)
st["ok"] = okv
st["exit_code"] = r["exit_code"]
st["elapsed_s"] = round2(time.Since(t0).Seconds())
if tr, _ := r["transcript"].(string); tr != "" {
st["transcript"] = tr
}
if e, _ := r["error"].(string); e != "" {
st["error"] = e
}
steps = append(steps, st)
if !okv {
allOK = false
}
note("step %d: ok=%v (%.0fs)", i+1, okv, time.Since(t0).Seconds())
}
res["steps"] = steps
// ── 5. collect: the runbook worktree after (the agent's notes/results live there) ──
if code, body := labGet(base+"/workspace/export?dir="+wsName, 60*time.Second); code == 200 {
res["worktree_b64"] = base64.StdEncoding.EncodeToString([]byte(body))
}
res["meter_s"] = round2(time.Since(start).Seconds())
res["ok"] = allOK
if allOK {
return 0
}
return 1
}
// ── helpers ──────────────────────────────────────────────────────────────────
func validWSName(s string) bool {
if s == "" || len(s) > 64 || strings.ContainsAny(s, "/\\ ") || strings.HasPrefix(s, ".") {
return false
}
return true
}
// labDeleteRows removes the lab's OWN entity_registry rows and reports how many.
// Three independent scopes, all required, so this can never reach a real entity:
// the `lab-` display-name prefix, the lab id range, and (for a run's own teardown)
// that run's id list. runID=="" + ids==nil is the orphan reap: every lab row in the range.
// Returns "" when it removed nothing.
func labDeleteRows(runID string, ids []int) string {
where := fmt.Sprintf("display_name LIKE 'lab-%s%%' AND id BETWEEN %d AND %d",
runID+map[bool]string{true: "-", false: ""}[runID != ""], labHostIDFloor, labHostIDCeil)
if len(ids) > 0 {
var in []string
for _, id := range ids {
in = append(in, strconv.Itoa(id)) // ints: nothing quotable reaches the SQL
}
where += " AND id IN (" + strings.Join(in, ",") + ")"
} else if runID != "" {
return "" // a run's teardown with no ids has nothing of its own to remove
}
sql := "DELETE FROM entity_registry WHERE " + where + ";"
f, err := os.CreateTemp("", "lab-del-*.sql")
if err != nil {
return ""
}
defer os.Remove(f.Name())
_ = os.Chmod(f.Name(), 0o644) // the postgres user reads it
_, _ = f.WriteString(sql)
f.Close()
out, err := runCmd(20*time.Second, "sudo", "-u", "postgres", "psql", "-d", "parallax", "-q",
"-v", "ON_ERROR_STOP=1", "-c", "\\set QUIET on", "-f", f.Name())
if err != nil {
return ""
}
out = strings.TrimSpace(out)
if out == "" || strings.HasSuffix(out, "DELETE 0") {
return ""
}
return out
}
// labLock serialises lab runs on this box. Returns the release func. An ordinary exit
// releases it; a killed run leaves a stale lock, so a lock whose pid is gone is taken over
// (and the takeover is what makes the reap below safe rather than racy).
func labLock() (func(), error) {
const path = "/var/tmp/lab-run.lock"
take := func() error {
f, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
if err != nil {
return err
}
fmt.Fprintf(f, "%d\n", os.Getpid())
return f.Close()
}
if err := take(); err != nil {
if b, rerr := os.ReadFile(path); rerr == nil {
if pid, perr := strconv.Atoi(strings.TrimSpace(string(b))); perr == nil {
if p, ferr := os.FindProcess(pid); ferr != nil || p.Signal(syscall.Signal(0)) != nil {
_ = os.Remove(path) // stale: the holder is gone
if err2 := take(); err2 == nil {
return func() { _ = os.Remove(path) }, nil
}
} else {
return nil, fmt.Errorf("another lab run is in flight on this box (pid %d) — labs are one at a time", pid)
}
}
}
return nil, fmt.Errorf("could not take the lab run lock: %v", err)
}
return func() { _ = os.Remove(path) }, nil
}
// labReapOrphans removes anything a PREVIOUS lab run left behind. Scoped three ways so it
// can only ever touch lab leavings: the `lab-` namespace prefix, the lab entity id range
// (the bench's own entity 900 is deliberately below it), and the `lab-` display-name prefix.
// Returns a short description of what it removed, or "".
func labReapOrphans() string {
var did []string
if out, err := kc(30*time.Second, "get", "ns", "-o", "name"); err == nil {
for _, l := range strings.Split(out, "\n") {
ns := strings.TrimPrefix(strings.TrimSpace(l), "namespace/")
if strings.HasPrefix(ns, "lab-") {
_, _ = kc(60*time.Second, "delete", "namespace", ns, "--ignore-not-found", "--wait=false")
did = append(did, "ns "+ns)
}
}
}
var ids []int
if out, err := kc(30*time.Second, "get", "entity", "-A",
"-o", "jsonpath={range .items[*]}{.spec.entityId}{\"\\n\"}{end}"); err == nil {
for _, l := range strings.Split(out, "\n") {
if v, e := strconv.Atoi(strings.TrimSpace(l)); e == nil && v >= labHostIDFloor && v <= labHostIDCeil {
ids = append(ids, v)
}
}
}
if len(ids) > 0 {
did = append(did, fmt.Sprintf("%d host(s) %v", len(ids), ids))
}
// Volumes + registry rows are cleaned for the id range even when no CR remains
// (a crash can leave either half).
labTeardown("", ids)
orphans, _ := filepath.Glob(fmt.Sprintf("%s/pvc-*_entity-9[0-9][0-9][0-9]_*", labStorageRoot))
for _, p := range orphans {
_ = os.RemoveAll(p)
did = append(did, "volume "+filepath.Base(p))
}
if n := labDeleteRows("", nil); n != "" {
did = append(did, "registry rows "+n)
}
if len(did) == 0 {
return ""
}
return strings.Join(did, ", ")
}
// labAllocateIDs picks N free entity ids in the lab range by reading the Entity CRs that exist.
func labAllocateIDs(n int) ([]int, error) {
out, err := kc(30*time.Second, "get", "entity", "-A", "-o", "jsonpath={range .items[*]}{.spec.entityId}{\"\\n\"}{end}")
if err != nil {
return nil, fmt.Errorf("list entities: %s", truncate(out, 160))
}
used := map[int]bool{}
for _, l := range strings.Split(out, "\n") {
if v, e := strconv.Atoi(strings.TrimSpace(l)); e == nil {
used[v] = true
}
}
ids := []int{}
for id := labHostIDFloor; id <= labHostIDCeil && len(ids) < n; id++ {
if !used[id] {
ids = append(ids, id)
}
}
if len(ids) < n {
return nil, fmt.Errorf("only %d free lab host ids", len(ids))
}
return ids, nil
}
// labRegistryRow gives the lab operator the registry row it resolves names from
// (the operator reads display_name off entity_registry). Lab-local postgres. entity_type is
// CHECK-constrained to parallax_native|operator, so lab rows are marked by their display_name
// prefix "lab-<run>-" (and confined to the lab id range) — teardown deletes exactly those.
func labRegistryRow(id int, name string) error {
// Clone the lab's TEMPLATE entity row (entity 900, the bench's own test entity, restored
// with the lab DB) rather than invent values: the table carries CHECKs and FKs
// (mastercode_version → mastercode_versions) that a hand-written row keeps tripping.
sql := fmt.Sprintf(`INSERT INTO entity_registry (id, entity_type, display_name, master_directive, license_tier,
seed_weights, current_weights, envelope_bounds, mastercode_version, status)
SELECT %d, entity_type, '%s', 'lab host under test', 'standard',
seed_weights, current_weights, envelope_bounds, mastercode_version, 'active'
FROM entity_registry WHERE id = %d
ON CONFLICT (id) DO NOTHING;`, id, name, labTemplateEntity)
f, err := os.CreateTemp("", "lab-row-*.sql")
if err != nil {
return err
}
defer os.Remove(f.Name())
_ = os.Chmod(f.Name(), 0o644)
if _, err := f.WriteString(sql); err != nil {
f.Close()
return err
}
f.Close()
if out, err := runCmd(20*time.Second, "sudo", "-u", "postgres", "psql", "-d", "parallax", "-v", "ON_ERROR_STOP=1", "-q", "-f", f.Name()); err != nil {
return fmt.Errorf("%s", truncate(out, 200))
}
return nil
}
// labHostProjectDir resolves the host's entity-apps PVC on the node's local-path storage and
// returns <pvc>/projects/<app>, created. The node is root over its own storage — the lab box's
// privilege, and exactly why this mode is lab-only.
func labHostProjectDir(id int, app string) (string, error) {
matches, _ := filepath.Glob(fmt.Sprintf("%s/pvc-*_entity-%d_entity-apps-entity-%d-0", labStorageRoot, id, id))
if len(matches) != 1 {
return "", fmt.Errorf("expected one entity-apps volume, found %d", len(matches))
}
dir := filepath.Join(matches[0], "projects", app)
if err := os.MkdirAll(dir, 0o755); err != nil {
return "", err
}
return dir, nil
}
// labWriteEnv writes the app settings as <dir>/.env (KEY=value, sorted, values single-quoted).
func labWriteEnv(dir string, envs map[string]string) error {
if len(envs) == 0 {
return nil
}
keys := make([]string, 0, len(envs))
for k := range envs {
keys = append(keys, k)
}
sort.Strings(keys)
var b strings.Builder
for _, k := range keys {
if !validEnvKey(k) {
return fmt.Errorf("bad env key %q", k)
}
b.WriteString(k + "='" + strings.ReplaceAll(envs[k], "'", "'\\''") + "'\n")
}
return os.WriteFile(filepath.Join(dir, ".env"), []byte(b.String()), 0o640)
}
func validEnvKey(k string) bool {
if k == "" || len(k) > 128 {
return false
}
for i, r := range k {
ok := r == '_' || (r >= 'A' && r <= 'Z') || (r >= 'a' && r <= 'z') || (i > 0 && r >= '0' && r <= '9')
if !ok {
return false
}
}
return true
}
// labHarnessPodManifest mirrors the pool's harness pod (same image, same env, same floor:
// non-root uid, drop ALL, no privilege escalation, RO rootfs, the personal seccomp profile)
// with emptyDirs where the harness must write.
func labHarnessPodManifest(ns, image, engine string, withCreds bool) string {
credsVol, credsMount, credsInit := "", "", ""
if withCreds {
credsVol = ` - name: creds-src
secret:
secretName: harness-creds
`
credsMount = ""
credsInit = ` initContainers:
- name: creds
image: busybox:1.36
command: ["sh", "-c", "cp /src/.credentials.json /dst/.credentials.json && chown 10005:10005 /dst/.credentials.json && chmod 0644 /dst/.credentials.json"]
securityContext:
runAsUser: 0
allowPrivilegeEscalation: false
capabilities: {drop: ["ALL"], add: ["CHOWN", "FOWNER", "DAC_OVERRIDE"]}
volumeMounts:
- {name: creds-src, mountPath: /src, readOnly: true}
- {name: home-claude, mountPath: /dst}
`
}
return fmt.Sprintf(`apiVersion: v1
kind: Pod
metadata:
name: harness
namespace: %s
labels: {app: lab-harness, privatae.ai/lab-run: "%s"}
spec:
restartPolicy: Never
automountServiceAccountToken: false
securityContext:
seccompProfile: {type: Localhost, localhostProfile: seccomp-personal.json}
%s containers:
- name: harness
image: %s
imagePullPolicy: IfNotPresent
env:
- {name: PORT, value: "%d"}
- {name: PRIVATAE_SIDECAR_BIND, value: "::"}
- {name: PRIVATAE_APP_ID, value: "%s"}
- {name: PRIVATAE_WORKSPACE, value: "%s"}
# Both spellings, on purpose: the harness image is published on its own
# schedule, so a lab run must work against one that reads either name.
# The older three come out once no published image reads them.
- {name: SKEMA_SIDECAR_BIND, value: "::"}
- {name: SKEMA_APP_ID, value: "%s"}
- {name: SKEMA_WORKSPACE, value: "%s"}
- {name: HOME, value: "/home/privatae"}
securityContext:
runAsUser: 10005
runAsGroup: 10005
runAsNonRoot: true
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities: {drop: ["ALL"]}
resources:
requests: {cpu: 250m, memory: 512Mi}
limits: {memory: 2Gi}
volumeMounts:
- {name: workspace, mountPath: %s}
- {name: home-claude, mountPath: /home/privatae/.claude}
- {name: tmp, mountPath: /tmp}
%s volumes:
- {name: workspace, emptyDir: {}}
- {name: home-claude, emptyDir: {}}
- {name: tmp, emptyDir: {}}
%s`, ns, strings.TrimPrefix(ns, "lab-"), credsInit, image, labHarnessPort,
engine, labWorkspace, engine, labWorkspace,
labWorkspace, credsMount, credsVol)
}
// labRunbookTar packs the runbook the harness works from: a README naming the app, the hosts
// and their addresses, the settings, and the ordered tests; plus a results/ dir for the agent.
func labRunbookTar(app string, tests []string, ids []int, ips map[int]string, envs map[string]string) ([]byte, error) {
dir, err := os.MkdirTemp("", "lab-runbook-")
if err != nil {
return nil, err
}
defer os.RemoveAll(dir)
var b strings.Builder
b.WriteString("# Lab run\n\nYou are the test harness inside a sealed lab. Nothing here can reach the internet or the platform.\n\n")
b.WriteString("## Application under test\n\n`" + app + "` — placed in every lab host under " + labWorkspace + "/" + app + "\n\n")
b.WriteString("## Lab hosts\n\n| host | pod address | kernel API |\n|---|---|---|\n")
for i, id := range ids {
b.WriteString(fmt.Sprintf("| lab host %d (entity-%d) | %s | http://[%s]:8800 |\n", i+1, id, ips[id], ips[id]))
}
if len(envs) > 0 {
b.WriteString("\n## Settings (also written to every host's " + app + "/.env)\n\n")
keys := make([]string, 0, len(envs))
for k := range envs {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
b.WriteString("- " + k + "\n")
}
}
b.WriteString("\n## Tests (run in order; write each verdict to results/<n>.md)\n\n")
for i, t := range tests {
b.WriteString(fmt.Sprintf("%d. %s\n", i+1, t))
}
if err := os.WriteFile(filepath.Join(dir, "README.md"), []byte(b.String()), 0o644); err != nil {
return nil, err
}
if err := os.MkdirAll(filepath.Join(dir, "results"), 0o755); err != nil {
return nil, err
}
_ = os.WriteFile(filepath.Join(dir, "results", ".keep"), nil, 0o644)
return makeTar(dir)
}
func labStepPrompt(n, total int, step, app string, ids []int, ips map[int]string) string {
var hosts []string
for i, id := range ids {
hosts = append(hosts, fmt.Sprintf("lab host %d = entity-%d at %s", i+1, id, ips[id]))
}
return fmt.Sprintf("Lab test step %d of %d for the application `%s`.\n\nTEST: %s\n\nLab hosts: %s.\n\n"+
"Read README.md first. Run the test against the lab hosts, then write a short verdict to results/%d.md "+
"(first line PASS or FAIL, then the evidence). Do not modify the application; only test it.",
n, total, app, step, strings.Join(hosts, "; "), n)
}
func labPost(url, ctype string, body []byte, timeout time.Duration) (int, string) {
c := &http.Client{Timeout: timeout}
resp, err := c.Post(url, ctype, bytes.NewReader(body))
if err != nil {
return 0, err.Error()
}
defer resp.Body.Close()
rb, _ := io.ReadAll(resp.Body)
return resp.StatusCode, string(rb)
}
func labGet(url string, timeout time.Duration) (int, string) {
c := &http.Client{Timeout: timeout}
resp, err := c.Get(url)
if err != nil {
return 0, err.Error()
}
defer resp.Body.Close()
rb, _ := io.ReadAll(resp.Body)
return resp.StatusCode, string(rb)
}
// labTeardown removes everything a run created: the harness namespace, the lab hosts (Entity
// CRs → the operator deletes their namespaces), their volumes on local-path storage, and the
// lab-only registry rows. Best-effort, always runs, never raises.
func labTeardown(runID string, ids []int) {
if runID != "" { // "" = the orphan reap, which deletes lab- namespaces itself
_, _ = kc(120*time.Second, "delete", "namespace", "lab-"+runID, "--ignore-not-found", "--wait=false")
}
for _, id := range ids {
_, _ = kc(60*time.Second, "-n", "default", "delete", "entity", fmt.Sprintf("entity-%d", id), "--ignore-not-found", "--wait=false")
}
// wait for the CR to go (finalizers) and the operator to drop the namespace before wiping storage
for _, id := range ids {
for i := 0; i < 40; i++ {
if _, err := kc(15*time.Second, "-n", "default", "get", "entity", fmt.Sprintf("entity-%d", id)); err != nil {
break
}
time.Sleep(3 * time.Second)
}
for i := 0; i < 40; i++ {
if _, err := kc(15*time.Second, "get", "namespace", fmt.Sprintf("entity-%d", id)); err != nil {
break
}
time.Sleep(3 * time.Second)
}
for _, m := range []string{"entity-apps", "entity-data"} {
matches, _ := filepath.Glob(fmt.Sprintf("%s/pvc-*_entity-%d_%s-entity-%d-0", labStorageRoot, id, m, id))
for _, p := range matches {
_ = os.RemoveAll(p)
}
}
}
_ = labDeleteRows(runID, ids)
}
// cmdLabSelftest — `privatae-bastion lab-selftest <spec.json>`: run mode=lab from a spec file on
// THIS box without the host relay, print the result. Operator tooling for the lab box; on any
// other box it prints the same refusal the relay would (wall #1).
func cmdLabSelftest(path string) error {
raw, err := os.ReadFile(path)
if err != nil {
return err
}
var spec harnessSpec
if err := json.Unmarshal(raw, &spec); err != nil {
return fmt.Errorf("spec: %w", err)
}
spec.Mode = "lab"
if spec.HarnessID == "" {
spec.HarnessID = "lab"
}
out, code := runLab(spec)
var pretty bytes.Buffer
if json.Indent(&pretty, out, "", " ") == nil {
fmt.Println(pretty.String())
} else {
fmt.Println(string(out))
}
if code != 0 {
return fmt.Errorf("lab run exit %d", code)
}
return nil
}
+31
View File
@@ -0,0 +1,31 @@
//go:build !lab
package main
// lab_off.go: mode=lab and `lab-selftest` in every release build. The lab
// harness is test tooling for one test lab and is built only with -tags lab,
// so a release bastion answers a lab run the way any bastion without a lab
// answers it: exit 126, in the same shape as an ungranted op.
import (
"encoding/json"
"errors"
"fmt"
"os"
)
const labUnavailable = "mode 'lab' not available on this server — this build has no lab harness"
func runLab(_ harnessSpec) ([]byte, int) {
b, _ := json.Marshal(map[string]any{"mode": "lab", "ran_on": orUnknown(hostnameSafe()), "ok": false, "error": labUnavailable})
return b, 126
}
// cmdLabSelftest prints the same refusal the relay would give.
func cmdLabSelftest(path string) error {
if _, err := os.Stat(path); err != nil {
return err
}
fmt.Fprintln(os.Stderr, labUnavailable)
return errors.New("lab run exit 126")
}
+21
View File
@@ -0,0 +1,21 @@
//go:build !lab
package main
import (
"encoding/json"
"testing"
)
// A release build has no lab harness: a lab run is refused with exit 126, the
// shape of an ungranted op, and touches nothing.
func TestAReleaseBuildRefusesALabRun(t *testing.T) {
out, code := runHarness(harnessSpec{Mode: "lab", HarnessID: "lab-1", Gate: "auto", Image: "x", Tests: []string{"a"}})
if code != 126 {
t.Fatalf("exit %d, want 126", code)
}
var m map[string]any
if err := json.Unmarshal(out, &m); err != nil || m["ok"] != false || m["error"] != labUnavailable {
t.Fatalf("reply %s", out)
}
}
+122
View File
@@ -0,0 +1,122 @@
package main
// What a paired host sends down a worker lane is read at a size this machine
// chose, never at one the host chose: the host picks a frame's length, and a
// frame of any length must not be held whole in this machine's memory. The
// egress lane (egress_relay.go) and the context socket (contextsock.go) follow
// the same rule.
//
// - A job's first frame (the H spec on a harness worker, the S spec on a
// session worker) is read up to its lane's bound and refused past it, by
// name, before any of it is parsed.
// - What a lane carries after that is streamed: terminal input into the pty
// (session.go), the replica's bytes into its socket (mirror.go). Frames that
// nothing reads (a harness run and a root command, which read nothing after
// their spec) are dropped as they arrive.
import (
"errors"
"fmt"
"io"
"net/http"
"github.com/gorilla/websocket"
)
const (
// harnessSpecMaxBytes bounds the H frame. Its bulk is the worktree tar
// (worktree_b64). The largest tree this lane hands back is
// maxReturnedTreeBytes (9 MiB, so 12 MiB in base64) inside one message of
// uvicornWSMaxSize (16 MiB), the most the host takes in (harness.go). A host
// that sends that tree into its next run needs those 12 MiB, plus the
// prompt and the run's credentials. Twice the lane's message size holds
// that with 20 MiB to spare.
harnessSpecMaxBytes = 2 * uvicornWSMaxSize
// sessionSpecMaxBytes bounds the S frame: a signed grant (op ids and the
// person's own words), ids, a public key, and for session:root:exec the
// command, itself held to rootExecCommandMax (64 KiB), which is at most
// 384 KiB even when JSON escapes every byte as \u00XX. 1 MiB is the bound
// Go's own HTTP server puts on a request head.
sessionSpecMaxBytes = http.DefaultMaxHeaderBytes
// sessionInputMaxBytes bounds one frame on an open terminal. The host relays
// what the person's browser sent it, one message each, and its own server
// takes a message up to uvicornWSMaxSize; the tag is one byte more. Input is
// streamed into the pty, so this caps a frame's length, not memory.
sessionInputMaxBytes = uvicornWSMaxSize + 1
// sessionResizeMaxBytes bounds the body of an R frame, {"cols":N,"rows":N}.
sessionResizeMaxBytes = 4 << 10
// heartbeatReplyMaxBytes bounds the host's answer to a heartbeat. It
// carries directives and the lists of grants this box should hold, each
// entry an id and a signature of about 130 bytes, so 4 MiB is tens of
// thousands of grants. Nothing in it is bulk: an update names a version,
// and the binary comes from the release manifest, not from this reply.
heartbeatReplyMaxBytes = 4 << 20
)
// errLaneNotJob is a first frame that is not a binary frame with a tag.
var errLaneNotJob = errors.New("the first frame is not a job")
// readLaneJob reads a worker's first frame: its tag, and its body up to
// maxFor(tag) bytes. A body past its bound is never kept: tooBig reports it,
// and body is nil. A tag the caller does not take should get a bound of 0;
// its body is then not read at all.
func readLaneJob(ws *websocket.Conn, maxFor func(tag byte) int64) (tag byte, body []byte, tooBig bool, err error) {
mt, r, err := ws.NextReader()
if err != nil {
return 0, nil, false, err
}
if mt != websocket.BinaryMessage {
return 0, nil, false, errLaneNotJob
}
var t [1]byte
if _, err := io.ReadFull(r, t[:]); err != nil {
return 0, nil, false, errLaneNotJob // an empty frame
}
max := maxFor(t[0])
if max <= 0 {
return t[0], nil, false, nil
}
body, err = io.ReadAll(io.LimitReader(r, max+1))
if err != nil {
return t[0], nil, false, err
}
if int64(len(body)) > max {
return t[0], nil, true, nil
}
return t[0], body, false, nil
}
// drainLane reads and drops every frame until the socket fails, holding none
// of them, and returns that failure. A lane that reads nothing after its job
// still has to read, to answer pings and to notice the host going away.
func drainLane(ws *websocket.Conn) error {
for {
_, r, err := ws.NextReader()
if err != nil {
return err
}
if _, err := io.Copy(io.Discard, r); err != nil {
return err
}
}
}
// readHeartbeatReply reads the host's answer to a heartbeat up to
// heartbeatReplyMaxBytes. ok is false for a reply past the bound or one that
// failed to arrive: such a beat is treated as unanswered, so none of its
// directives runs and it does not count as the host answering.
func readHeartbeatReply(resp *http.Response) (body []byte, ok bool) {
body, err := io.ReadAll(io.LimitReader(resp.Body, heartbeatReplyMaxBytes+1))
if err != nil {
return nil, false
}
if len(body) > heartbeatReplyMaxBytes {
fmt.Printf("heartbeat: the host's reply is more than %d MiB; ignored\n", heartbeatReplyMaxBytes>>20)
return nil, false
}
return body, true
}
+126
View File
@@ -0,0 +1,126 @@
package main
// The helpers lane_bounds.go and harness.go added for the reads that were
// inline before (the heartbeat reply in run.go, the agent's /run reply in
// driveHarnessAgent, the drains in harness.go and rootexec.go). They have no
// counterpart on main to run against.
import (
"bytes"
"io"
"net/http"
"net/http/httptest"
"runtime"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/gorilla/websocket"
)
// boxLaneHost stands up the HOST side of one worker socket (host runs on the
// upgraded server end) and returns the box side, dialled.
func boxLaneHost(t *testing.T, host func(c *websocket.Conn)) *websocket.Conn {
t.Helper()
up := websocket.Upgrader{}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
c, err := up.Upgrade(w, r, nil)
if err != nil {
return
}
defer c.Close()
host(c)
}))
t.Cleanup(srv.Close)
ws, _, err := websocket.DefaultDialer.Dial("ws"+strings.TrimPrefix(srv.URL, "http"), nil)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = ws.Close() })
return ws
}
// endless is a body that never ends; served counts what was taken from it.
type endless struct{ served *atomic.Int64 }
func (e endless) Read(p []byte) (int, error) {
for i := range p {
p[i] = 'z'
}
e.served.Add(int64(len(p)))
return len(p), nil
}
func TestHeartbeatReplyIsReadOnlyToItsBound(t *testing.T) {
var served atomic.Int64
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(200)
_, _ = io.Copy(w, io.LimitReader(endless{&served}, 64<<20))
}))
defer srv.Close()
resp, err := http.Get(srv.URL)
if err != nil {
t.Fatal(err)
}
body, ok := readHeartbeatReply(resp)
resp.Body.Close()
if ok || body != nil {
t.Fatalf("a 64 MiB heartbeat reply was taken (%d bytes)", len(body))
}
small := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte(`{"ok":true,"dcn_grants":{"stores":[]}}`))
}))
defer small.Close()
resp, err = http.Get(small.URL)
if err != nil {
t.Fatal(err)
}
body, ok = readHeartbeatReply(resp)
resp.Body.Close()
if !ok || !strings.Contains(string(body), "dcn_grants") {
t.Fatalf("an ordinary reply was not taken: ok=%v %q", ok, body)
}
}
func TestHarnessRunReplyIsReadOnlyToItsBound(t *testing.T) {
var served atomic.Int64
if _, err := readHarnessRunReply(endless{&served}); err == nil || !strings.Contains(err.Error(), "more than the 16 MiB") {
t.Fatalf("an endless /run reply: %v", err)
}
if n := served.Load(); n > 17<<20 {
t.Fatalf("an endless /run reply was read to %d MiB", n>>20)
}
big := append([]byte(`{"ok":true,"pad":"`), bytes.Repeat([]byte{'a'}, 15<<20)...)
big = append(big, `"}`...)
if rb, err := readHarnessRunReply(bytes.NewReader(big)); err != nil || len(rb) != len(big) {
t.Fatalf("a 15 MiB reply the lane carries: %d bytes, %v", len(rb), err)
}
}
// A drained lane holds no frame whole, and ends when the host goes away.
func TestDrainLaneHoldsNoFrame(t *testing.T) {
const frame = 64 << 20
payload := bytes.Repeat([]byte{'d'}, frame)
start := make(chan struct{})
box := boxLaneHost(t, func(c *websocket.Conn) {
<-start
_ = c.WriteMessage(websocket.BinaryMessage, payload)
})
runtime.GC()
var before, after runtime.MemStats
runtime.ReadMemStats(&before)
close(start)
done := make(chan error, 1)
go func() { done <- drainLane(box) }()
select {
case <-done:
case <-time.After(20 * time.Second):
t.Fatal("drainLane did not return after the host went away")
}
runtime.ReadMemStats(&after)
if grew := after.TotalAlloc - before.TotalAlloc; grew > 8<<20 {
t.Fatalf("draining a %d MiB frame allocated %d MiB", frame>>20, grew>>20)
}
}
+481
View File
@@ -0,0 +1,481 @@
//go:build !windows
package main
// What a paired host sends down the harness, session, root-exec and mirror
// lanes, and what a coder run sends its egress proxy, is read at the size this
// machine chose (lane_bounds.go). Every test here drives only code that main
// already has, so each one also runs against main, where it fails.
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"runtime"
"strings"
"sync"
"sync/atomic"
"syscall"
"testing"
"time"
"github.com/gorilla/websocket"
)
// hostSendsAJob is a host that writes one job frame while it reads the box's
// replies, and reports whether its write completed and what came back.
type hostResult struct {
werr error
frames map[byte][]byte
}
func hostSendsAJob(frame []byte, out chan<- hostResult) func(c *websocket.Conn) {
return hostSendsAJobAs(func(c *websocket.Conn) error {
return c.WriteMessage(websocket.BinaryMessage, frame)
}, out)
}
// writeInPieces writes msg as ONE message carried by many small frames: the
// writer flushes a continuation frame each time its 4 KiB buffer fills
// (gorilla's default write buffer), so a bound that counted a frame instead
// of the message would let all of it through.
func writeInPieces(msg []byte) func(c *websocket.Conn) error {
return func(c *websocket.Conn) error {
w, err := c.NextWriter(websocket.BinaryMessage)
if err != nil {
return err
}
for off := 0; off < len(msg); off += 1 << 10 {
if _, err := w.Write(msg[off:min(off+1<<10, len(msg))]); err != nil {
return err
}
}
return w.Close()
}
}
func hostSendsAJobAs(write func(c *websocket.Conn) error, out chan<- hostResult) func(c *websocket.Conn) {
return func(c *websocket.Conn) {
got := map[byte][]byte{}
readDone := make(chan struct{})
go func() {
defer close(readDone)
_ = c.SetReadDeadline(time.Now().Add(20 * time.Second))
for {
_, d, err := c.ReadMessage()
if err != nil || len(d) == 0 {
return
}
got[d[0]] = append(got[d[0]], d[1:]...)
if d[0] == 'X' {
return
}
}
}()
werr := write(c)
<-readDone
out <- hostResult{werr, got}
}
}
// jsonFrame is tag + a JSON object whose field `pad` carries n bytes of 'a'.
func jsonFrame(tag byte, head string, n int) []byte {
b := make([]byte, 0, len(head)+n+16)
b = append(b, tag)
b = append(b, head...)
b = append(b, `,"pad":"`...)
b = append(b, bytes.Repeat([]byte{'a'}, n)...)
b = append(b, `"}`...)
return b
}
// A harness spec far past the bound is not read to its end: the box answers
// that it is too large, names the bound, and closes, so the host's write of
// the rest fails. On main the box read all of it into memory, then ran it.
func TestHarnessWorkerStopsReadingASpecPastItsBound(t *testing.T) {
const frame = 96 << 20
out := make(chan hostResult, 1)
box := boxLaneHost(t, hostSendsAJob(jsonFrame('H', `{"harness_id":"big","mode":"echo","prompt":"ping"`, frame), out))
handleHarnessConn(box, "", "")
_ = box.Close()
r := <-out
if r.werr == nil {
t.Fatalf("the box read the whole %d MiB spec (it replied %.200q); it must stop at its bound", frame>>20, r.frames['J'])
}
if !strings.Contains(string(r.frames['J']), "more than the 32 MiB") || string(r.frames['X']) != "1" {
t.Fatalf("want a refusal naming the 32 MiB bound and exit 1: J=%.200q X=%q", r.frames['J'], r.frames['X'])
}
}
// The bound counts the whole message, not one frame of it: a spec past the
// bound, sent as one message in thousands of 4 KiB frames, is refused the same
// way and not read to its end. On main the box read all of it.
func TestHarnessWorkerBoundCountsEveryFrameOfTheSpec(t *testing.T) {
const frame = 40 << 20
out := make(chan hostResult, 1)
spec := jsonFrame('H', `{"harness_id":"pieces","mode":"echo","prompt":"ping"`, frame)
box := boxLaneHost(t, hostSendsAJobAs(writeInPieces(spec), out))
handleHarnessConn(box, "", "")
_ = box.Close()
r := <-out
if r.werr == nil {
t.Fatalf("the box read the whole %d MiB spec sent in small frames (it replied %.200q); it must stop at its bound", frame>>20, r.frames['J'])
}
if !strings.Contains(string(r.frames['J']), "more than the 32 MiB") || string(r.frames['X']) != "1" {
t.Fatalf("want a refusal naming the 32 MiB bound and exit 1: J=%.200q X=%q", r.frames['J'], r.frames['X'])
}
}
// The same for a session spec sent in small frames.
func TestSessionWorkerBoundCountsEveryFrameOfTheSpec(t *testing.T) {
env, _, _ := sessionTestEnv(t, []string{sessionOpOpen}, nil)
withSink(t)
const frame = 64 << 20
out := make(chan hostResult, 1)
spec := jsonFrame('S', `{"op":"session:open","session_id":"sess-pieces"`, frame)
box := boxLaneHost(t, hostSendsAJobAs(writeInPieces(spec), out))
handleSessionWorkerConn(box, "", env)
_ = box.Close()
r := <-out
if r.werr == nil {
t.Fatalf("the box read the whole %d MiB session spec sent in small frames (E=%q X=%q); it must stop at its bound",
frame>>20, r.frames['E'], r.frames['X'])
}
if !strings.Contains(string(r.frames['E']), "more than the 1024 KiB") || string(r.frames['X']) != "1" {
t.Fatalf("want a refusal naming the bound and exit 1: E=%q X=%q", r.frames['E'], r.frames['X'])
}
}
// A spec the lane can carry still runs: a tree as large as the most this lane
// hands back (12 MiB in base64), with room on top, past the 16 MiB a host
// takes in.
func TestHarnessWorkerTakesALargeSpecTheLaneCarries(t *testing.T) {
out := make(chan hostResult, 1)
box := boxLaneHost(t, hostSendsAJob(jsonFrame('H', `{"harness_id":"tree","mode":"echo","prompt":"ping","worktree_b64":"dGFy"`, 20<<20), out))
handleHarnessConn(box, "", "")
r := <-out
if r.werr != nil {
t.Fatalf("the host could not send a 20 MiB spec: %v", r.werr)
}
var res map[string]any
if err := json.Unmarshal(r.frames['J'], &res); err != nil || res["ok"] != true || res["echo"] != "ping" {
t.Fatalf("a 20 MiB spec did not run: J=%.300q X=%q", r.frames['J'], r.frames['X'])
}
}
// The same on a session worker: an S frame far past its 1 MiB bound is
// refused by name and not read to its end.
func TestSessionWorkerStopsReadingASpecPastItsBound(t *testing.T) {
env, _, _ := sessionTestEnv(t, []string{sessionOpOpen}, nil)
withSink(t) // a box that wrongly took the spec opens no real shell
const frame = 64 << 20
out := make(chan hostResult, 1)
box := boxLaneHost(t, hostSendsAJob(jsonFrame('S', `{"op":"session:open","session_id":"sess-big"`, frame), out))
handleSessionWorkerConn(box, "", env)
_ = box.Close()
r := <-out
if r.werr == nil {
t.Fatalf("the box read the whole %d MiB session spec (E=%q X=%q); it must stop at its bound",
frame>>20, r.frames['E'], r.frames['X'])
}
if !strings.Contains(string(r.frames['E']), "more than the 1024 KiB") || string(r.frames['X']) != "1" {
t.Fatalf("want a refusal naming the bound and exit 1: E=%q X=%q", r.frames['E'], r.frames['X'])
}
}
// A session spec near its bound (a grant, and room the size of a root command
// escaped six times over) still opens a terminal.
func TestSessionWorkerOpensASpecNearItsBound(t *testing.T) {
env, terms, sig := sessionTestEnv(t, []string{sessionOpOpen}, nil)
withFake(t)
// Here the test is the host: it takes the server end, and the box runs on
// the dialled end.
hostEnd := make(chan *websocket.Conn, 1)
hold := make(chan struct{})
t.Cleanup(func() { close(hold) })
boxEnd := boxLaneHost(t, func(c *websocket.Conn) { hostEnd <- c; <-hold })
b := <-hostEnd
head := fmt.Sprintf(`{"op":"session:open","session_id":"sess-near","grant_terms_b64":%q,"grant_sig_b64":%q`, terms, sig)
done := make(chan struct{})
go func() { defer close(done); handleSessionWorkerConn(boxEnd, "", env) }()
if err := b.WriteMessage(websocket.BinaryMessage, jsonFrame('S', head, 900<<10)); err != nil {
t.Fatal(err)
}
if err := b.WriteMessage(websocket.BinaryMessage, []byte("Ihello\n")); err != nil {
t.Fatal(err)
}
got := readUntil(t, b, 'O', 5*time.Second)
if string(got['O']) != "hello\n" {
t.Fatalf("a 900 KiB spec did not open a terminal: %q", got)
}
_ = b.Close()
<-done
}
// sinkProc is a terminal that counts what is typed into it and prints nothing.
type sinkProc struct {
n atomic.Int64
killed chan struct{}
once sync.Once
}
func (p *sinkProc) Read(b []byte) (int, error) { <-p.killed; return 0, io.EOF }
func (p *sinkProc) Write(b []byte) (int, error) { p.n.Add(int64(len(b))); return len(b), nil }
func (p *sinkProc) Resize(c, r int) error { return nil }
func (p *sinkProc) Wait() int { <-p.killed; return 0 }
func (p *sinkProc) Kill() { p.once.Do(func() { close(p.killed) }) }
func (p *sinkProc) User() string { return "sink" }
func (p *sinkProc) Shell() string { return "/bin/sink" }
func withSink(t *testing.T) *sinkProc {
t.Helper()
sp := &sinkProc{killed: make(chan struct{})}
old := sessionSpawn
sessionSpawn = func(op, shell string, cols, rows int) (sessionProc, error) { return sp, nil }
t.Cleanup(func() { sessionSpawn = old; sp.Kill() })
return sp
}
func waitFor(d time.Duration, cond func() bool) bool {
end := time.Now().Add(d)
for time.Now().Before(end) {
if cond() {
return true
}
time.Sleep(5 * time.Millisecond)
}
return cond()
}
// Terminal input is streamed into the pty: a 16 MiB paste, the most a host
// relays in one frame, all arrives, and the box never holds it whole.
func TestSessionInputIsStreamedIntoTheTerminal(t *testing.T) {
env, terms, sig := sessionTestEnv(t, []string{sessionOpOpen}, nil)
sp := withSink(t)
ws := sessionDial(t, env)
sendSpec(t, ws, sessionSpec{Op: sessionOpOpen, GrantB64: terms, SigB64: sig, SessionID: "sess-paste"})
const paste = 16 << 20
frame := append([]byte{'I'}, bytes.Repeat([]byte{'x'}, paste)...)
runtime.GC()
var before, after runtime.MemStats
runtime.ReadMemStats(&before)
if err := ws.WriteMessage(websocket.BinaryMessage, frame); err != nil {
t.Fatal(err)
}
if !waitFor(20*time.Second, func() bool { return sp.n.Load() >= paste }) {
t.Fatalf("the terminal got %d of %d bytes", sp.n.Load(), paste)
}
runtime.ReadMemStats(&after)
if grew := after.TotalAlloc - before.TotalAlloc; grew > 8<<20 {
t.Fatalf("typing a %d MiB paste allocated %d MiB: the frame was held whole", paste>>20, grew>>20)
}
}
// One frame of input past the most a host relays ends the session, and the
// terminal never receives all of it.
func TestSessionInputFramePastItsBoundEndsTheSession(t *testing.T) {
env, terms, sig := sessionTestEnv(t, []string{sessionOpOpen}, nil)
sp := withSink(t)
ws := sessionDial(t, env)
sendSpec(t, ws, sessionSpec{Op: sessionOpOpen, GrantB64: terms, SigB64: sig, SessionID: "sess-huge"})
const frame = 17 << 20
go func() {
_ = ws.WriteMessage(websocket.BinaryMessage, append([]byte{'I'}, bytes.Repeat([]byte{'x'}, frame)...))
}()
ended := func() bool {
select {
case <-sp.killed:
return true
default:
return false
}
}
if !waitFor(20*time.Second, ended) {
t.Fatalf("a %d MiB input frame did not end the session; the terminal got %d bytes", frame>>20, sp.n.Load())
}
if n := sp.n.Load(); n >= frame {
t.Fatalf("the terminal received all %d bytes of a frame past the bound", n)
}
}
// The mirror forwarder streams a host frame into the replica's socket: reading
// 4 KiB of a 64 MiB frame does not need the frame held whole.
func TestMirrorForwarderStreamsAHostFrame(t *testing.T) {
const frame = 64 << 20
payload := bytes.Repeat([]byte{'m'}, frame)
start := make(chan struct{})
up := websocket.Upgrader{}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
c, err := up.Upgrade(w, r, nil)
if err != nil {
return
}
defer c.Close()
_, _, _ = c.ReadMessage() // the BEARER hello
<-start
_ = c.WriteMessage(websocket.BinaryMessage, payload)
time.Sleep(3 * time.Second)
}))
defer srv.Close()
addr := srv.Listener.Addr().String()
dialer := &websocket.Dialer{NetDialContext: func(ctx context.Context, n, a string) (net.Conn, error) {
return (&net.Dialer{}).DialContext(ctx, "tcp", addr)
}}
boxEnd, replica := net.Pipe()
defer replica.Close()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go pipeMirrorConn(ctx, boxEnd, dialer, "ws://"+addr+"/", "b")
buf := make([]byte, 4096)
runtime.GC()
var before, after runtime.MemStats
runtime.ReadMemStats(&before)
close(start)
_ = replica.SetReadDeadline(time.Now().Add(20 * time.Second))
if _, err := io.ReadFull(replica, buf); err != nil || buf[0] != 'm' {
t.Fatalf("the replica read: %v", err)
}
runtime.ReadMemStats(&after)
if grew := after.TotalAlloc - before.TotalAlloc; grew > 8<<20 {
t.Fatalf("passing 4 KiB of a %d MiB host frame to the replica allocated %d MiB: the frame was held whole",
frame>>20, grew>>20)
}
}
// The forwarder carries at most eight connections at once. It listens on a
// fixed port (5433), so the test needs that port free: run it in its own
// network namespace on a machine where something already holds it.
func TestMirrorForwarderCapsConnectionsInFlight(t *testing.T) {
probe, err := net.Listen("tcp", "127.0.0.1:5433")
if err != nil {
t.Skipf("127.0.0.1:5433 is in use here (%v); run this test in its own network namespace", err)
}
_ = probe.Close()
var live, most atomic.Int64
release := make(chan struct{})
up := websocket.Upgrader{}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
c, err := up.Upgrade(w, r, nil)
if err != nil {
return
}
defer c.Close()
n := live.Add(1)
for {
m := most.Load()
if n <= m || most.CompareAndSwap(m, n) {
break
}
}
<-release
live.Add(-1)
}))
defer srv.Close()
defer close(release)
addr := srv.Listener.Addr().String()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go serveMirrorForward(ctx, "127.0.0.1", "127.0.0.0/8", "ws://"+addr+"/", nil,
func(ctx context.Context) (net.Conn, error) { return (&net.Dialer{}).DialContext(ctx, "tcp", addr) }, "b")
var conns []net.Conn
defer func() {
for _, c := range conns {
_ = c.Close()
}
}()
if !waitFor(5*time.Second, func() bool {
c, err := net.Dial("tcp", "127.0.0.1:5433")
if err == nil {
conns = append(conns, c)
}
return err == nil
}) {
t.Fatal("the forwarder never listened")
}
for i := 1; i < 12; i++ {
c, err := net.Dial("tcp", "127.0.0.1:5433")
if err != nil {
t.Fatal(err)
}
conns = append(conns, c)
}
waitFor(3*time.Second, func() bool { return most.Load() >= 12 })
if m := most.Load(); m != 8 {
t.Fatalf("12 local connections made %d WebSockets to the host at once; want 8", m)
}
}
// A coder run's egress proxy reads a request head under a bound: a request
// line that never ends is not read to its end.
func TestCoderEgressHeadIsBounded(t *testing.T) {
boxEnd, runEnd := net.Pipe()
var refused int64
done := make(chan struct{})
go func() { defer close(done); handleCoderEgressConn(boxEnd, nil, &refused) }()
chunk := bytes.Repeat([]byte{'a'}, 64<<10)
_, _ = runEnd.Write([]byte("GET http://example.invalid/"))
sent := 0
var werr error
for sent < 8<<20 {
_ = runEnd.SetWriteDeadline(time.Now().Add(5 * time.Second))
var n int
n, werr = runEnd.Write(chunk)
sent += n
if werr != nil {
break
}
}
_ = runEnd.Close()
<-done
if werr == nil {
t.Fatalf("the proxy read %d MiB of a request line with no end; it must stop at a bound", sent>>20)
}
}
// A coder run's proxy carries at most 1024 connections at once; the next is
// answered 503 at once, never left waiting.
func TestCoderEgressCapsConnectionsInFlight(t *testing.T) {
dir := t.TempDir()
run := "lane-bounds-test"
if err := coderListenEgress(run, dir, os.Getuid()); err != nil {
t.Fatal(err)
}
defer coderCloseEgress(run)
sock := filepath.Join(dir, coderEgressSock)
var conns []net.Conn
defer func() {
for _, c := range conns {
_ = c.Close()
}
}()
for i := 0; i < 1024; i++ {
c, err := net.Dial("unix", sock)
if err != nil {
if errors.Is(err, syscall.EMFILE) || errors.Is(err, syscall.ENFILE) {
t.Skipf("this process may not open %d sockets: %v", 2*1024, err)
}
t.Fatal(err)
}
conns = append(conns, c)
}
// Every one of them is idle in its head, so the proxy holds them all.
time.Sleep(200 * time.Millisecond)
extra, err := net.Dial("unix", sock)
if err != nil {
t.Fatal(err)
}
conns = append(conns, extra)
_ = extra.SetReadDeadline(time.Now().Add(3 * time.Second))
line, _ := io.ReadAll(io.LimitReader(extra, 256))
if !strings.HasPrefix(string(line), "HTTP/1.1 503") {
t.Fatalf("connection 1025 of one run got %q; want a 503 at once", line)
}
}
+197
View File
@@ -0,0 +1,197 @@
package main
// What the host sends down a lane is read at the size the bastion asks for,
// never at the size the host chooses: one frame of any length must not be
// held whole in this machine's memory, and an egress request head that never
// ends must not be either.
import (
"bytes"
"context"
"errors"
"io"
"net"
"net/http"
"net/http/httptest"
"runtime"
"strconv"
"strings"
"testing"
"time"
"github.com/gorilla/websocket"
)
func TestEgressLaneDoesNotHoldAWholeHostFrame(t *testing.T) {
const frame = 64 << 20
payload := make([]byte, frame+1)
payload[0] = 'D'
for i := 1; i < len(payload); i++ {
payload[i] = 'a'
}
start := make(chan struct{})
up := websocket.Upgrader{}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
ws, err := up.Upgrade(w, r, nil)
if err != nil {
return
}
defer ws.Close()
<-start
_ = ws.WriteMessage(websocket.BinaryMessage, payload)
time.Sleep(2 * time.Second)
}))
defer srv.Close()
ws, _, err := websocket.DefaultDialer.Dial("ws"+strings.TrimPrefix(srv.URL, "http"), nil)
if err != nil {
t.Fatal(err)
}
defer ws.Close()
c := &wsConn{ws: ws}
buf := make([]byte, 4096)
runtime.GC()
var before, after runtime.MemStats
runtime.ReadMemStats(&before)
close(start)
n, err := c.Read(buf)
runtime.ReadMemStats(&after)
if err != nil || n == 0 || buf[0] != 'a' {
t.Fatalf("read: n=%d err=%v", n, err)
}
if grew := after.TotalAlloc - before.TotalAlloc; grew > 8<<20 {
t.Fatalf("reading 4 KiB of a %d MiB host frame allocated %d MiB: the frame was held whole", frame>>20, grew>>20)
}
}
func TestEgressRequestHeadIsBounded(t *testing.T) {
bastionEnd, hostEnd := net.Pipe()
done := make(chan struct{})
go func() {
defer close(done)
handleEgressConn(bastionEnd, func(ctx context.Context, network, addr string) (net.Conn, error) {
return nil, errors.New("no dial in this test")
}, "test")
}()
chunk := []byte(strings.Repeat("a", 64<<10))
_, _ = hostEnd.Write([]byte("GET http://example.invalid/"))
sent := 0
var werr error
for sent < 8<<20 {
_ = hostEnd.SetWriteDeadline(time.Now().Add(5 * time.Second))
var n int
n, werr = hostEnd.Write(chunk)
sent += n
if werr != nil {
break
}
}
_ = hostEnd.Close()
<-done
if werr == nil {
t.Fatalf("the bastion read %d MiB of a request line with no end; it must stop at a bound", sent>>20)
}
}
// Streaming a frame must not change what the reader sees: every byte of every
// 'D' frame in order, with empty frames, text frames and unknown tags skipped,
// a frame boundary never visible, and 'E' as the end, read through a buffer
// far smaller than the frames.
func TestEgressLaneFramesReadInOrderThroughASmallBuffer(t *testing.T) {
var want []byte
frames := [][]byte{{'D'}, {}, {'X', 'z', 'z'}}
for i, size := range []int{1, 7, 4096, 70000, 3} {
f := []byte{'D'}
for j := 0; j < size; j++ {
f = append(f, byte('a'+(i+j)%26))
}
want = append(want, f[1:]...)
frames = append(frames, f, []byte{'D'})
}
up := websocket.Upgrader{}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
ws, err := up.Upgrade(w, r, nil)
if err != nil {
return
}
defer ws.Close()
_ = ws.WriteMessage(websocket.TextMessage, []byte("Dnot data"))
for _, f := range frames {
_ = ws.WriteMessage(websocket.BinaryMessage, f)
}
_ = ws.WriteMessage(websocket.BinaryMessage, []byte{'E'})
time.Sleep(time.Second)
}))
defer srv.Close()
ws, _, err := websocket.DefaultDialer.Dial("ws"+strings.TrimPrefix(srv.URL, "http"), nil)
if err != nil {
t.Fatal(err)
}
defer ws.Close()
c := &wsConn{ws: ws}
var got []byte
buf := make([]byte, 13)
for {
n, err := c.Read(buf)
if n == 0 && err == nil {
t.Fatal("Read returned 0, nil")
}
got = append(got, buf[:n]...)
if err == io.EOF {
break
}
if err != nil {
t.Fatalf("read: %v", err)
}
}
if !bytes.Equal(got, want) {
t.Fatalf("read %d bytes, want %d; equal prefix %d", len(got), len(want), commonPrefix(got, want))
}
}
func commonPrefix(a, b []byte) int {
i := 0
for i < len(a) && i < len(b) && a[i] == b[i] {
i++
}
return i
}
// A large upload through the lane still arrives whole: the head bound applies
// to the head alone, and the bytes the reader buffered past it go upstream
// first, in order.
func TestEgressLargeUploadPassesTheHeadBound(t *testing.T) {
bastionEnd, hostEnd := net.Pipe()
upBastion, upFar := net.Pipe()
done := make(chan struct{})
go func() {
defer close(done)
handleEgressConn(bastionEnd, func(ctx context.Context, network, addr string) (net.Conn, error) {
return upBastion, nil
}, "")
}()
body := make([]byte, 3*egressMaxHeadBytes)
for i := range body {
body[i] = byte(i % 251)
}
head := "POST http://198.51.100.20/upload HTTP/1.1\r\nHost: 198.51.100.20\r\nContent-Length: " +
strconv.Itoa(len(body)) + "\r\n\r\n"
// Head and body in ONE write, so the reader holds body bytes when the head ends.
go func() {
_, _ = hostEnd.Write(append([]byte(head), body...))
}()
// The lane rebuilds a plain-HTTP head (egress_http.go): this request comes
// out the same, with `Connection: close` added as its last header.
want := head[:len(head)-len("\r\n")] + "Connection: close\r\n\r\n"
got := make([]byte, len(want)+len(body))
_ = upFar.SetReadDeadline(time.Now().Add(20 * time.Second))
if _, err := io.ReadFull(upFar, got); err != nil {
t.Fatalf("upstream got %v", err)
}
if string(got[:len(want)]) != want || !bytes.Equal(got[len(want):], body) {
t.Fatalf("upstream did not receive the head (%q) and the %d-byte body intact",
got[:len(want)], len(body))
}
_ = hostEnd.Close()
_ = upFar.Close()
<-done
}
+3 -2
View File
@@ -493,7 +493,7 @@ func startLaneHostOn(t *testing.T, raw net.Listener, cert tls.Certificate, clien
} }
} }
} }
for _, p := range []string{"/bastion-harness", "/bastion-mgmt", "/bastion-egress"} { for _, p := range []string{"/bastion-harness", "/bastion-mgmt", "/bastion-egress", "/bastion-serve"} {
mux.HandleFunc(p, ws) mux.HandleFunc(p, ws)
} }
srv := &http.Server{Handler: mux, ErrorLog: log.New(io.Discard, "", 0)} srv := &http.Server{Handler: mux, ErrorLog: log.New(io.Discard, "", 0)}
@@ -622,7 +622,8 @@ func TestTheUnixLaneCarriesTheBearerAndTheSameProtocol(t *testing.T) {
h.mu.Unlock() h.mu.Unlock()
if maxPark != harnessWorkerCount || count["/bastion-harness"] < harnessWorkerCount || if maxPark != harnessWorkerCount || count["/bastion-harness"] < harnessWorkerCount ||
count["/bastion-harness session"] != sessionWorkerCount || count["/bastion-harness session"] != sessionWorkerCount ||
count["/bastion-mgmt"] != mgmtWorkerCount || count["/bastion-egress"] != egressWorkerCount { count["/bastion-mgmt"] != mgmtWorkerCount || count["/bastion-egress"] != egressWorkerCount ||
count["/bastion-serve"] != serveWorkerCount {
t.Fatalf("the worker pools over the socket: %v, parked at once %d", count, maxPark) t.Fatalf("the worker pools over the socket: %v, parked at once %d", count, maxPark)
} }
select { select {
+2 -1
View File
@@ -417,7 +417,8 @@ func TestTheTCPLaneCarriesTheCertTheBearerAndTheSameProtocol(t *testing.T) {
h.mu.Unlock() h.mu.Unlock()
if maxPark != harnessWorkerCount || count["/bastion-harness"] < harnessWorkerCount || if maxPark != harnessWorkerCount || count["/bastion-harness"] < harnessWorkerCount ||
count["/bastion-harness session"] != sessionWorkerCount || count["/bastion-harness session"] != sessionWorkerCount ||
count["/bastion-mgmt"] != mgmtWorkerCount || count["/bastion-egress"] != egressWorkerCount { count["/bastion-mgmt"] != mgmtWorkerCount || count["/bastion-egress"] != egressWorkerCount ||
count["/bastion-serve"] != serveWorkerCount {
t.Fatalf("the worker pools over the port: %v, parked at once %d", count, maxPark) t.Fatalf("the worker pools over the port: %v, parked at once %d", count, maxPark)
} }
for _, n := range sni { for _, n := range sni {
+7
View File
@@ -566,6 +566,13 @@ func main() {
os.Exit(125) os.Exit(125)
} }
os.Exit(opSandboxChildMain(os.Args[2])) os.Exit(opSandboxChildMain(os.Args[2]))
case opSandboxScopeSubcommand:
// A root op's scope (ops_taskcap_linux.go): never typed by a person;
// systemd-run starts it for the daemon.
if len(os.Args) < 3 {
os.Exit(125)
}
os.Exit(opSandboxScopeMain(os.Args[2]))
case opActionSubcommand, opActionChildSubcommand, opActionProbeSubcommand: case opActionSubcommand, opActionChildSubcommand, opActionProbeSubcommand:
// An action op's unit and its sandbox's pid 1 (ops_action_linux.go): // An action op's unit and its sandbox's pid 1 (ops_action_linux.go):
// never typed by a person; the daemon starts them. // never typed by a person; the daemon starts them.
+8
View File
@@ -45,6 +45,9 @@ func TestMain(m *testing.M) {
if len(os.Args) >= 3 && os.Args[1] == opSandboxSubcommand { if len(os.Args) >= 3 && os.Args[1] == opSandboxSubcommand {
os.Exit(opSandboxChildMain(os.Args[2])) os.Exit(opSandboxChildMain(os.Args[2]))
} }
if len(os.Args) >= 3 && os.Args[1] == opSandboxScopeSubcommand {
os.Exit(opSandboxScopeMain(os.Args[2]))
}
if len(os.Args) >= 3 && os.Args[1] == opActionSubcommand { if len(os.Args) >= 3 && os.Args[1] == opActionSubcommand {
os.Exit(opActionOuterMain(os.Args[2])) os.Exit(opActionOuterMain(os.Args[2]))
} }
@@ -115,8 +118,13 @@ func TestMain(m *testing.M) {
// /proc/101 on the machine running the suite and the decision would depend // /proc/101 on the machine running the suite and the decision would depend
// on whatever process holds that number. The real-signal test opts back in. // on whatever process holds that number. The real-signal test opts back in.
procIdentity = func(int) (string, bool, bool) { return "", false, false } procIdentity = func(int) (string, bool, bool) { return "", false, false }
// THE SWEEP IS ONE TOO. A test runs the ops' startup sweep for real, and it
// stops every op unit its names match: the suite's names are its own, so
// it never stops an op the machine's own bastion is running.
opIsolateOpsUnits()
code := m.Run() code := m.Run()
opIsolatedOpsUnitsDone()
_ = os.RemoveAll(dir) _ = os.RemoveAll(dir)
os.Exit(code) os.Exit(code)
} }
+26 -5
View File
@@ -491,6 +491,12 @@ func ensureMirrorNet(rt, name string) (string, string, error) {
// container and a BASE_BACKUP of the user's memory. Nothing here is defence in // container and a BASE_BACKUP of the user's memory. Nothing here is defence in
// depth; this is the depth. // depth; this is the depth.
// //
// mirrorForwardMaxConns bounds the connections the forwarder carries at once.
// The replica holds at most two (`pg_basebackup -X stream` while it seeds; one
// walreceiver after), so four times that leaves room for a diagnostic session
// and a reconnect that overlaps the connection it replaces.
const mirrorForwardMaxConns = 8
// One WebSocket per accepted connection, because `pg_basebackup -X stream` // One WebSocket per accepted connection, because `pg_basebackup -X stream`
// holds two replication connections at once and multiplexing them would mean // holds two replication connections at once and multiplexing them would mean
// inventing a framing layer for no gain. // inventing a framing layer for no gain.
@@ -507,6 +513,11 @@ func serveMirrorForward(ctx context.Context, gw, subnet, wsURL string,
allowed = n allowed = n
} }
} }
// Connections carried at once, across both listeners. Loopback is let in
// for diagnostics, so any local user can connect, and each connection is a
// WebSocket to the host and two pumps here; past the cap one is closed at
// once and its client tries again.
slots := make(chan struct{}, mirrorForwardMaxConns)
for _, addr := range []string{ for _, addr := range []string{
net.JoinHostPort(gw, strconv.Itoa(mirrorForwardPort)), net.JoinHostPort(gw, strconv.Itoa(mirrorForwardPort)),
net.JoinHostPort("127.0.0.1", strconv.Itoa(mirrorForwardPort)), net.JoinHostPort("127.0.0.1", strconv.Itoa(mirrorForwardPort)),
@@ -527,7 +538,15 @@ func serveMirrorForward(ctx context.Context, gw, subnet, wsURL string,
_ = c.Close() _ = c.Close()
continue continue
} }
go pipeMirrorConn(ctx, c, dialer, wsURL, bearer) select {
case slots <- struct{}{}:
go func() {
defer func() { <-slots }()
pipeMirrorConn(ctx, c, dialer, wsURL, bearer)
}()
default:
_ = c.Close()
}
} }
}(ln) }(ln)
} }
@@ -578,15 +597,17 @@ func pipeMirrorConn(ctx context.Context, local net.Conn, dialer *websocket.Diale
done <- struct{}{} done <- struct{}{}
}() }()
go func() { go func() {
// Each host frame is streamed into the replica's socket as it arrives,
// never held whole: the host picks a frame's length (lane_bounds.go).
for { for {
mt, data, err := ws.ReadMessage() mt, r, err := ws.NextReader()
if err != nil { if err != nil {
break break
} }
if mt != websocket.BinaryMessage || len(data) == 0 { if mt != websocket.BinaryMessage {
continue continue // NextReader drops what is left of it
} }
if _, werr := local.Write(data); werr != nil { if _, err := io.Copy(local, r); err != nil {
break break
} }
} }
+29 -15
View File
@@ -19,7 +19,9 @@ import (
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"os" "os"
"slices"
"strings" "strings"
"sync"
"testing" "testing"
"time" "time"
) )
@@ -186,17 +188,29 @@ func TestFilterVerdictPolicy(t *testing.T) {
// fakeOllama answers the handful of routes the filter cares about and records // fakeOllama answers the handful of routes the filter cares about and records
// what actually reached it — the point of most of these assertions is that a // what actually reached it — the point of most of these assertions is that a
// refused request never arrives at all. // refused request never arrives at all. Its handler runs on the server's
// goroutines, beside the test's (and a pairing's beat probes it on its own),
// so the record is read through requests(), under the lock.
type fakeOllama struct { type fakeOllama struct {
srv *httptest.Server srv *httptest.Server
mu sync.Mutex
seen []string seen []string
} }
// requests is a copy of what reached the fake so far.
func (f *fakeOllama) requests() []string {
f.mu.Lock()
defer f.mu.Unlock()
return slices.Clone(f.seen)
}
func newFakeOllama(t *testing.T) *fakeOllama { func newFakeOllama(t *testing.T) *fakeOllama {
t.Helper() t.Helper()
f := &fakeOllama{} f := &fakeOllama{}
f.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { f.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
f.mu.Lock()
f.seen = append(f.seen, r.Method+" "+r.URL.Path) f.seen = append(f.seen, r.Method+" "+r.URL.Path)
f.mu.Unlock()
switch r.URL.Path { switch r.URL.Path {
case "/api/tags": case "/api/tags":
_, _ = w.Write([]byte(`{"models":[{"name":"qwen4-embed:latest","size":42},` + _, _ = w.Write([]byte(`{"models":[{"name":"qwen4-embed:latest","size":42},` +
@@ -264,8 +278,8 @@ func TestFilteredForwardRefusesAnotherModelAndNeverForwardsIt(t *testing.T) {
if b := bodyOf(t, resp); !strings.Contains(b, "llama3:70b") || !strings.Contains(b, "qwen4-embed") { if b := bodyOf(t, resp); !strings.Contains(b, "llama3:70b") || !strings.Contains(b, "qwen4-embed") {
t.Fatalf("the refusal must name what was asked for and what is shared: %s", b) t.Fatalf("the refusal must name what was asked for and what is shared: %s", b)
} }
if len(up.seen) != 0 { if len(up.requests()) != 0 {
t.Fatalf("a refused request reached the upstream: %v", up.seen) t.Fatalf("a refused request reached the upstream: %v", up.requests())
} }
resp = roundTrip(t, addr, laneOllama, tMember, "POST", "/api/embed", resp = roundTrip(t, addr, laneOllama, tMember, "POST", "/api/embed",
@@ -273,8 +287,8 @@ func TestFilteredForwardRefusesAnotherModelAndNeverForwardsIt(t *testing.T) {
if resp.StatusCode != http.StatusOK { if resp.StatusCode != http.StatusOK {
t.Fatalf("the shared model must work: %d", resp.StatusCode) t.Fatalf("the shared model must work: %d", resp.StatusCode)
} }
if len(up.seen) != 1 || up.seen[0] != "POST /api/embed" { if len(up.requests()) != 1 || up.requests()[0] != "POST /api/embed" {
t.Fatalf("the allowed request must reach the upstream once: %v", up.seen) t.Fatalf("the allowed request must reach the upstream once: %v", up.requests())
} }
} }
@@ -295,8 +309,8 @@ func TestFilteredForwardRefusesTheStoreMutatingHalf(t *testing.T) {
t.Fatalf("%s: status %d, want 403", p, resp.StatusCode) t.Fatalf("%s: status %d, want 403", p, resp.StatusCode)
} }
} }
if len(up.seen) != 0 { if len(up.requests()) != 0 {
t.Fatalf("a store-mutating request reached ollama: %v", up.seen) t.Fatalf("a store-mutating request reached ollama: %v", up.requests())
} }
// ...and using a model is untouched by that refusal. // ...and using a model is untouched by that refusal.
if resp := roundTrip(t, addr, laneOllama, tMember, "POST", "/api/embed", if resp := roundTrip(t, addr, laneOllama, tMember, "POST", "/api/embed",
@@ -344,8 +358,8 @@ func TestAnchoredBoxRefusesModelChangesThroughTheForwardForTheAnchor(t *testing.
t.Fatalf("%s for the anchor: status %d, want 403", p, resp.StatusCode) t.Fatalf("%s for the anchor: status %d, want 403", p, resp.StatusCode)
} }
} }
if len(up.seen) != 0 { if len(up.requests()) != 0 {
t.Fatalf("a store-mutating request from the anchor reached ollama: %v", up.seen) t.Fatalf("a store-mutating request from the anchor reached ollama: %v", up.requests())
} }
if resp := roundTrip(t, addr, laneOllama, tAnchor, "POST", "/api/embed", `{"model":"anything"}`); resp.StatusCode != http.StatusOK { if resp := roundTrip(t, addr, laneOllama, tAnchor, "POST", "/api/embed", `{"model":"anything"}`); resp.StatusCode != http.StatusOK {
t.Fatalf("the anchor still USES models: %d", resp.StatusCode) t.Fatalf("the anchor still USES models: %d", resp.StatusCode)
@@ -581,8 +595,8 @@ func TestOversizedBodyIsRefusedNotForwarded(t *testing.T) {
if resp.StatusCode != http.StatusForbidden { if resp.StatusCode != http.StatusForbidden {
t.Fatalf("a request naming no model must be refused: %d", resp.StatusCode) t.Fatalf("a request naming no model must be refused: %d", resp.StatusCode)
} }
if len(up.seen) != 0 { if len(up.requests()) != 0 {
t.Fatalf("it reached the upstream anyway: %v", up.seen) t.Fatalf("it reached the upstream anyway: %v", up.requests())
} }
} }
@@ -612,8 +626,8 @@ func TestAScopedPairingCannotRenameAModelIntoItsScope(t *testing.T) {
t.Fatalf("%s: the refusal must say why: %s", p, b) t.Fatalf("%s: the refusal must say why: %s", p, b)
} }
} }
if len(up.seen) != 0 { if len(up.requests()) != 0 {
t.Fatalf("a rename reached ollama: %v", up.seen) t.Fatalf("a rename reached ollama: %v", up.requests())
} }
} }
@@ -803,8 +817,8 @@ func TestAnEndlessHeaderIsCutOff(t *testing.T) {
t.Fatal("an endless header was read without limit") t.Fatal("an endless header was read without limit")
} }
client.Close() client.Close()
if len(up.seen) != 0 { if len(up.requests()) != 0 {
t.Fatalf("it reached the upstream: %v", up.seen) t.Fatalf("it reached the upstream: %v", up.requests())
} }
} }
+70 -28
View File
@@ -422,35 +422,50 @@ var (
modelDaemonMu sync.Mutex modelDaemonMu sync.Mutex
modelDaemonUp bool modelDaemonUp bool
modelDaemonAt time.Time modelDaemonAt time.Time
modelDaemonAddr string // the address the cached answer is for
) )
// modelDaemonReachable probes the SAME address the rest of this binary uses // modelDaemonReachable probes the SAME address the rest of this binary uses
// (localOllamaAddr, run.go:714-719) so the box can never advertise management // (localOllamaAddr, run.go:714-719) so the box can never advertise management
// of a daemon the forwards do not point at. // of a daemon the forwards do not point at.
func modelDaemonReachable() bool { func modelDaemonReachable() bool { return modelDaemonReachableAt(localOllamaAddr()) }
// modelDaemonReachableAt is modelDaemonReachable's probe of one address. The
// cached answer is for the address it was asked of, so a pairing whose
// address is the vetted one (modelDaemonTarget) never reads another's answer.
func modelDaemonReachableAt(addr string) bool {
modelDaemonMu.Lock() modelDaemonMu.Lock()
defer modelDaemonMu.Unlock() defer modelDaemonMu.Unlock()
if !modelDaemonAt.IsZero() && time.Since(modelDaemonAt) < modelDaemonCacheFor { if !modelDaemonAt.IsZero() && modelDaemonAddr == addr && time.Since(modelDaemonAt) < modelDaemonCacheFor {
return modelDaemonUp return modelDaemonUp
} }
cl := &http.Client{Timeout: 3 * time.Second} cl := &http.Client{Timeout: 3 * time.Second}
resp, err := cl.Get("http://" + localOllamaAddr() + "/api/tags") resp, err := cl.Get("http://" + addr + "/api/tags")
up := false up := false
if err == nil { if err == nil {
up = resp.StatusCode == http.StatusOK up = resp.StatusCode == http.StatusOK
resp.Body.Close() resp.Body.Close()
} }
modelDaemonUp, modelDaemonAt = up, time.Now() modelDaemonUp, modelDaemonAt, modelDaemonAddr = up, time.Now(), addr
return up return up
} }
// resetModelDaemonCache exists for the tests. // resetModelDaemonCache exists for the tests.
func resetModelDaemonCache() { func resetModelDaemonCache() {
modelDaemonMu.Lock() modelDaemonMu.Lock()
modelDaemonUp, modelDaemonAt = false, time.Time{} modelDaemonUp, modelDaemonAt, modelDaemonAddr = false, time.Time{}, ""
modelDaemonMu.Unlock() modelDaemonMu.Unlock()
} }
// modelSelf is the pairing a model:* job came from, or "" when the job was
// driven without one (modelDaemonTarget then keeps OLLAMA_ADDR, as before).
func modelSelf(ws *websocket.Conn) string {
if a := mgmtAccountOf(ws); a != nil {
return a.self
}
return ""
}
// modelSwapCapable answers the heartbeat's question: could this box have a // modelSwapCapable answers the heartbeat's question: could this box have a
// model placed on it BY THIS PAIRING. Two conditions, and the caps one is // model placed on it BY THIS PAIRING. Two conditions, and the caps one is
// first so a pairing outside the family costs no probe — the rule the // first so a pairing outside the family costs no probe — the rule the
@@ -459,13 +474,19 @@ func modelSwapCapable(self string) bool {
if capsAllow(self, modelGateOp) != "" { if capsAllow(self, modelGateOp) != "" {
return false return false
} }
return modelDaemonReachable() // A Host with no platform is offered management only of a model server on
// this machine, and a refused one is not probed at all (modelDaemonTarget).
addr, why := modelDaemonTarget(self)
if why != "" {
return false
}
return modelDaemonReachableAt(addr)
} }
// modelPostJSON posts one JSON body and returns (status, body). The body is // modelPostJSON posts one JSON body and returns (status, body). The body is
// read whatever the status is: a streaming ollama reports a FAILED pull inside // read whatever the status is: a streaming ollama reports a FAILED pull inside
// a 200 (measured, see the header), so the caller needs both. // a 200 (measured, see the header), so the caller needs both.
func modelPostJSON(method, path string, payload any, timeout time.Duration) (int, []byte, error) { func modelPostJSON(addr, method, path string, payload any, timeout time.Duration) (int, []byte, error) {
var rdr io.Reader var rdr io.Reader
if payload != nil { if payload != nil {
b, err := json.Marshal(payload) b, err := json.Marshal(payload)
@@ -474,7 +495,7 @@ func modelPostJSON(method, path string, payload any, timeout time.Duration) (int
} }
rdr = bytes.NewReader(b) rdr = bytes.NewReader(b)
} }
req, err := http.NewRequest(method, "http://"+localOllamaAddr()+path, rdr) req, err := http.NewRequest(method, "http://"+addr+path, rdr)
if err != nil { if err != nil {
return 0, nil, err return 0, nil, err
} }
@@ -528,8 +549,8 @@ func modelCosine(a, b []float64) float64 {
return dot / (math.Sqrt(na) * math.Sqrt(nb)) return dot / (math.Sqrt(na) * math.Sqrt(nb))
} }
func modelEmbedOnce(model, text string) ([]float64, error) { func modelEmbedOnce(addr, model, text string) ([]float64, error) {
status, body, err := modelPostJSON(http.MethodPost, "/api/embed", status, body, err := modelPostJSON(addr, http.MethodPost, "/api/embed",
map[string]any{"model": model, "input": []string{text}}, modelProbeTimeout) map[string]any{"model": model, "input": []string{text}}, modelProbeTimeout)
if err != nil { if err != nil {
return nil, err return nil, err
@@ -549,8 +570,8 @@ func modelEmbedOnce(model, text string) ([]float64, error) {
return out.Embeddings[0], nil return out.Embeddings[0], nil
} }
func modelGenerateOnce(model, prompt string) (string, error) { func modelGenerateOnce(addr, model, prompt string) (string, error) {
status, body, err := modelPostJSON(http.MethodPost, "/api/generate", status, body, err := modelPostJSON(addr, http.MethodPost, "/api/generate",
map[string]any{"model": model, "prompt": prompt, "stream": false, "think": false}, modelProbeTimeout) map[string]any{"model": model, "prompt": prompt, "stream": false, "think": false}, modelProbeTimeout)
if err != nil { if err != nil {
return "", err return "", err
@@ -573,12 +594,12 @@ func modelGenerateOnce(model, prompt string) (string, error) {
// probeEmbedModel proves the model EMBEDS: the same text twice must come back // probeEmbedModel proves the model EMBEDS: the same text twice must come back
// as one answer, and a different text must move it. Returns the measured width // as one answer, and a different text must move it. Returns the measured width
// and "" , or 0 and the reason it is not usable. // and "" , or 0 and the reason it is not usable.
func probeEmbedModel(model string) (int, string) { func probeEmbedModel(addr, model string) (int, string) {
v1, err := modelEmbedOnce(model, modelProbeTextA) v1, err := modelEmbedOnce(addr, model, modelProbeTextA)
if err != nil { if err != nil {
return 0, "it was downloaded but does not answer as an embedding model: " + err.Error() return 0, "it was downloaded but does not answer as an embedding model: " + err.Error()
} }
v2, err := modelEmbedOnce(model, modelProbeTextA) v2, err := modelEmbedOnce(addr, model, modelProbeTextA)
if err != nil { if err != nil {
return 0, "it answered once and then failed: " + err.Error() return 0, "it answered once and then failed: " + err.Error()
} }
@@ -587,7 +608,7 @@ func probeEmbedModel(model string) (int, string) {
return 0, fmt.Sprintf("it gave two different answers to the SAME text (similarity %.4f) — "+ return 0, fmt.Sprintf("it gave two different answers to the SAME text (similarity %.4f) — "+
"nothing routed at it would be reproducible", same) "nothing routed at it would be reproducible", same)
} }
v3, err := modelEmbedOnce(model, modelProbeTextB) v3, err := modelEmbedOnce(addr, model, modelProbeTextB)
if err != nil { if err != nil {
return 0, "the control text failed: " + err.Error() return 0, "the control text failed: " + err.Error()
} }
@@ -604,15 +625,15 @@ func probeEmbedModel(model string) (int, string) {
// probeGenerateModel proves the model GENERATES: a non-empty answer, and a // probeGenerateModel proves the model GENERATES: a non-empty answer, and a
// different prompt must change it (the same constant-answer trap). // different prompt must change it (the same constant-answer trap).
func probeGenerateModel(model string) string { func probeGenerateModel(addr, model string) string {
a, err := modelGenerateOnce(model, modelProbePromptA) a, err := modelGenerateOnce(addr, model, modelProbePromptA)
if err != nil { if err != nil {
return "it was downloaded but does not answer as a text model: " + err.Error() return "it was downloaded but does not answer as a text model: " + err.Error()
} }
if strings.TrimSpace(a) == "" { if strings.TrimSpace(a) == "" {
return "it answered with nothing at all" return "it answered with nothing at all"
} }
b, err := modelGenerateOnce(model, modelProbePromptB) b, err := modelGenerateOnce(addr, model, modelProbePromptB)
if err != nil { if err != nil {
return "it answered once and then failed: " + err.Error() return "it answered once and then failed: " + err.Error()
} }
@@ -704,7 +725,12 @@ func modelReply(ws *websocket.Conn, v any) {
// here — the two together are what makes an honest proposal possible: the host // here — the two together are what makes an honest proposal possible: the host
// can see a model is present without being able to claim it may remove it. // can see a model is present without being able to claim it may remove it.
func handleModelList(ws *websocket.Conn, root string, _ []string) { func handleModelList(ws *websocket.Conn, root string, _ []string) {
status, body, err := modelPostJSON(http.MethodGet, "/api/tags", nil, modelListTimeout) addr, why := modelDaemonTarget(modelSelf(ws))
if why != "" {
modelFail(ws, "model:list: "+why)
return
}
status, body, err := modelPostJSON(addr, http.MethodGet, "/api/tags", nil, modelListTimeout)
if err != nil { if err != nil {
modelFail(ws, "model:list: no model daemon answering on this machine: "+err.Error()) modelFail(ws, "model:list: no model daemon answering on this machine: "+err.Error())
return return
@@ -974,12 +1000,12 @@ func modelPullRunningNote(j *modelPullJob) string {
// runBackupFetch updates job.Got inside its read loop (backupfetch.go:309-311): // runBackupFetch updates job.Got inside its read loop (backupfetch.go:309-311):
// progress that only exists when the transfer ends is not progress. nil is // progress that only exists when the transfer ends is not progress. nil is
// allowed — the direct-call tests pass it. // allowed — the direct-call tests pass it.
func modelPullStreaming(model string, maxBytes int64, note func(want, got int64)) (int64, error) { func modelPullStreaming(addr, model string, maxBytes int64, note func(want, got int64)) (int64, error) {
body, err := json.Marshal(map[string]any{"model": model, "stream": true}) body, err := json.Marshal(map[string]any{"model": model, "stream": true})
if err != nil { if err != nil {
return 0, err return 0, err
} }
httpReq, err := http.NewRequest(http.MethodPost, "http://"+localOllamaAddr()+"/api/pull", httpReq, err := http.NewRequest(http.MethodPost, "http://"+addr+"/api/pull",
bytes.NewReader(body)) bytes.NewReader(body))
if err != nil { if err != nil {
return 0, err return 0, err
@@ -1109,6 +1135,17 @@ func modelPullStart(ws *websocket.Conn, root string, req modelReq) {
return return
} }
// WHERE THE PULL GOES, decided once and held for the download and its
// probe: a Host with no platform pulls only onto a model server on this
// machine (modelDaemonTarget). Refused before the approval is read, so a
// pull that cannot go here costs the person nothing. For such a pairing a
// name in OLLAMA_ADDR is looked up here; nothing is dialled.
addr, why := modelDaemonTarget(modelSelf(ws))
if why != "" {
modelRefuse(ws, req.Model, why)
return
}
// THE APPROVAL GATE. Nothing above this line has touched the network; nothing // THE APPROVAL GATE. Nothing above this line has touched the network; nothing
// below it runs until the user has said yes to THIS proposal on this machine. // below it runs until the user has said yes to THIS proposal on this machine.
// The async split moved the download off this frame and deliberately did NOT // The async split moved the download off this frame and deliberately did NOT
@@ -1152,7 +1189,7 @@ func modelPullStart(ws *websocket.Conn, root string, req modelReq) {
snap := *job snap := *job
modelPullJobsMu.Unlock() modelPullJobsMu.Unlock()
go runModelPull(job, root, req) go runModelPull(job, root, req, addr)
modelReply(ws, snap) modelReply(ws, snap)
} }
@@ -1167,7 +1204,7 @@ func modelPullStart(ws *websocket.Conn, root string, req modelReq) {
// bytes on this disk", which is true the moment the download succeeds. A model // bytes on this disk", which is true the moment the download succeeds. A model
// that downloaded but failed its probe must stay droppable, or a bad pull would // that downloaded but failed its probe must stay droppable, or a bad pull would
// leave dead weight on the user's machine that nothing is allowed to remove. // leave dead weight on the user's machine that nothing is allowed to remove.
func runModelPull(job *modelPullJob, root string, req modelReq) { func runModelPull(job *modelPullJob, root string, req modelReq, addr string) {
set := func(f func(j *modelPullJob)) { set := func(f func(j *modelPullJob)) {
modelPullJobsMu.Lock() modelPullJobsMu.Lock()
f(job) f(job)
@@ -1201,7 +1238,7 @@ func runModelPull(job *modelPullJob, root string, req modelReq) {
// The 200-with-an-error-in-the-body case that stream:false was chosen for is // The 200-with-an-error-in-the-body case that stream:false was chosen for is
// handled either way: modelBodyError reads EVERY line, so a failure on the // handled either way: modelBodyError reads EVERY line, so a failure on the
// last line of a stream is caught exactly as before. // last line of a stream is caught exactly as before.
total, err := modelPullStreaming(req.Model, req.MaxBytes, total, err := modelPullStreaming(addr, req.Model, req.MaxBytes,
func(want, got int64) { set(func(j *modelPullJob) { j.Want, j.Got = want, got }) }) func(want, got int64) { set(func(j *modelPullJob) { j.Want, j.Got = want, got }) })
if err != nil { if err != nil {
finish("failed", func(j *modelPullJob) { finish("failed", func(j *modelPullJob) {
@@ -1242,9 +1279,9 @@ func runModelPull(job *modelPullJob, root string, req modelReq) {
var dim int var dim int
var why string var why string
if req.Role == "embed" { if req.Role == "embed" {
dim, why = probeEmbedModel(req.Model) dim, why = probeEmbedModel(addr, req.Model)
} else { } else {
why = probeGenerateModel(req.Model) why = probeGenerateModel(addr, req.Model)
} }
if why != "" { if why != "" {
finish("done", func(j *modelPullJob) { finish("done", func(j *modelPullJob) {
@@ -1284,7 +1321,12 @@ func handleModelDrop(ws *websocket.Conn, root string, args []string) {
"not remove it. Only a model this host pulled here is its to drop.") "not remove it. Only a model this host pulled here is its to drop.")
return return
} }
status, body, err := modelPostJSON(http.MethodDelete, "/api/delete", addr, why := modelDaemonTarget(modelSelf(ws))
if why != "" {
modelFail(ws, "model:drop: "+why)
return
}
status, body, err := modelPostJSON(addr, http.MethodDelete, "/api/delete",
map[string]any{"model": req.Model}, modelListTimeout) map[string]any{"model": req.Model}, modelListTimeout)
if err != nil { if err != nil {
modelFail(ws, "model:drop: the model daemon did not answer: "+err.Error()) modelFail(ws, "model:drop: the model daemon did not answer: "+err.Error())
+1 -1
View File
@@ -446,7 +446,7 @@ func effectOK(f string) error {
return errors.New("must be absolute") return errors.New("must be absolute")
} }
if strings.Contains(f, "{") || strings.Contains(f, "..") { if strings.Contains(f, "{") || strings.Contains(f, "..") {
return errors.New("no placeholders, no ..") return errors.New("no placeholders, no \"..\"")
} }
if strings.Contains(f, "\\") { if strings.Contains(f, "\\") {
// A backslash is filepath.Match's escape: `\n` is a pattern the // A backslash is filepath.Match's escape: `\n` is a pattern the
+12 -3
View File
@@ -72,7 +72,6 @@ const (
opUIDFile = "op_uid_next" opUIDFile = "op_uid_next"
opActionRunTmpfsBytes = 16 << 20 opActionRunTmpfsBytes = 16 << 20
opBuilderInSandbox = "/run/privatae-builder/buildkitd.sock" opBuilderInSandbox = "/run/privatae-builder/buildkitd.sock"
opActionUnitPrefix = "privatae-op-"
landlockRuleNetPort = 2 // LANDLOCK_RULE_NET_PORT landlockRuleNetPort = 2 // LANDLOCK_RULE_NET_PORT
// opActionRetryAfter: a machine that could not run action ops is asked // opActionRetryAfter: a machine that could not run action ops is asked
// again after this long, not only at the next restart. // again after this long, not only at the next restart.
@@ -817,12 +816,21 @@ type landlockNetPortAttr struct {
Port uint64 Port uint64
} }
// landlockRulesetAttr is the kernel's struct landlock_ruleset_attr up to
// "scoped", declared here like landlockNetPortAttr so the call does not move
// with x/sys's naming of it.
type landlockRulesetAttr struct {
HandledAccessFS uint64
HandledAccessNet uint64
Scoped uint64
}
// opLandlock: TCP connects to the declared ports only, no TCP bind, and no // opLandlock: TCP connects to the declared ports only, no TCP bind, and no
// abstract unix socket or signal that leaves the sandbox. The addresses are // abstract unix socket or signal that leaves the sandbox. The addresses are
// the unit's allowlist; the two together are the destinations. // the unit's allowlist; the two together are the destinations.
func opLandlock(ports []uint16) error { func opLandlock(ports []uint16) error {
attr := unix.LandlockRulesetAttr{ attr := landlockRulesetAttr{
Access_net: unix.LANDLOCK_ACCESS_NET_BIND_TCP | unix.LANDLOCK_ACCESS_NET_CONNECT_TCP, HandledAccessNet: unix.LANDLOCK_ACCESS_NET_BIND_TCP | unix.LANDLOCK_ACCESS_NET_CONNECT_TCP,
Scoped: unix.LANDLOCK_SCOPE_ABSTRACT_UNIX_SOCKET | unix.LANDLOCK_SCOPE_SIGNAL, Scoped: unix.LANDLOCK_SCOPE_ABSTRACT_UNIX_SOCKET | unix.LANDLOCK_SCOPE_SIGNAL,
} }
fd, _, e := unix.Syscall(unix.SYS_LANDLOCK_CREATE_RULESET, uintptr(unsafe.Pointer(&attr)), unsafe.Sizeof(attr), 0) fd, _, e := unix.Syscall(unix.SYS_LANDLOCK_CREATE_RULESET, uintptr(unsafe.Pointer(&attr)), unsafe.Sizeof(attr), 0)
@@ -861,6 +869,7 @@ func opStartupSweep() {
opKillUnit(u) opKillUnit(u)
_ = exec.Command("systemctl", "stop", u).Run() _ = exec.Command("systemctl", "stop", u).Run()
} }
opSweepScopes()
entries, _ := os.ReadDir(root) entries, _ := os.ReadDir(root)
for _, e := range entries { for _, e := range entries {
if !strings.HasPrefix(e.Name(), "run-") { if !strings.HasPrefix(e.Name(), "run-") {
+4 -3
View File
@@ -7,9 +7,10 @@ package main
// //
// Run them in a private PID namespace but NOT a private mount namespace: the // Run them in a private PID namespace but NOT a private mount namespace: the
// unit is started by the machine's systemd, in the machine's mounts, and must // unit is started by the machine's systemd, in the machine's mounts, and must
// see the run's tmpfs the test mounted: // see the run's tmpfs the test mounted. GOTMPDIR and TMPDIR name a folder
// on a disk, not under /tmp (a run gets its own /tmp):
// //
// GOTMPDIR=/srv/work/tmp-gotest TMPDIR=/srv/work/tmp-gotest unshare --pid --fork go test -run Action . // GOTMPDIR=<dir> TMPDIR=<dir> unshare --pid --fork go test -run Action .
import ( import (
"bytes" "bytes"
@@ -725,7 +726,7 @@ func TestMemoryFloorSitsAboveTheKubeletsEviction(t *testing.T) {
// A kill that does not land (no such unit, no cgroup yet) is never reported // A kill that does not land (no such unit, no cgroup yet) is never reported
// as the op stopped. // as the op stopped.
func TestTheMemoryWatchReportsOnlyAKillThatLanded(t *testing.T) { func TestTheMemoryWatchReportsOnlyAKillThatLanded(t *testing.T) {
stop := opMemoryWatch("privatae-op-nonexistent-optest", 1<<62) stop := opMemoryWatch(opActionUnitPrefix+"nonexistent", 1<<62)
time.Sleep(4 * opMemoryWatchPeriod) time.Sleep(4 * opMemoryWatchPeriod)
v := stop() v := stop()
if v.Stopped { if v.Stopped {
+1 -1
View File
@@ -131,7 +131,7 @@ func TestActionManifestRefusesWhatIsNotDeclaredOrNotBounded(t *testing.T) {
} }
// A files op carrying an action op's fields is refused, not read as one. // A files op carrying an action op's fields is refused, not read as one.
files := strings.Replace(opTestManifest, "modes:", "verbs:\n - {verb: unit.restart, unit: a.service}\nmodes:", 1) files := strings.Replace(opTestManifest, "modes:", "verbs:\n - {verb: unit.restart, unit: a.service}\nmodes:", 1)
if _, err := parseOpManifest([]byte(strings.Replace(files, "%s/", "/srv/work/optest/x/", 1))); err == nil || !strings.Contains(err.Error(), "belong to an action op") { if _, err := parseOpManifest([]byte(strings.Replace(files, "%s/", "/srv/privatae-optest/x/", 1))); err == nil || !strings.Contains(err.Error(), "belong to an action op") {
t.Fatalf("a files op with verbs: %v", err) t.Fatalf("a files op with verbs: %v", err)
} }
} }
+82 -14
View File
@@ -224,6 +224,11 @@ type opReport struct {
// per change, in path order. An op_run apply is bound to it (ops.go opTerms.Changes). // per change, in path order. An op_run apply is bound to it (ops.go opTerms.Changes).
ChangesSHA256 string `json:"changes_sha256"` ChangesSHA256 string `json:"changes_sha256"`
Committed []string `json:"committed,omitempty"` Committed []string `json:"committed,omitempty"`
// TasksCapped: a cap held the run's tasks, root's included, at
// opSandboxTasksMax. A root op where none could be held runs uncapped,
// and TasksNote says so and why.
TasksCapped bool `json:"tasks_capped,omitempty"`
TasksNote string `json:"tasks_note,omitempty"`
// An action op's apply (ops_verbs_linux.go): each verb as it ran, each // An action op's apply (ops_verbs_linux.go): each verb as it ran, each
// check after it, and the receipts for what its program pushed. // check after it, and the receipts for what its program pushed.
Verbs []opVerbResult `json:"verbs,omitempty"` Verbs []opVerbResult `json:"verbs,omitempty"`
@@ -421,18 +426,49 @@ func opRunSandboxed(opDir string, m *opManifest, mode, params string) (*opReport
limit := time.Duration(m.TimeLimit) * time.Second limit := time.Duration(m.TimeLimit) * time.Second
ctx, cancel := context.WithTimeout(context.Background(), limit) ctx, cancel := context.WithTimeout(context.Background(), limit)
defer cancel() defer cancel()
cmd := exec.CommandContext(ctx, self, opSandboxSubcommand, specPath) // A user op's tasks are held by RLIMIT_NPROC. Root is exempt from it, so
cmd.SysProcAttr = &syscall.SysProcAttr{ // a root op's are held by a scope of its own (ops_taskcap_linux.go), or
Cloneflags: syscall.CLONE_NEWNS | syscall.CLONE_NEWPID | syscall.CLONE_NEWNET | syscall.CLONE_NEWIPC | syscall.CLONE_NEWUTS, // by nothing where none can be made, and the report says which.
Setpgid: true, unit := ""
if spec.AsUser {
rep.TasksCapped = true
} else if why := opTasksScopeWhyNot(); why != "" {
rep.TasksNote = opTasksNotCapped(why)
} else {
unit = opScopeUnit(run)
} }
var out *cappedBuffer
launch := func(unit string) error {
cmd := opSandboxCmd(ctx, self, specPath, unit)
cmd.Stdin = strings.NewReader(params) cmd.Stdin = strings.NewReader(params)
out := &cappedBuffer{max: opOutputMaxBytes} out = &cappedBuffer{max: opOutputMaxBytes}
cmd.Stdout, cmd.Stderr = out, out cmd.Stdout, cmd.Stderr = out, out
cmd.Env = []string{"PATH=/usr/sbin:/usr/bin:/sbin:/bin"} // systemd-run hands this environment on to the scope's process.
cmd.Env = []string{"PATH=/usr/sbin:/usr/bin:/sbin:/bin", opSandboxHelperEnv}
cmd.WaitDelay = 2 * time.Second // the pipes close when the namespace is gone; never wait past that cmd.WaitDelay = 2 * time.Second // the pipes close when the namespace is gone; never wait past that
return opRunSandboxCmd(cmd)
}
start := time.Now() start := time.Now()
runErr := cmd.Run() runErr := launch(unit)
if unit != "" {
if ctx.Err() == context.DeadlineExceeded {
opKillScope(unit)
}
switch got, ran := opScopeTasks(run); {
case ran && got == strconv.Itoa(opSandboxScopeTasks):
rep.TasksCapped = true
case ran:
rep.TasksNote = opTasksNotCapped(fmt.Sprintf("its scope's pids.max read %q", truncate(got, 200)))
case ctx.Err() != nil:
rep.TasksNote = opTasksNotCapped("its scope did not start inside the time limit")
default:
// systemd-run could not make the scope, or the scope's process
// died before it recorded the cap: either way nothing ran, and the
// op runs as it does where there is no systemd.
rep.TasksNote = opTasksNotCapped("its scope never started the op: " + truncate(opLastLine(out.String()), 200))
runErr = launch("")
}
}
rep.Seconds = time.Since(start).Seconds() rep.Seconds = time.Since(start).Seconds()
rep.Output = out.String() rep.Output = out.String()
if ctx.Err() == context.DeadlineExceeded { if ctx.Err() == context.DeadlineExceeded {
@@ -506,9 +542,16 @@ func opMatchDir(dir, like string) error {
return nil return nil
} }
// opUnprivilegedIdentityHook, when set, is the identity a user-privilege op
// runs as. Only tests set it.
var opUnprivilegedIdentityHook func() (uint32, uint32, error)
// opUnprivilegedIdentity is the session lane's user (session_unix.go): the // opUnprivilegedIdentity is the session lane's user (session_unix.go): the
// dedicated account if it exists, else nobody. // dedicated account if it exists, else nobody.
func opUnprivilegedIdentity() (uint32, uint32, error) { func opUnprivilegedIdentity() (uint32, uint32, error) {
if opUnprivilegedIdentityHook != nil {
return opUnprivilegedIdentityHook()
}
var u *user.User var u *user.User
var err error var err error
for _, name := range sessionUnprivUsers { for _, name := range sessionUnprivUsers {
@@ -869,8 +912,10 @@ func opSandboxChildMain(arg string) int {
return 125 return 125
} }
// Rlimits are inherited by the grandchild: no fork bombs, no giant files, // Rlimits are inherited by the grandchild: no fork bombs, no giant files,
// a CPU cap beside the wall-clock one the parent enforces. // a CPU cap beside the wall-clock one the parent enforces. The kernel
_ = unix.Setrlimit(unix.RLIMIT_NPROC, &unix.Rlimit{Cur: 64, Max: 64}) // does not hold root to RLIMIT_NPROC; a root op's tasks are capped by
// the scope it runs in (ops_taskcap_linux.go).
_ = unix.Setrlimit(unix.RLIMIT_NPROC, &unix.Rlimit{Cur: opSandboxTasksMax, Max: opSandboxTasksMax})
_ = unix.Setrlimit(unix.RLIMIT_FSIZE, &unix.Rlimit{Cur: opUpperMaxBytes, Max: opUpperMaxBytes}) _ = unix.Setrlimit(unix.RLIMIT_FSIZE, &unix.Rlimit{Cur: opUpperMaxBytes, Max: opUpperMaxBytes})
_ = unix.Setrlimit(unix.RLIMIT_NOFILE, &unix.Rlimit{Cur: 256, Max: 256}) _ = unix.Setrlimit(unix.RLIMIT_NOFILE, &unix.Rlimit{Cur: 256, Max: 256})
_ = unix.Setrlimit(unix.RLIMIT_CPU, &unix.Rlimit{Cur: uint64(spec.TimeLimit), Max: uint64(spec.TimeLimit)}) _ = unix.Setrlimit(unix.RLIMIT_CPU, &unix.Rlimit{Cur: uint64(spec.TimeLimit), Max: uint64(spec.TimeLimit)})
@@ -884,6 +929,7 @@ func opSandboxChildMain(arg string) int {
fmt.Fprintln(os.Stderr, "sandbox: no_new_privs:", err) fmt.Fprintln(os.Stderr, "sandbox: no_new_privs:", err)
return 125 return 125
} }
newCmd := func(ownUserNS bool) *exec.Cmd {
cmd := exec.Command(spec.Interpreter, opSandboxMount+"/run", "--mode", spec.Mode) cmd := exec.Command(spec.Interpreter, opSandboxMount+"/run", "--mode", spec.Mode)
cmd.Dir = "/" cmd.Dir = "/"
cmd.Env = []string{ cmd.Env = []string{
@@ -894,13 +940,35 @@ func opSandboxChildMain(arg string) int {
"PRIVATAE_OP_MODE=" + spec.Mode, "PRIVATAE_OP_MODE=" + spec.Mode,
} }
cmd.Stdin, cmd.Stdout, cmd.Stderr = os.Stdin, os.Stdout, os.Stderr cmd.Stdin, cmd.Stdout, cmd.Stderr = os.Stdin, os.Stdout, os.Stderr
if spec.AsUser { if !spec.AsUser {
cmd.Env = append(cmd.Env, "USER=privatae-op")
cmd.SysProcAttr = &syscall.SysProcAttr{Credential: &syscall.Credential{Uid: spec.UID, Gid: spec.GID, NoSetGroups: true}}
} else {
cmd.Env = append(cmd.Env, "USER=root") cmd.Env = append(cmd.Env, "USER=root")
return cmd
} }
if err := cmd.Run(); err != nil { cmd.Env = append(cmd.Env, "USER=privatae-op")
attr := &syscall.SysProcAttr{Credential: &syscall.Credential{Uid: spec.UID, Gid: spec.GID, NoSetGroups: true}}
if ownUserNS {
// RLIMIT_NPROC counts every task of a user in its user namespace,
// so in the machine's own one the cap above would count whatever
// else runs as this user (nobody is shared with system daemons)
// and refuse the op on a busy machine. In a user namespace of its
// own the count is the op's tasks alone. The map is the identity:
// the op is the same user and sees every file's real owner.
attr.Cloneflags = syscall.CLONE_NEWUSER
attr.UidMappings = []syscall.SysProcIDMap{{ContainerID: 0, HostID: 0, Size: 1<<31 - 1}}
attr.GidMappings = []syscall.SysProcIDMap{{ContainerID: 0, HostID: 0, Size: 1<<31 - 1}}
}
cmd.SysProcAttr = attr
return cmd
}
cmd := newCmd(true)
err = cmd.Run()
if spec.AsUser && err != nil && cmd.Process == nil {
// No user namespace here (not built in, or turned off): run the op
// as before, with the cap counting the user's other tasks too.
cmd = newCmd(false)
err = cmd.Run()
}
if err != nil {
var ee *exec.ExitError var ee *exec.ExitError
if errors.As(err, &ee) { if errors.As(err, &ee) {
if ws, ok := ee.Sys().(syscall.WaitStatus); ok && ws.Signaled() { if ws, ok := ee.Sys().(syscall.WaitStatus); ok && ws.Signaled() {

Some files were not shown because too many files have changed in this diff Show More