diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..de5fa3d --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +# Test tooling for one test lab, built only with -tags lab; it is not part of +# any release build, so it is left out of the published source. +client-go/lab.go export-ignore diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index b2ed814..19f83a8 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -16,7 +16,7 @@ on: branches: [main] env: - GO_IMAGE: golang@sha256:1ae0735f00daffa3aaf1363a5184c0d2dc55c78e3db4ec70241cdac97bf84b59 + GO_IMAGE: golang@sha256:27f2d3b83cfc5803c8078f8e730af0f7c431bf067afaec8f60a36fb763187cba # The pull request's own head, or the pushed commit. WANT: ${{ gitea.event.pull_request.head.sha || gitea.sha }} @@ -58,7 +58,10 @@ jobs: echo "== gofmt"; out="$(gofmt -l .)"; [ -z "$out" ] || { echo "$out"; exit 1; } for os in linux darwin windows; do echo "== go vet ($os)"; GOOS=$os go vet ./...; done echo "== go build"; go build -o /tmp/privatae-bastion . - echo "== go test"; go test -count=1 ./...' + if [ -f lab.go ]; then echo "== go vet -tags lab"; go vet -tags lab ./...; fi + echo "== go test"; go test -count=1 ./... + echo "== staticcheck"; sh ../deploy/staticcheck.sh + echo "== the GitHub release step, against a fake API"; python3 -I ../deploy/test_github_release.py' - name: the same for deploy/release-sign run: | docker run --rm --cpus 6 -e GOMAXPROCS=6 -v "$PWD":/src -w /src/deploy/release-sign \ @@ -80,9 +83,14 @@ jobs: test "$(git rev-parse HEAD)" = "$WANT" - name: no person's data in the tree # Reads files, runs nothing from the tree. The term check needs the - # maintainers' key, which a CI job never holds, so here it is skipped and - # the run says so; the entity-id, provenance, email and phone checks run. - run: python3 scripts/test_no_personal_data.py + # maintainers' key and the entity-id check the reserved ids, which a CI + # job never holds, so the job names files that do not exist: both + # classes are skipped and the run prints a NOTE for each; the + # provenance, email and phone checks run. Left at their defaults, a + # path that exists but cannot be read fails the run instead. + run: | + none="$PWD/.ci-holds-none/none" + NO_PERSONAL_DATA_KEY_FILE="$none" RESERVED_ENTITY_IDS_FILE="$none" python3 scripts/test_no_personal_data.py - name: every test the claims table cites exists # docs/claims.md names, for each security claim, the test that fails # if the claim stops being true. A renamed or deleted test would leave diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..a8022da --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,241 @@ +# The checks of .gitea/workflows/ci.yml, for the copy of this repository on +# GitHub. Gitea reads .gitea/workflows when it exists and then ignores this +# directory, so each forge runs one of the two files, never both. +# +# The jobs, image and commands are the same as there. Every job runs in the Go +# toolchain image deploy/build.sh builds releases with, pinned by the same +# digest, so CI and the release use the same compiler. GitHub does not let a +# job's container name come from env, so the digest is written out in each +# job; it must match GO_IMAGE in deploy/build.sh and .gitea/workflows/ci.yml. +# No action from the marketplace is used: the commit is fetched with git from +# this repository's public URL, so nothing outside this file and the pinned +# image runs. The job token can only read. +name: ci + +on: + push: + branches: [main] + pull_request: + branches: [main] + +permissions: + contents: read + +env: + # The pull request's own head, or the pushed commit. + WANT: ${{ github.event.pull_request.head.sha || github.sha }} + REPO_URL: ${{ github.server_url }}/${{ github.repository }}.git + GOTOOLCHAIN: local + CGO_ENABLED: "0" + +defaults: + run: + shell: sh -eu {0} + +jobs: + test: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + container: + image: golang@sha256:27f2d3b83cfc5803c8078f8e730af0f7c431bf067afaec8f60a36fb763187cba + env: + PRIVATAE_REQUIRE_AGE: "1" + steps: + - name: tools + # pdftotext, python3, setpriv and git are what the extract and + # peer-identity tests call; without them those tests fail rather than + # skip. Alpine's git has no http-backend (it is in git-daemon), which + # the forge-token test serves its repository with. + run: apk add -q --no-cache poppler-utils python3 setpriv curl git git-daemon + - name: fetch the commit + # The checkout belongs to the runner's user and the job runs as root, + # so git is told the directory is safe; go build reads it to stamp the + # binary with the commit. + run: | + set -o pipefail + git config --global --add safe.directory "$PWD" + git init -q . + git fetch -q --depth=1 "$REPO_URL" "$WANT" + git checkout -q --detach FETCH_HEAD + test "$(git rev-parse HEAD)" = "$WANT" + - name: age + # The backup tests check this code against the age command itself: a + # pinned age release, its archive checked against the SHA-256 below, + # and PRIVATAE_REQUIRE_AGE makes those tests fail rather than skip if + # it is missing. + run: | + case "$(uname -m)" in + x86_64) a=amd64; sum=cbe24006683f8eb669266162894b9a522a1af52f2665fbc63a4bb032ed26ac10 ;; + aarch64) a=arm64; sum=6b8dc4333c53a5a57c9e5834e3a48f92605d7154014cd07269ff3327db5d37f4 ;; + *) echo "no pinned age release for $(uname -m)"; exit 1 ;; + esac + curl -fsSL -o /tmp/age.tgz "https://github.com/FiloSottile/age/releases/download/v1.3.2/age-v1.3.2-linux-$a.tar.gz" + echo "$sum /tmp/age.tgz" | sha256sum -c - + tar -xzf /tmp/age.tgz -C /tmp age/age + /tmp/age/age --version + echo "AGE_BIN=/tmp/age/age" >> "$GITHUB_ENV" + - name: gofmt, go vet on linux/darwin/windows, build, go test + working-directory: client-go + run: | + echo "== gofmt"; out="$(gofmt -l .)"; [ -z "$out" ] || { echo "$out"; exit 1; } + for os in linux darwin windows; do echo "== go vet ($os)"; GOOS=$os go vet ./...; done + echo "== go build"; go build -o /tmp/privatae-bastion . + echo "== go test"; go test -count=1 ./... + - name: the same for deploy/release-sign + working-directory: deploy/release-sign + run: | + out="$(gofmt -l .)"; [ -z "$out" ] || { echo "$out"; exit 1; } + for os in linux darwin windows; do GOOS=$os go vet ./...; done + go test -count=1 ./... + - name: staticcheck on linux, darwin and windows + # The pinned staticcheck release, built with this toolchain from the + # module proxy and checked against the checksum database. + run: sh deploy/staticcheck.sh + - name: the release workflow's GitHub step, against a fake API + run: python3 -I deploy/test_github_release.py + + personal-data: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + container: + image: golang@sha256:27f2d3b83cfc5803c8078f8e730af0f7c431bf067afaec8f60a36fb763187cba + steps: + - name: tools + run: apk add -q --no-cache bash git python3 + - name: fetch the commit + run: | + set -o pipefail + git config --global --add safe.directory "$PWD" + git init -q . + git fetch -q --depth=1 "$REPO_URL" "$WANT" + git checkout -q --detach FETCH_HEAD + test "$(git rev-parse HEAD)" = "$WANT" + - name: this file's image is the image releases are built with + # Each job here names the digest itself, so a change to GO_IMAGE in + # deploy/build.sh that missed this file fails here. + run: | + want="$(sed -n 's/^GO_IMAGE="\(golang@sha256:[0-9a-f]\{64\}\)"$/\1/p' deploy/build.sh)" + test -n "$want" + test "$(grep -c "^ image: $want\$" .github/workflows/ci.yml)" -eq 3 + test "$(grep -c '^ image: ' .github/workflows/ci.yml)" -eq 3 + grep -q "^ GO_IMAGE: $want\$" .gitea/workflows/ci.yml + grep -q "^ GO_IMAGE: $want\$" .github/workflows/ci.yml + - name: no person's data in the tree + # Reads files, runs nothing from the tree. The term check needs the + # maintainers' key and the entity-id check the reserved ids, which + # this job never holds, so both are skipped and the run prints a NOTE + # for each; the provenance, email and phone checks run. + run: python3 -I scripts/test_no_personal_data.py + - name: every test the claims table cites exists + # docs/claims.md names, for each security claim, the test that fails + # if the claim stops being true. A renamed or deleted test would leave + # the claim looking covered; this fails the run instead. Reads files + # and greps; runs nothing from the tree. + run: bash deploy/check-claims.sh + - name: no Host source file names in the tree + # The member repository publishes this tree as a release was built, + # and its release step refuses a path into the Host's Python + # (server/... or brain/... .py): this repo has no such files, so the + # path can only name another program's source. git grep exits 1 when + # nothing matches and above 1 when it could not search; only 1 passes. + run: | + rc=0 + git grep -nIE 'server/[a-z_/]+\.py|brain/[a-z_/]+\.py' || rc=$? + case "$rc" in + 1) ;; + 0) echo "these lines name another program's source file; describe it instead" >&2 + exit 1 ;; + *) echo "git grep could not search the tree (exit $rc), so it was not checked" >&2 + exit 1 ;; + esac + + fuzz: + needs: test + runs-on: ubuntu-24.04 + timeout-minutes: 20 + container: + image: golang@sha256:27f2d3b83cfc5803c8078f8e730af0f7c431bf067afaec8f60a36fb763187cba + steps: + - name: tools + run: apk add -q --no-cache bash git + - name: fetch the commit + run: | + set -o pipefail + git config --global --add safe.directory "$PWD" + git init -q . + git fetch -q --depth=1 "$REPO_URL" "$WANT" + git checkout -q --detach FETCH_HEAD + test "$(git rev-parse HEAD)" = "$WANT" + - name: every fuzz target, 5 seconds each + # A short pass on every commit. Run deploy/fuzz.sh with its default of + # 20 seconds before changing a wall a host's input goes through. + run: bash deploy/fuzz.sh 5 + + claims-on-a-machine: + # The claim tests a container cannot run: the op and action-op sandboxes + # (root, systemd units, overlayfs, a cgroup with the pids controller), + # the egress tests in private user and network namespaces, and the root + # command watch (nftables). They run here on the runner's own virtual + # machine, as root, because that is what they test: the bastion as a + # networked machine's system service. The machine is GitHub's and is + # discarded after the job. The toolchain is the one in the pinned image. + # Every test docs/claims.md cites is run, and the job fails if any of + # them skips. + needs: test + runs-on: ubuntu-24.04 + timeout-minutes: 30 + # This job runs on the runner itself, not in the image, and Ubuntu + # 24.04's sh is dash 0.5.12-6ubuntu5, which has no pipefail (Debian added + # it in 0.5.12-7): `set -o pipefail` there is a fatal error. bash has it. + defaults: + run: + shell: bash -eu {0} + env: + GO_IMAGE: golang@sha256:27f2d3b83cfc5803c8078f8e730af0f7c431bf067afaec8f60a36fb763187cba + steps: + - name: fetch the commit + run: | + set -o pipefail + git init -q . + git fetch -q --depth=1 "$REPO_URL" "$WANT" + git checkout -q --detach FETCH_HEAD + test "$(git rev-parse HEAD)" = "$WANT" + - name: tools, and the pinned toolchain on the machine + run: | + sudo apt-get update -qq + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends nftables iproute2 poppler-utils >/dev/null + # The forge-token claim test needs git 2.46 or later; Ubuntu 24.04's + # own git is 2.43. + if ! git version | awk '{split($3, v, "."); exit !(v[1] > 2 || (v[1] == 2 && v[2] >= 46))}'; then + sudo add-apt-repository -y ppa:git-core/ppa >/dev/null + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq git >/dev/null + fi + git version + docker create --name go "$GO_IMAGE" >/dev/null + sudo docker cp go:/usr/local/go /opt/go-pinned + docker rm go >/dev/null + /opt/go-pinned/bin/go version + - name: age + # The same pinned age release as the test job. + run: | + case "$(uname -m)" in + x86_64) a=amd64; sum=cbe24006683f8eb669266162894b9a522a1af52f2665fbc63a4bb032ed26ac10 ;; + aarch64) a=arm64; sum=6b8dc4333c53a5a57c9e5834e3a48f92605d7154014cd07269ff3327db5d37f4 ;; + *) echo "no pinned age release for $(uname -m)"; exit 1 ;; + esac + curl -fsSL -o "$RUNNER_TEMP/age.tgz" "https://github.com/FiloSottile/age/releases/download/v1.3.2/age-v1.3.2-linux-$a.tar.gz" + echo "$sum $RUNNER_TEMP/age.tgz" | sha256sum -c - + tar -xzf "$RUNNER_TEMP/age.tgz" -C "$RUNNER_TEMP" age/age + echo "AGE_BIN=$RUNNER_TEMP/age/age" >> "$GITHUB_ENV" + - name: every cited test, as root + working-directory: client-go + run: | + set -o pipefail + tests="$(grep -o '_test\.go`: `[A-Za-z0-9_]*`' ../docs/claims.md | sed 's/.*`: `//; s/`$//' | sort -u | paste -sd'|' -)" + test -n "$tests" + sudo env PATH="/opt/go-pinned/bin:$PATH" HOME="$HOME" AGE_BIN="$AGE_BIN" PRIVATAE_REQUIRE_AGE=1 GOTOOLCHAIN=local CGO_ENABLED=0 \ + GOFLAGS=-mod=readonly GOMODCACHE="$HOME/go/pkg/mod" GOCACHE="$HOME/.cache/go-build" \ + go test -count=1 -json -run "^($tests)\$" . > "$RUNNER_TEMP/claims.json" || { + grep -h '"Action":"fail"' "$RUNNER_TEMP/claims.json" | head -40; exit 1; } + - name: none of them skipped + run: bash deploy/check-claims.sh --skipped "$RUNNER_TEMP/claims.json" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..60f7ec4 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,67 @@ +# Attaches each release's binaries, its signed manifest and the manifest's +# signature to the release on GitHub, for the copy of this repository there. +# Gitea reads .gitea/workflows when it exists and then ignores this directory. +# +# Nothing here is trusted to say what the release is. deploy/release-assets.sh +# rebuilds the binaries from the release tag with the tag's own pinned build, +# writes the release manifest from them, and fetches the signature published +# for that manifest, which only the release key holder can make and which is +# named by the manifest's digest. Unless the rebuilt bytes are exactly the +# signed release, there is no signature by that name and nothing is attached. +# The release key never comes near this workflow. +# +# A tag push runs it once the release is published. Run it by hand +# (workflow_dispatch, from main) to attach the files to an earlier release +# whose tag is here; its signature is still published, so that works for any +# release from 0.53.2 on. +# +# No action from the marketplace is used: the commit is fetched with git from +# this repository's public URL, and the build runs in the Go toolchain image +# deploy/build.sh pins by digest. The job's token can write the release and +# is passed only to the last step. +name: release + +on: + push: + tags: ['v[0-9]+.[0-9]+.[0-9]+'] + workflow_dispatch: + inputs: + tag: + description: the release tag, vX.Y.Z + required: true + +permissions: + contents: read + +defaults: + run: + shell: bash -euo pipefail {0} + +jobs: + assets: + runs-on: ubuntu-24.04 + timeout-minutes: 40 + permissions: + contents: write + env: + TAG: ${{ github.event_name == 'workflow_dispatch' && inputs.tag || github.ref_name }} + REPO_URL: ${{ github.server_url }}/${{ github.repository }}.git + steps: + - name: fetch this workflow's commit and the release tags + run: | + git init -q . + git fetch -q --tags "$REPO_URL" "$GITHUB_SHA" + git checkout -q --detach FETCH_HEAD + test "$(git rev-parse HEAD)" = "$GITHUB_SHA" + - name: rebuild the release and check it against its signature + run: deploy/release-assets.sh "$TAG" "$RUNNER_TEMP/assets" + - name: attach the files to the release + # The one step with the token. Its shell, git and Python are named by + # path, and Python runs isolated (-I), so nothing an earlier step put + # on PATH or in the environment chooses what runs here. + shell: /usr/bin/bash -euo pipefail {0} + env: + GH_TOKEN: ${{ github.token }} + run: | + /usr/bin/git show "refs/tags/$TAG:CHANGELOG.md" > "$RUNNER_TEMP/CHANGELOG.md" + /usr/bin/python3 -I deploy/github-release.py "$GITHUB_REPOSITORY" "$TAG" "$RUNNER_TEMP/assets" "$RUNNER_TEMP/CHANGELOG.md" diff --git a/CHANGELOG.md b/CHANGELOG.md index b8bb5ba..5432546 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,338 @@ From 0.39.1 on, a heading marks a version *not published* or *held* once that is settled. The newest entries may not be published yet; the manifest says which version is. Older entries are not marked. +## 0.55.3 + +- **A pairing change reaches every Host on the machine at once.** Each pairing + beats its Host every 60 seconds, and the beat is how every Host on the + machine learns that a pairing went active or was revoked. Approving or + revoking a pairing changed it on disk immediately but triggered no beat, so + every Host kept the old state for up to a minute: a connection request to a + Host the anchor had just approved was refused as not on a shared machine. Now + a grant that makes a pairing active, and a revoke, make every running + pairing beat right away. A refused grant, or a re-sync of a pairing that is + already active, beats nothing. + +## 0.55.2 + +- **A search reply says how its text was sanitized.** Every successful reply + from the bastion agent (search, read range, catalog) was stamped + `sanitized: true`, while the agent the bastion deploys runs in `flag` mode, + which replaces e-mail addresses, phone numbers, card, bank and ID numbers, + IP and MAC addresses and keys, and leaves names, street addresses and + dates of birth in. A reply now carries `sanitize_mode` (the mode this + bastion wrote into the agent's manifest, or for an agent run by hand the + mode it reports, else `unknown`), `sanitized` true only for `sterilize`, + and `sanitize_note` saying what the mode leaves in. A reply cannot raise + the mode this bastion wrote. +- **The README says what the bastion needs.** It said "no Docker, no Python". + The bastion itself needs neither, and the README now names the optional + features that do: coder runs, the service slot and the memory mirror need a + container runtime, folder search needs Python 3.8 or later, Python ops need + `/usr/bin/python3`, and the Linux accessibility reader needs python3 with + AT-SPI. It also said Go 1.25; `client-go/go.mod` needs 1.26. +- **GitHub releases carry their files.** A new workflow, + `.github/workflows/release.yml`, runs on each release tag (and by hand for + an earlier one): `deploy/release-assets.sh` rebuilds the release from its + tag, writes the release manifest from the rebuilt binaries with + `deploy/manifest.py` (which `publish.sh` now uses too, so the two cannot + differ), and fetches the signature published for that manifest. Only when + the signature verifies are the six binaries, `version.json`, its signature + and `SHA256SUMS` attached to the release. Releases 0.53.2, 0.54.0 and + 0.54.1 were rebuilt this way and each matched its signed manifest. +- **The lab harness is out of the release build.** `mode=lab` drove one test + lab's own cluster and database, and its layout was in the published source. + It is now built only with `-tags lab` and is not in the published source; a + release bastion answers a lab run with exit 126, as a bastion without a lab + always did. +- **staticcheck is clean** on linux, darwin and windows, for both Go modules, + and CI runs it (`deploy/staticcheck.sh`, a pinned staticcheck release, with + the per-OS runs merged so code used on one OS only is not reported unused). + Unused code is removed and error strings no longer end in punctuation; no + behaviour changes. +- **The claim tests a container cannot run now run on GitHub.** 25 tests the + claims table cites (the op and action-op sandboxes, the egress tests in + private namespaces, the root command watch) skipped in CI, because a CI + container has no systemd, no namespaces of its own and no nftables. A new + job runs every cited test as root on GitHub's discarded virtual machine and + fails if any skips; `deploy/check-claims.sh --skipped` names them. + +## 0.55.1 + +- **A root op's tasks are capped.** An op's tasks were capped at 64 with + `RLIMIT_NPROC`, which the kernel never applies to root, so a root op could + start as many tasks as it liked until its time limit. A root op now runs + in a systemd scope of its own in `privatae_ops.slice` with `TasksMax=96`, + which counts every task in it, root's included: the op's 64 and the + bastion's two processes that start it. Those two run with one CPU's worth + of Go runtime threads (`GOMAXPROCS=1`), so they take the same share of the + cap on a machine of any size: with the runtime set for 64 CPUs they took + 44 of the 96, and set for 256 no root op could start. The scope's process + reads its own cgroup's `pids.max` before the op starts, and the op's + report carries `tasks_capped` only when that is the cap. Where no scope + can be made (no systemd, no pids controller, or `systemd-run` failing), + the op runs as before and its report's `tasks_note` says its tasks were + not capped, and why. A user op's tasks are held by `RLIMIT_NPROC` as + before, and its report says so. A scope is killed with the run at the + time limit, when the bastion stops, and by the startup sweep. Two ops can + run at once, so an action op's memory watch now reads each other op's + unit or scope in the slice as another program: it yields when a root op + beside it makes the machine wait on memory, as it did when root ops ran + in the bastion's own cgroup, and its own unit still never counts. + +## 0.55.0 + +- **A local machine's egress lane never reaches the machine itself.** Before, + on a machine with no anchor the lane vetted no destination, so a host could + dial this machine's own services through it: `POST + http://127.0.0.1:11434/api/pull` reached ollama whatever the `models` switch + said. It now refuses `localhost` names, loopback and unspecified addresses + (`0.0.0.0/8` and `::` included), link-local addresses (the cloud metadata + address included), the rest of `64:ff9b::/32`, the deprecated IPv6 forms + below, and any address this machine takes as its own, as the anchored lane + checks them, also inside an IPv4-mapped, NAT64 or 6to4 address. The LAN + stays reachable: a private, CGNAT or reserved address that is not this + machine's is dialled. Every address a name resolves to is checked, and only + those addresses are dialled, never the name again. +- **A container or VM on this machine is this machine, not the LAN.** On + Linux, an address the kernel routes straight onto a container's veth, or + onto a bridge whose every port is a veth, a tap or a dummy (the bridge + networks Docker, Podman, LXD and libvirt make for their containers and + VMs), is refused as this machine. A route through a gateway on such a link + leaves, and a link an IPv4 or IPv6 default route of the main table goes + out of (any nexthop's, for a multipath default) is never inside: a bastion + that is itself in a container reaches the internet through the machine it + runs on, and dials the network on its own veth, its LAN included, as + before. Before, the local egress lane dialled such an address as LAN, + so a host could reach a service in a container here, `172.17.0.2:11434` + for one, though the same service on loopback was refused. A bridge with a + network card, a bond or a VLAN among its ports is still the LAN. Every lane + reads the same check, so a coder run's proxy, the anchored lane and the + backup lanes now also refuse a public address on such a network; their + private addresses were refused already. macOS and Windows have no route + lookup here, so there such an address is still dialled. +- **Every lane refuses the deprecated IPv6 forms.** IPv4-compatible `::/96`, + IPv4-translated `::ffff:0:0/96` and site-local `fec0::/10` are no + destination of their own, and where each is delivered depends on the + platform. A coder run's proxy and the backup lanes already refused them; the + egress lane now does too, anchored or local, by address and in any answer a + name resolves to. The anchored lane refuses nothing else it did not refuse + before. All the lanes now read one address classifier. +- **Both egress lanes try each vetted address, as Happy Eyeballs does.** A + name with more than one address is dialled at each in order, the next + starting as soon as an attempt fails or when none has answered within + 300 ms, and the first to answer is used; the others are cancelled, and one + that connects anyway is closed. Before, the anchored lane dialled only the + first address, and a coder run's proxy gave each address an even share of + the dial budget, so an address whose packets go nowhere held up the next + one for seconds. The next one now starts within 300 ms, about what the OS + dialler took when it was given the name. +- **`pair` says to enrol a second factor after every pairing that has none.** + It said so only after it anchored a networked machine, so a local pairing, + one over TCP or a unix socket, and a later host on an anchored machine were + told nothing, and on a local machine a folder grant then installs on the + host's signed grant alone. The line names `--host` when more than one host + is paired, and `sudo -H` only when `pair` itself ran with machine authority, + so a local install is told to enrol as its own user. A pairing that keeps + its factor (a `--replace`), or a machine with an approval device pinned, is + not told. +- **A root-command grant lasts at most 30 days.** It was a year. `root-exec + grant --for 31d` is refused and says the longest term; grant again before + it ends. A grant given under the longer term ends 30 days after it was + given, and `root-exec` shows that end. A grant file with no grant time is + refused, since it cannot be held to the term. +- **SECURITY.md, the README's license section and `docs/host-on-this-machine.md` + name each kind of Host.** They described only a Host on Privatae's platform. + They now say that a C-Series Host runs on your own machine in its own Linux + virtual machine, is open source (Privatae/cseries, AGPL-3.0-only), needs no + Privatae account, and that `npx cseries` sets the bastion up networked, + paired, anchored and with a second factor on every pairing (the commands are + shown); that C-Series Cloud and a network membership are optional and paid; + and that a Host on Privatae's own platform is the case where the Host's + software is not open source. +- **Security reports go to security@privatae.ai,** the address + privatae.ai/.well-known/security.txt names. SECURITY.md said info@. +- **`docs/models.md` says what stops a host changing your models on a local + machine.** It told you to scope the pairing on your host, which a local + machine never reads. There, switching `models` off for the host is what + filters the ollama forward, and the egress lane is no way around it. On an + anchored machine the forward filters for every pairing, which the page now + says too. +- **A tunnel endpoint the host sends must be one host and port.** The + pairing response's `endpoint` and `endpoints`, and the endpoints in a + heartbeat's rotate directive, were written into the WireGuard + configuration unchecked. That configuration is one setting per line, so an + endpoint with a newline in it added settings of its own: + `203.0.113.7:51820` followed by a newline and `allowed_ip=::/0` let the + tunnel carry every IPv6 address, not only the host's. An endpoint must now + be an IP address or a host name with a port from 1 to 65535, and nothing + else, no space or control character. A pairing response with any other is + refused and nothing is saved, and a rotate directive with one is not acted + on. A WireGuard key must be 32 bytes. The tunnel's peer is now configured in + one place, which checks every value it writes. +- **A pairing file that holds such an endpoint is listed but never served.** + An older build may have saved one. Its tunnel and lanes never come up, and + `run` names the file and says why. Every other pairing still loads and + serves, `status` shows the pairing as `invalid` with the same reason, and + `unpair` removes it, as does pairing that host again with `pair --replace`. + One such file no longer stops `run`, `status` and `unpair` for the whole + machine. +- **Such a pairing cannot hold the anchor or be approved, and no host is told + of it.** `claim-anchor` marked it active and anchored the machine to it; it + now refuses, says why, and changes nothing. An approval from the anchor + host marked it active and started it with the endpoint its file holds; it + is now refused. The heartbeat and the anchor host's pairing list leave it + out: a host knows only the states active, pending and revoked, and could + show a pairing in any other state as active. +- **A harness run's agent credential is readable only by the harness.** The + credential file was readable by every user on the machine (0644), in a + folder anyone could open and write (0777), for the length of the run. A + bastion running as root now hands the folder (0700) and the file (0600) to + the user the harness image runs its agent as (uid 10005). One that is not + root cannot hand them over, so it leaves them open for the agent, and the + folder sits inside one that only the bastion's own user can enter. +- **A harness run's tree is out of other users' reach.** In a worktree run the + tree mounted at `/work` was opened to every user (0777), its top folder + included, in the shared temporary folder. Any local user could read or + change the Host's tree during the run. It now sits in a folder only the + bastion's own user can enter, as a git run's tree already did, and a run + refuses a tree whose folder other users can enter. Opening the tree for the + agent no longer follows a link in it. Before, on a bastion running as root, + a link in the repository, to `/etc/shadow` for example, made the linked + file world-writable. A file is opened as 0666, or 0777 if it was + executable, so a git run no longer commits every file as executable. +- **A worktree run returns only what is in its tree.** A file swapped for a + link while the tree was being packed, or a folder swapped for a link, was + read through the link, so a bastion running as root could return any file + on the machine to the Host. Each name is now opened inside the folder + already open above it, never through a link, and goes in only if it is the + plain file or folder that was looked at. On Linux a device or pipe swapped + in for a file in that gap is not opened either: opening some devices does + something by itself. Links, pipes, devices and sockets are left out + (before, each came back as an empty entry of its kind), and the run's + result names them in `worktree_skipped`, with `worktree_skipped_count`. +- **A coding agent can no longer run code as the bastion through the + repository's git settings.** In a git run the bastion committed and pushed + in the agent's own folder, `.git` included, so a hook, a file system + monitor, a filter, a signing program or a changed remote the agent set ran + as the bastion's own user, which can be root, or sent the push, with its + token, to another server. The bastion now makes its own fresh clone, in a + folder the agent cannot reach, copies in the agent's plain files and + folders (never a link, never anything of its `.git`), and commits and + pushes only there. The branch gets one commit with the agent's final + files; the agent's own commits are not pushed one by one. A link the + agent made is left out and named in `worktree_skipped`; a link the + repository already holds stays as it is. Every git the bastion runs for a + harness or a mirror runs with no hooks, no file system monitor, no `ext::` + transport, and no system or user git settings of the machine. +- **The agent in a git run no longer holds the push token, and no process + argument shows it.** The agent's clone named the repository with the + run's token in it, so the agent could push straight to the repository + during the run, past the bastion's copy, and every git the bastion ran + for the run had the token in its arguments, which any local user can + read. Every git call now names the repository without the token and gets + it from its environment, as an `Authorization` header sent to that + repository's address only; no clone's config holds it, and a run is + refused if the token is in any file of the agent's clone's `.git`. +- **A user-privilege op is no longer refused because other programs run as + its user.** An op's tasks are capped at 64, and the cap counted every task + the machine ran as the op's user (`privatae-session`, or `nobody`, which + system daemons share), so on a busy machine a user op could not start or + could not fork at all. The op now runs in a user namespace of its own, + as the same user and seeing every file's real owner, where the cap counts + only its own tasks. Where the machine has no user namespaces, the op runs + as before. +- **A harness run's git calls carry the harness identity.** git made one + from this machine's user and host name instead, which cost a host name + lookup per call (seconds where there is no resolver) and wrote this + machine's name into the clone handed to the agent. +- **A run whose agent container cannot be removed uses nothing of its + tree.** A failed removal was ignored, so an agent still running could go + on writing the tree while the bastion read it. The run now fails, says + so, and copies, returns and pushes nothing. +- **A git run whose commit fails pushes nothing and fails.** A failed add or + commit was ignored and the branch pushed anyway, and the run read as + successful. It now ends the run with an error and no push. A run in which + the agent changed no file pushes no branch and reports `changed: false`. +- **Repositories that keep files in Git LFS work in git runs again.** No + user git settings are read any more, so the LFS filter `git lfs install` + sets there was lost. When `git-lfs` is installed, the bastion now supplies + the standard LFS filter itself on its own git commands, sends the large + files before the push, and sends them only to the repository's own LFS + address, whatever a `.lfsconfig` the agent wrote names. A filter of any + other name in a `.gitattributes` still runs nothing. Tested with git-lfs + 3.7.1. +- **Names that open as `.git` on Windows or macOS are left out of a git + run's copy.** Besides `.git` in any letter case, the copy now leaves out + `.git` followed by dots or spaces or a `:`, the short name `GIT~1`, and + `.git` with the characters HFS+ ignores, the names git itself refuses, so + no file the agent made can be written into the clean clone's own `.git`. +- **A run's tree has a size limit, and a file is never read into memory + whole.** A file of any size in the tree, even one that takes no disk + space, was read whole into memory, so an agent could make the bastion run + out of memory. A worktree run now returns at most 9 MiB of tree, which is + what fits in one message to the Host beside the transcript; a larger tree + is refused with `worktree_too_large` and none of it is returned. A git run + copies at most 2 GiB, and refuses a larger tree with `tree_too_large` + before it pushes anything; each file and folder counts 512 bytes and its + name besides its content, so a tree of millions of empty files is refused + too, and the copy keeps nothing per file in memory. Of the entries left + out, the count is kept in full and the first 100 names. A folder is read + 512 names at a time, each counted as it is read, so a folder of millions + of names is refused without being read whole first. A result too large for one message to the Host + is refused with `result_too_large` instead of being lost on the way. +- **The local console shows a pairing that is not served as "invalid".** A + pairing whose file holds an endpoint this build refuses showed as active + on the console when its stored state was empty, as it is in older files. + It now reads as the setup page and `status` read it. +- **Built with Go 1.26.10, golang.org/x/net 0.61.0, x/crypto 0.58.0 and x/sys + 0.49.0.** `govulncheck` finds 13 known vulnerabilities this code reaches in + Go 1.25.14 and x/net 0.57.0, which 0.54.1 was built with: in crypto/x509 + (intermediate certificates since found compromised were still trusted), + crypto/tls, net/http and HTTP/2, net/textproto, os and html/template. With + these versions it finds none. The + build image is still pinned by digest, and CI runs in the same one. The + module now needs Go 1.26, as x/net's fixed release does. A published release + is still rebuilt by `deploy/verify-release.sh` with the toolchain its own + tag pins. +- **`go test ./...` passes for a contributor who is not root.** A root + command now starts in `/` when the process cannot enter root's home, as it + already did when that home is missing; the bastion runs these commands only + as root, so on a machine nothing changes, and the root-command tests now run + as any user. `TestTreeCheckoutMovesOnlyForward`, which needs a tree root + owns, skips as another user, and `TestClaimAnchorWithNoToken` checks the + refusal without machine authority and then skips the rest, where it passed + having checked nothing more. +- **The tests name no folder of our own build machine.** The sandbox tests' + fixture folder is `/srv/privatae-optest` (`/tmp` cannot hold an op's + effects), and the action tests' run note asks for any folder outside + `/tmp`. +- **How this repository is published (docs only).** From this release on, + GitHub receives each change merged since the last release as its own commit, + titled as the change was, instead of one commit per release. The README, + CONTRIBUTING.md and SECURITY.md say so, and say that releases from 0.53.2 on + can be verified from a clone, since the earlier tags are not public. +- **The personal-data check reads what it used to miss.** A listed term + written right after a backslash escape (`\n`, `\t`, `\u00e9`) is found, and + so is any value inside a run of base64 or base64url that decodes to text. + The reserved entity ids are read from `/etc/privatae/reserved-entity-ids` + (one per line, `RESERVED_ENTITY_IDS_FILE` to point elsewhere) instead of + being written in the script; where that file is absent, as in public CI, + the entity-id check is skipped and the run prints a NOTE, as the term check + already does without its key. A file that holds no id, or a line that is + not one, fails the run. +- **The former-name check reads comments and Markdown.** A kept identifier + is set aside only in Go code; in a comment, this changelog or any other + file it is a hit like the bare word, and the two entries that spelled the + device-fingerprint prefix out now name `fpSANPrefix` instead. +- **CI on GitHub.** `.github/workflows/ci.yml` runs the jobs of + `.gitea/workflows/ci.yml` on GitHub Actions in the same Go image, pinned by + the same digest, with no action from the marketplace and a token that can + only read. A step fails if its digest stops matching `deploy/build.sh`. +- Tests use documentation addresses (`2001:db8::/32`, `203.0.113.0/24`) and + neutral paths, and comments state their reason instead of citing a pull + request by number; a test keeps published source free of such numbers. + ## 0.54.1 - **No change to what the bastion does.** The README is now a short front @@ -225,13 +557,13 @@ says which version is. Older entries are not marked. - **The device fingerprint is sent under a second name to a Host with no platform.** A pairing with such a Host (`--redeem-url unix://…` or `https://…`) sends an operator certificate request that carries the - fingerprint as two URI SANs: first `urn:skema:hwfp:` (`fpSANPrefix`), - then `urn:privatae:hwfp:`, which that Host reads. A platform pairing's - request names it once, under `urn:skema:hwfp:`, as before: the platform - signs a request with that one name and refuses one with any other. A - signed accounting verifies with the fingerprint under either name, and a - signer certificate that names two different fingerprints is refused rather - than resolved. + fingerprint as two URI SANs: first under the original device-fingerprint + URN prefix (`fpSANPrefix` in pair.go), then `urn:privatae:hwfp:`, which + that Host reads. A platform pairing's request names it once, under + `fpSANPrefix`, as before: the platform signs a request with that one name + and refuses one with any other. A signed accounting verifies with the + fingerprint under either name, and a signer certificate that names two + different fingerprints is refused rather than resolved. ## 0.52.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6c99861..5cbf576 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -9,10 +9,9 @@ their approval on that machine. Issues and pull requests are welcome on GitHub, at [github.com/Privatae/bastion](https://github.com/Privatae/bastion). -Development happens in a private repository, and the public repository -receives the source of each published release as one snapshot commit, so an -accepted change is carried into the development repository and arrives with -the next release. Sign-in on the git.privatae.ai forge is not reachable from +Development happens in a private repository, so an accepted change is carried +into the development repository and arrives here with the next release, as +its own commit. Sign-in on the git.privatae.ai forge is not reachable from the internet, so issues and pull requests cannot be opened there. You can also read, clone, build and verify a release, and fork under the GPL. Report a security problem by email, not in a public issue (see @@ -26,8 +25,15 @@ gofmt -l . # must print nothing go vet ./... CGO_ENABLED=0 go build . go test ./... +cd .. && deploy/staticcheck.sh # must find nothing, on linux, darwin and windows ``` +Some claim tests need a real machine and root: the op sandboxes, the +namespace tests and the root command watch skip in a container and as a +user. GitHub's CI runs every test `docs/claims.md` cites as root on its own +virtual machine and fails if one skips; to run them yourself, use a machine +you can throw away. + The backup tests also check this code against the age command itself when one is on PATH or named by `AGE_BIN`; CI installs a pinned age release and sets `PRIVATAE_REQUIRE_AGE`, so there they fail rather than skip. diff --git a/README.md b/README.md index 90f5c09..a5765dd 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,22 @@ # privatae-bastion [![License: GPL v3+](https://img.shields.io/badge/license-GPL--3.0--or--later-blue.svg)](LICENSE) -[![Go 1.25+](https://img.shields.io/badge/go-1.25%2B-00ADD8.svg)](client-go/go.mod) +[![Go 1.26+](https://img.shields.io/badge/go-1.26%2B-00ADD8.svg)](client-go/go.mod) [![Platforms](https://img.shields.io/badge/platforms-linux%20%7C%20macOS%20%7C%20windows-lightgrey.svg)](#supported-platforms) [![ci](https://git.privatae.ai/privatae/privatae-bastion/actions/workflows/ci.yml/badge.svg?branch=main)](https://git.privatae.ai/privatae/privatae-bastion/actions) **One static Go binary that gives your AI Host access to a machine without SSH keys: it dials out, checks every signed grant on the machine itself, and keeps its own audit log.** -No Docker, no Python, no database, no separate WireGuard install. You pair the -binary with a short token from your host, and from then on your host can reach +The bastion itself needs no Docker, no Python, no database and no separate +WireGuard install. A few optional features use software the machine must +already have: coder runs, the service slot and the memory mirror need a +container runtime (Docker, Podman or nerdctl); folder search runs the bastion +agent, which needs Python 3.8 or later; host-written ops in Python need +`/usr/bin/python3`; and reading an app's accessibility tree on Linux needs +python3 with its AT-SPI bindings. Without them, those features say what is +missing and the rest works. + +You pair the binary with a short token from your host, and from then on your host can reach this machine only through the tunnel this bastion dials out and the lanes it gates. Your files stay where they are. Nothing on this machine drives the host: there is no MCP endpoint here. @@ -261,14 +269,15 @@ Each point in full: [docs/security-model.md](docs/security-model.md); the link a - **Egress for someone else reaches the public internet, never this machine**: a coder run's proxy and an anchored machine's egress lane refuse internal addresses and this machine's own (a coder run reaches a LAN host - you named). A local machine's egress lane vets no destination, and an address - that reaches this machine only through a 1:1 NAT is not covered. ([full text](docs/security-model.md)) + you named). A local machine's egress lane refuses this machine and keeps + your LAN. An address that reaches this machine only through a 1:1 NAT is + not covered. ([full text](docs/security-model.md)) - **A terminal is checked here and logged.** On a local machine it opens when the host asks; on an anchored one each needs an approval for that session; a root terminal needs a code entered here. ([session lane](docs/lanes.md#session-lane)) - **Root commands run only under a grant given here with the code**, to the - anchor host, until the time you chose; a revoke or expiry stops a running - command within 2 seconds. ([session lane](docs/lanes.md#session-lane)) + anchor host, until the time you chose and at most 30 days; a revoke or + expiry stops a running command within 2 seconds. ([session lane](docs/lanes.md#session-lane)) - **Root only where there is no one else to ask**: claiming ownership or the anchor. ([modes](#local-and-networked-modes)) - **On macOS and Windows, a networked bastion is the system's service**, which @@ -295,15 +304,20 @@ To report a security issue, please do not open a public issue. See Every security claim names the test that fails if it stops being true, and says where there is none. The full table is one file, [docs/claims.md](docs/claims.md); `deploy/check-claims.sh` checks that every -test it cites exists in `client-go`. +test it cites exists in `client-go`. On GitHub, CI also runs every cited test +as root on the runner's own virtual machine, where the sandbox and namespace +tests a container cannot run do run, and fails if any of them skips. ## How this repo is published -Development happens in a private repository. This repository receives the -source of each published release as one snapshot commit, tagged -`v`, with the tree identical to the commit the signed release -manifest names as `built_from`. Every binary can be rebuilt from the tag and -compared with the published one: see [Verify a release](#verify-a-release). +Development happens in a private repository, and its history is not +published. When a release is published, each change merged since the last one +arrives on GitHub as a commit with the same files, titled as the change was +and dated when it was merged; the copy at git.privatae.ai receives the release +as one commit. In both, the release is tagged `v` on a commit whose +tree is identical to the commit the signed release manifest names as +`built_from`. Every binary can be rebuilt from the tag and compared with the +published one: see [Verify a release](#verify-a-release). Issues and pull requests are welcome on GitHub, at [github.com/Privatae/bastion](https://github.com/Privatae/bastion). An @@ -326,7 +340,7 @@ rather than faked. ## Build from source -Requires Go 1.25 or newer. The binary is pure Go and builds with `cgo` off. +Requires Go 1.26 or newer (`client-go/go.mod`). The binary is pure Go and builds with `cgo` off. ```sh git clone https://git.privatae.ai/privatae/privatae-bastion.git @@ -358,8 +372,8 @@ It checks the release manifest's signature against the keys pinned in `client-go/release_sign.go`, checks out the release's source, rebuilds it, and compares every binary's sha256 with the published one. -From this release on, each public snapshot in this repository carries the tag -`v` (for example `v0.43.0`). The manifest's `built_from` names the +Each published release in this repository carries the tag `v` (for +example `v0.54.1`). The manifest's `built_from` names the commit in the repository the release was built in, which is not a commit here, so the script checks out the tag `v` for the version the manifest names, and prints which source it used. The tag is annotated and its message @@ -372,8 +386,26 @@ bytes. If the published version has no tag in your clone, the script says so and stops without building. If it cannot fetch tags, it says that instead. -Releases from 0.43.0 on can be checked this way from a clone of this -repository. +Releases from 0.53.2 on can be checked this way from a clone of this +repository; the tags of earlier releases are not in it. + +Each release on GitHub carries the six binaries, the signed manifest +`version.json`, its signature `version.json..sig` +and `SHA256SUMS`. The release workflow (`.github/workflows/release.yml`) +attaches them only after `deploy/release-assets.sh` has rebuilt the binaries +from the tag and the published signature has verified against the manifest +written from them, so they are the same bytes the installer and the +self-updater fetch. To check a download yourself, compare its sha256 with the +manifest's, and check the manifest's signature against the keys this source +pins (needs Go): + +```sh +cd deploy/release-sign +go run . verify bastion /version.json /version.json.<…>.sig $(go run . pinned ../../client-go/release_sign.go) +``` + +`deploy/release-assets.sh v ` does all of it from a clone, +rebuild included, for any release from 0.53.2 on. ## Contributing @@ -397,8 +429,11 @@ version 3 or (at your option) any later version. See [LICENSE](LICENSE). Copyright Privatae LLC. You may run it, read it, change it and redistribute it under that license. The -Privatae platform the bastion connects to is separate, proprietary software and -is not covered by this license. +Host the bastion connects to is separate software and is not covered by this +license: a C-Series Host is [Privatae/cseries](https://github.com/Privatae/cseries), +under AGPL-3.0-only, and a Host on Privatae's own platform is proprietary +software. See [SECURITY.md](SECURITY.md#what-you-still-have-to-trust) for what +the bastion holds each to. The local console vendors [htmx](https://htmx.org) 2.0.10 (`client-go/uiassets/htmx.min.js`), which is distributed under the Zero-Clause diff --git a/SECURITY.md b/SECURITY.md index 37ee600..efc3657 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -4,7 +4,7 @@ Please do not open a public issue for a security problem. -Email **info@privatae.ai** with: +Email **security@privatae.ai** with: - the version (`privatae-bastion version`) and platform, - what you found and how to reproduce it, @@ -27,9 +27,10 @@ notes if you wish. ## What is out of scope -- The Privatae platform the bastion connects to. It is separate software with - its own reporting path; reports about it sent to the address above are - forwarded. +- The Host the bastion connects to. It is separate software: a C-Series Host + is [Privatae/cseries](https://github.com/Privatae/cseries), and a Host on + Privatae's own platform is software that is not open source. Reports about + either sent to the address above are forwarded. - Findings that require root on the machine already. The bastion's root ops assume the machine's administrator is trusted. @@ -101,7 +102,7 @@ one installed. **Checking our work.** Release builds are reproducible. `deploy/verify-release.sh` fetches the published manifest, checks its signature, checks out the release's source, rebuilds it, and compares every -binary's digest. From 0.43.0 on, each public snapshot in this repository +binary's digest. From 0.53.2 on, each published release in this repository carries the annotated tag `v`, whose message records the release's `built_from`. That field names the commit in the repository the release was built in, which is not a commit here, so the script uses the tag and prints @@ -114,9 +115,30 @@ the folder you granted. The agent is a separate program in its own repository (`privatae/bastion-agent`), and it reads your files in order to index them. What leaves your machine is what its manifest and your grant allow. Its extraction and PII redaction happen inside it, so check that repository, not this one, -for how well those work. +for how well those work. The bastion runs it in `flag` mode, which does not +find names, street addresses or dates of birth, and every search reply says so +([DCN stores](docs/context.md#distributed-context-network-dcn-stores)). -**The platform.** Your host runs on our platform, which is not open source. The -bastion keeps the host to your grant and your approvals whatever the platform -does. What the host does with what you share with it is governed by software -you cannot read here. +**The Host.** The bastion holds your Host to your grant and your approvals. +What the Host does with what you share with it is the Host's own software, and +which software that is depends on where the Host runs. + +- **A C-Series Host** runs on your own machine, in its own Linux virtual + machine. Its source is [Privatae/cseries](https://github.com/Privatae/cseries), + under AGPL-3.0-only; the bastion and the bastion agent are GPL-3.0-or-later. + `npx cseries` installs the bastion beside it as the machine's system service + and, on every pairing, runs `set-mode networked`, then + `pair --redeem-url --host-ca -` + with the token on standard input, then `claim-anchor` and + `stepup-enroll --console-only --host `: the machine is anchored to + that Host and you approve its writes here with your own authenticator. No + Privatae account is needed + ([A Host in a VM on this machine](docs/host-on-this-machine.md#a-host-in-a-vm-on-this-machine)). +- **Optional and paid, from Privatae:** C-Series Cloud, which runs a Host on a + server of its own that Privatae sets up and keeps patched, and a Privatae + network membership, which joins a Host on your own machine to the Privatae + network. +- **A Host on Privatae's own platform** is the case where the Host's software + is not open source. There, the bastion keeps the Host to your grant and your + approvals whatever the platform does, and what the Host does with what you + share with it is governed by software you cannot read here. diff --git a/client-go/anchor.go b/client-go/anchor.go index e137b67..894c2c6 100644 --- a/client-go/anchor.go +++ b/client-go/anchor.go @@ -810,6 +810,11 @@ type pairingEntry struct { // --replace, so a pending host re-presenting a token keeps its original age // (the 10-minute unapproved clock does not restart). nil when the directory // cannot be read — the host keeps what it has, as with peers. +// +// A pairing whose file this build refuses (Config.invalid) is left out: it is +// never served, so to a host it is not on this box. A host knows the states +// active, pending and revoked and may read any other as active, so "invalid" +// is never sent. The machine's own status and setup page still show it. func pairingsReport() []pairingEntry { cfgs, err := loadPairings() if err != nil { @@ -817,7 +822,7 @@ func pairingsReport() []pairingEntry { } out := make([]pairingEntry, 0, len(cfgs)) for _, c := range cfgs { - if c.HostName == "" { + if c.HostName == "" || c.invalid != "" { continue } out = append(out, pairingEntry{HostName: c.HostName, Handle: handleOf(c.HostName), @@ -849,7 +854,7 @@ func unapprovedAttempts(now time.Time) []peerAnnounce { if err != nil || now.Sub(since) < unapprovedAfter { continue } - out = append(out, peerAnnounce{HostName: e.HostName, Handle: e.Handle, State: e.State, Since: e.Since}) + out = append(out, peerAnnounce(e)) } return out } @@ -1113,9 +1118,16 @@ func cmdClaimAnchor(token string, rotate bool, owner, host string) error { case len(cfgs) == 0: return fmt.Errorf("this box has no pairing yet — pair it to your host first (`privatae-bastion pair `, token from Network → Set up a bastion), then `privatae-bastion claim-anchor` with no token") default: - return fmt.Errorf("this box serves %d hosts — say which one should hold the anchor:\n%s\n e.g. `privatae-bastion claim-anchor --host %s`\nThe anchor host approves every pairing made here from then on; the other pairings keep the lanes they already have.", + return fmt.Errorf("this box serves %d hosts — say which one should hold the anchor:\n%s\n e.g. `privatae-bastion claim-anchor --host %s`\nThe anchor host approves every pairing made here from then on; the other pairings keep the lanes they already have", len(cfgs), " "+strings.Join(hostNames(cfgs), "\n "), hostNames(cfgs)[0]) } + // A pairing whose file this build refuses is never served, so the box + // could not reach the host it would be anchored to, and that host would + // approve every pairing made here. Refused with the reason, before + // anything is written. + if cfg.invalid != "" { + return fmt.Errorf("%s cannot hold the anchor: %s", cfg.HostName, cfg.invalid) + } // REVOKED is not "waiting", it is a decision, and it must not be undone by // a command the user ran to do something else. pair.go holds the same line // for a re-pair (pending again, never straight back to active), diff --git a/client-go/anchor_test.go b/client-go/anchor_test.go index bb2f002..74b05ca 100644 --- a/client-go/anchor_test.go +++ b/client-go/anchor_test.go @@ -606,7 +606,7 @@ func TestClaimAnchorWithNoToken(t *testing.T) { if err := cmdClaimAnchor("", false, "", ""); err == nil || !contains(err.Error(), "machine authority") { t.Fatalf("claim-anchor must need machine authority whether or not a token is given: %v", err) } - return + t.Skip("the rest needs root: without machine authority claim-anchor stops at the refusal checked above") } t.Run("local mode refuses — the token is not what gates this", func(t *testing.T) { diff --git a/client-go/auditlog_notwindows.go b/client-go/auditlog_notwindows.go index 4fadee6..bc76a4d 100644 --- a/client-go/auditlog_notwindows.go +++ b/client-go/auditlog_notwindows.go @@ -32,9 +32,3 @@ func systemAuditWrite(line []byte) { auditSyslog = nil } } - -// registerAuditEventSource has nothing to register outside Windows: syslog -// takes any tag. -func registerAuditEventSource() error { return nil } - -func removeAuditEventSource() {} diff --git a/client-go/backupfetch_test.go b/client-go/backupfetch_test.go index 5aa5a52..0032cb9 100644 --- a/client-go/backupfetch_test.go +++ b/client-go/backupfetch_test.go @@ -16,7 +16,6 @@ import ( "path/filepath" "strings" "testing" - "time" "github.com/gorilla/websocket" ) @@ -64,48 +63,6 @@ func drive(t *testing.T, backupRoot string, req backupFetchReq) (string, string, } } -func awaitJob(t *testing.T, backupRoot, id string) backupJob { - t.Helper() - deadline := time.Now().Add(10 * time.Second) - for time.Now().Before(deadline) { - out, _, code := drive(t, backupRoot, backupFetchReq{Op: "status", JobID: id}) - if code != 0 { - t.Fatalf("status exited %d", code) - } - var j backupJob - if err := json.Unmarshal([]byte(out), &j); err != nil { - t.Fatalf("status is not JSON: %v (%s)", err, out) - } - if j.State != "running" { - return j - } - time.Sleep(25 * time.Millisecond) - } - t.Fatal("job never left running") - return backupJob{} -} - -func startAndAwait(t *testing.T, root string, req backupFetchReq) backupJob { - t.Helper() - out, errOut, code := drive(t, root, req) - if code != 0 { - t.Fatalf("start exited %d: %s", code, errOut) - } - var started struct { - ID string `json:"id"` - State string `json:"state"` - } - if err := json.Unmarshal([]byte(out), &started); err != nil { - t.Fatalf("start reply is not JSON: %v (%s)", err, out) - } - if started.State == "done" { // already-present short circuit - var j backupJob - _ = json.Unmarshal([]byte(out), &j) - return j - } - return awaitJob(t, root, started.ID) -} - func artifactServer(t *testing.T, body []byte) (*httptest.Server, string) { t.Helper() srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/client-go/backupreseal.go b/client-go/backupreseal.go index 89c8984..4ba3fc9 100644 --- a/client-go/backupreseal.go +++ b/client-go/backupreseal.go @@ -655,6 +655,9 @@ func cmdBackupDecide(host, prefix string, approve bool) error { } } r, err = resealApprove(dir, who, id, code) + if err != nil { + return err + } } else { r, err = resealRefuse(dir, resealWho{Name: cfg.HostName}, id, "") } diff --git a/client-go/backupreseal_bind_test.go b/client-go/backupreseal_bind_test.go index e5880e4..1f060fa 100644 --- a/client-go/backupreseal_bind_test.go +++ b/client-go/backupreseal_bind_test.go @@ -10,6 +10,7 @@ package main import ( "encoding/json" "fmt" + "os" "path/filepath" "strings" "testing" @@ -149,3 +150,36 @@ func TestHostNumberComesFromTheTunnelAddress(t *testing.T) { } } } + +// With a second factor enrolled, a wrong code is refused by resealApprove, +// and cmdBackupDecide returns that refusal. It once checked the outer err +// while the approval's error went to the one its `who, err :=` declared, so +// a wrong code fell through to r.State on a nil request and panicked. +func TestResealDecideWithAWrongCodeReturnsTheRefusal(t *testing.T) { + dirs := withFixturePairings(t, map[string]int64{fixtureHost: fixtureEntity}) + fx := newBackupFixture(t, "") + fx.installAsBastionKey(t) + own, _ := age.GenerateX25519Identity() + id := resealStartAs(t, fixtureHost, dirs[fixtureHost], resealReq{Op: "start", Sidecar: string(fx.sidecar(t)), + To: own.Recipient().String(), Stamp: fixtureStamp, HostLabel: "B"}) + if err := stepupSave(dirs[fixtureHost], &stepupFile{SecretB32: "GEZDGNBVGY3TQOJQGEZDGNBVGY3TQOJQ", + EnrolledAt: "2026-08-29T00:00:00Z", LastCounter: -1}); err != nil { + t.Fatal(err) + } + r, w, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + _, _ = w.WriteString("not-a-code\n") + _ = w.Close() + realStdin := os.Stdin + os.Stdin = r + t.Cleanup(func() { os.Stdin = realStdin; _ = r.Close() }) + + if err := cmdBackupDecide(fixtureHost, id, true); err == nil { + t.Fatal("a wrong second-factor code approved the request") + } + if st := resealStatusAs(t, fixtureHost, dirs[fixtureHost], id); st["state"] == "done" || st["sidecar"] != "" { + t.Fatalf("a wrong code handed the backup key over: %v", st) + } +} diff --git a/client-go/beatnow.go b/client-go/beatnow.go new file mode 100644 index 0000000..7fadefc --- /dev/null +++ b/client-go/beatnow.go @@ -0,0 +1,80 @@ +package main + +import ( + "context" + "sync" + "time" +) + +// A beat out of turn, for every pairing on this box. +// +// Each running pairing beats its own Host on a 60-second ticker, and every +// beat carries the box's pairings as `peers` with the state of each (peers.go). +// That is how every Host on the machine learns that a pairing went active or was +// revoked. A pairing_grant_sync or pairing_grant_drop changes that state on disk +// at once, but nothing beat: every Host kept the old state for up to a minute. +// So a pairing approved a second after a beat still read as pending for the +// next minute, and a Host asking to connect to it in that time was refused. The +// anchor Host can read its own confirmed grant, but no other Host on the +// machine has one, and a revoke reached the other Hosts no sooner. +// +// So a change to a pairing's state asks every running pairing's beat loop to beat +// now. The signal carries nothing: the beat reads the pairings from disk as it +// always does, so it reports the state the change wrote. One pending signal per +// loop is kept and the rest are dropped, so a burst of changes is one extra beat +// per Host, never one per change. + +var beatNow = struct { + sync.Mutex + subs map[chan struct{}]struct{} +}{subs: map[chan struct{}]struct{}{}} + +// subscribeBeatNow registers one beat loop. The channel holds at most one +// pending signal; stop unregisters it. +func subscribeBeatNow() (<-chan struct{}, func()) { + ch := make(chan struct{}, 1) + beatNow.Lock() + beatNow.subs[ch] = struct{}{} + beatNow.Unlock() + return ch, func() { + beatNow.Lock() + delete(beatNow.subs, ch) + beatNow.Unlock() + } +} + +// kickBeats asks every registered beat loop to beat now. It never blocks: a +// loop that already has a signal pending, or is mid-beat, beats once more. +func kickBeats() { + beatNow.Lock() + defer beatNow.Unlock() + for ch := range beatNow.subs { + select { + case ch <- struct{}{}: + default: + } + } +} + +// runBeatLoop beats every `every`, and also whenever kickBeats asks. beforeTick +// runs before a ticked beat only (the tunnel's path failover, which belongs to +// the clock, not to a pairing change); it may be nil. It returns when ctx ends. +func runBeatLoop(ctx context.Context, every time.Duration, beforeTick, beat func()) { + kick, stop := subscribeBeatNow() + defer stop() + t := time.NewTicker(every) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + if beforeTick != nil { + beforeTick() + } + beat() + case <-kick: + beat() + } + } +} diff --git a/client-go/beatnow_pairing_test.go b/client-go/beatnow_pairing_test.go new file mode 100644 index 0000000..c226fb3 --- /dev/null +++ b/client-go/beatnow_pairing_test.go @@ -0,0 +1,66 @@ +//go:build !windows + +// Tagged !windows like anchor_test.go, whose box helpers (setupBox, driveOp, +// signTerms) it uses; beatnow_test.go runs the signal and the loop everywhere. + +package main + +import ( + "crypto/ed25519" + "os" + "testing" + "time" +) + +// optest v0.16.145: an approval left every Host on the machine reading +// "pending" until the next 60-second beat. A grant that flips a pairing active, +// and a revoke, ask every pairing to beat now; a refused grant, and a re-sync +// of a pairing that is already active, change no state and ask nothing. +func TestPairingStateChangeBeatsEveryHost(t *testing.T) { + priv := setupBox(t) + writePairingState(t, tMember, pairingPending) + hn, _ := os.Hostname() + future := time.Now().Add(time.Hour).UTC().Format(time.RFC3339) + kick, stop := subscribeBeatNow() + defer stop() + + _, otherPriv, _ := ed25519.GenerateKey(nil) + terms, sig := signTerms(t, otherPriv, grantTerms(tMember, []string{"observe"}, future)) + if _, _, code := driveOp(t, tAnchor, "pairing_grant_sync", []string{terms, sig}); code != 126 { + t.Fatalf("a grant under another key must be refused (126): %d", code) + } + if signalled(kick, 100*time.Millisecond) { + t.Fatal("a refused grant asked for a beat") + } + + terms, sig = signTerms(t, priv, grantTerms(tMember, []string{"observe"}, future)) + if _, errOut, code := driveOp(t, tAnchor, "pairing_grant_sync", []string{terms, sig}); code != 0 { + t.Fatalf("a good grant must be accepted: exit %d, %s", code, errOut) + } + if loadState(t, tMember) != pairingActive { + t.Fatal("grant_sync must flip the pairing active") + } + if !signalled(kick, time.Second) { + t.Fatal("a grant that flipped a pairing active did not ask every pairing to beat now") + } + + terms, sig = signTerms(t, priv, grantTerms(tMember, []string{"observe", "dcn"}, future)) + if _, errOut, code := driveOp(t, tAnchor, "pairing_grant_sync", []string{terms, sig}); code != 0 { + t.Fatalf("a re-sync of an active pairing must be accepted: exit %d, %s", code, errOut) + } + if signalled(kick, 100*time.Millisecond) { + t.Fatal("a re-sync that changed no pairing state asked for a beat") + } + + terms, sig = signTerms(t, priv, map[string]any{"anchor": tAnchor, "issued_at": time.Now().UTC().Format(time.RFC3339), + "kind": "pairing_revoke", "server": hn, "subject": tMember}) + if _, errOut, code := driveOp(t, tAnchor, "pairing_grant_drop", []string{tMember, terms, sig}); code != 0 { + t.Fatalf("a good revoke must be accepted: %d %s", code, errOut) + } + if loadState(t, tMember) != pairingRevoked { + t.Fatal("grant_drop must mark the pairing revoked") + } + if !signalled(kick, time.Second) { + t.Fatal("a revoke did not ask every pairing to beat now") + } +} diff --git a/client-go/beatnow_test.go b/client-go/beatnow_test.go new file mode 100644 index 0000000..16e1201 --- /dev/null +++ b/client-go/beatnow_test.go @@ -0,0 +1,79 @@ +package main + +import ( + "context" + "sync/atomic" + "testing" + "time" +) + +// signalled reports whether ch carries a signal within d. +func signalled(ch <-chan struct{}, d time.Duration) bool { + select { + case <-ch: + return true + case <-time.After(d): + return false + } +} + +// One signal reaches every registered loop, a burst is one pending signal per +// loop, and a loop that stopped is not signalled. +func TestKickBeatsReachesEveryLoopOnce(t *testing.T) { + a, stopA := subscribeBeatNow() + defer stopA() + b, stopB := subscribeBeatNow() + gone, stopGone := subscribeBeatNow() + stopGone() + kickBeats() + kickBeats() + kickBeats() + stopB() + for name, ch := range map[string]<-chan struct{}{"a": a, "b": b} { + if !signalled(ch, time.Second) { + t.Fatalf("loop %s was not asked to beat", name) + } + if signalled(ch, 50*time.Millisecond) { + t.Fatalf("loop %s holds more than one pending beat after a burst", name) + } + } + if signalled(gone, 50*time.Millisecond) { + t.Fatal("a loop that unregistered was still signalled") + } +} + +// The beat loop beats on a kick without waiting for its tick, and the +// tick-only step (path failover) does not run for it. +func TestRunBeatLoopBeatsOnKick(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + var beats, ticks atomic.Int32 + done := make(chan struct{}) + go func() { + defer close(done) + runBeatLoop(ctx, time.Hour, func() { ticks.Add(1) }, func() { beats.Add(1) }) + }() + // The loop registers inside its goroutine; keep asking until it hears. + deadline := time.Now().Add(3 * time.Second) + for beats.Load() == 0 && time.Now().Before(deadline) { + kickBeats() + time.Sleep(10 * time.Millisecond) + } + if beats.Load() == 0 { + t.Fatal("a kicked beat loop did not beat before its 1-hour tick") + } + if ticks.Load() != 0 { + t.Fatal("a kicked beat ran the tick-only step") + } + cancel() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("the beat loop did not return when its context ended") + } + beatNow.Lock() + left := len(beatNow.subs) + beatNow.Unlock() + if left != 0 { + t.Fatalf("a returned beat loop stayed registered (%d left)", left) + } +} diff --git a/client-go/ciimage_test.go b/client-go/ciimage_test.go new file mode 100644 index 0000000..418ecf8 --- /dev/null +++ b/client-go/ciimage_test.go @@ -0,0 +1,53 @@ +package main + +import ( + "os" + "path/filepath" + "regexp" + "testing" +) + +// The Go toolchain image is pinned by digest in deploy/build.sh, which builds +// releases, and again in each forge's CI, so CI tests with the compiler the +// release is built with. GitHub's workflow checks that the three agree, but +// only GitHub runs that file, and only once a commit is published there. This +// test runs wherever go test runs, so a digest changed in one place and not the +// others fails before the change is merged. +var ( + releaseImageRe = regexp.MustCompile(`(?m)^GO_IMAGE="(golang@sha256:[0-9a-f]{64})"$`) + giteaImageRe = regexp.MustCompile(`(?m)^ GO_IMAGE: (\S+)$`) + githubImageRe = regexp.MustCompile(`(?m)^ image: (\S+)$`) +) + +func TestCIUsesTheReleaseGoImage(t *testing.T) { + read := func(rel string) string { + data, err := os.ReadFile(filepath.Join("..", rel)) + if err != nil { + t.Fatal(err) + } + return string(data) + } + release := releaseImageRe.FindAllStringSubmatch(read("deploy/build.sh"), -1) + if len(release) != 1 { + t.Fatalf(`deploy/build.sh: want one GO_IMAGE="golang@sha256:<64 hex>" line, found %d`, len(release)) + } + want := release[0][1] + for _, c := range []struct { + rel string + re *regexp.Regexp + n int + }{ + {".gitea/workflows/ci.yml", giteaImageRe, 1}, + {".github/workflows/ci.yml", githubImageRe, 3}, + } { + got := c.re.FindAllStringSubmatch(read(c.rel), -1) + if len(got) != c.n { + t.Errorf("%s: want %d image line(s), found %d", c.rel, c.n, len(got)) + } + for _, g := range got { + if g[1] != want { + t.Errorf("%s names %s; deploy/build.sh builds releases with %s", c.rel, g[1], want) + } + } + } +} diff --git a/client-go/claims_tunnel_test.go b/client-go/claims_tunnel_test.go index 10b216d..cb345ee 100644 --- a/client-go/claims_tunnel_test.go +++ b/client-go/claims_tunnel_test.go @@ -90,6 +90,70 @@ func TestATunnelAddressThatIsNotOneHostIsRefused(t *testing.T) { } } +// Tunnel endpoints as the host may send them. The first is the hostile +// shape: the configuration text is one setting per line, so a newline ends +// the endpoint and starts a setting of the sender's choosing. +var ( + badTunnelEndpoints = []string{ + "203.0.113.7:51820\nallowed_ip=::/0", "[fe80::1%x\nallowed_ip=::/0]:51820", + "203.0.113.7:51820\r", "203.0.113.7:51820\t", " 203.0.113.7:51820", "203.0.113.7:51820 ", + "relay\x00.example.net:51820", "relay a.example.net:51820", "", "203.0.113.7", + "203.0.113.7:0", "203.0.113.7:65536", "203.0.113.7:+80", "relay.example.net:port", + "-relay.example.net:51820", "relay..example.net:51820", "[relay.example.net]:51820", + "[fe80::1%eth0]:51820", "0.0.0.0:51820", "[::]:51820", "2001:db8::7:51820", + } + goodTunnelEndpoints = []string{"203.0.113.7:51820", "[2001:db8::7]:51820", "relay-a.example.net:51820"} +) + +// An endpoint that is not one host:port is refused at every place the peer +// is configured, and nothing is set on the device. +func TestATunnelEndpointThatIsNotHostPortIsRefused(t *testing.T) { + for _, bad := range badTunnelEndpoints { + cfg := &Config{WGPrivateKey: testWGKey(t), EndpointPubKey: testWGKey(t), + Endpoint: bad, HostTunnelIP: "2001:db8::e:3"} + dev := testWGDevice(t, "2001:db8::a:1") + if err := configureTunnelPeer(dev, cfg); err == nil { + t.Errorf("%q: configureTunnelPeer must refuse", bad) + } + if err := swapPeer(cfg, dev, bad, testWGKey(t)); err == nil { + t.Errorf("%q: swapPeer must refuse", bad) + } + if peers := wgPeers(t, dev); len(peers) != 0 { + t.Errorf("%q: a refused endpoint must leave the device with no peer, it holds %v", bad, peers) + } + } +} + +// uapiRecorder keeps the configuration text it is given, in place of a device. +type uapiRecorder struct{ texts []string } + +func (r *uapiRecorder) IpcSet(s string) error { r.texts = append(r.texts, s); return nil } + +// Whatever endpoint the pairing holds, the text the device is given names +// one peer with exactly one allowed address, the host's alias, or the +// pairing is refused before any text is built. +func TestThePeerConfigurationHasExactlyOneAllowedAddress(t *testing.T) { + for _, ep := range append(append([]string{}, goodTunnelEndpoints...), badTunnelEndpoints...) { + rec := &uapiRecorder{} + cfg := &Config{WGPrivateKey: testWGKey(t), EndpointPubKey: testWGKey(t), + Endpoint: ep, HostTunnelIP: "2001:db8::e:3"} + if err := configureTunnelPeer(rec, cfg); err != nil { + continue + } + for _, text := range rec.texts { + seen := map[string]int{} + for _, line := range strings.Split(strings.TrimSuffix(text, "\n"), "\n") { + k, _, _ := strings.Cut(line, "=") + seen[k]++ + } + if seen["allowed_ip"] != 1 || seen["public_key"] != 1 || seen["endpoint"] != 1 || + !strings.Contains(text, "\nallowed_ip=2001:db8::e:3/128\n") { + t.Errorf("%q: the configuration must name one peer with one allowed address, got %q", ep, text) + } + } + } +} + func TestTheTunnelReachesOnlyThePairedHost(t *testing.T) { for _, hostIP := range []string{"2001:db8::e:3", "2001:db8::e:ffff", "2001:db8::1"} { cfg := &Config{ diff --git a/client-go/coder_egress.go b/client-go/coder_egress.go index da6df30..48798d7 100644 --- a/client-go/coder_egress.go +++ b/client-go/coder_egress.go @@ -10,8 +10,8 @@ package main // over the machine's own connection. // // THE VETTING APPLIES ALWAYS, anchored box or not. The egress lane -// (egress.go) only vets on an anchored box, because on a local box the host -// using it is the person's own. A coding agent is not the person: it runs +// (egress.go) keeps a local box's LAN reachable, because on a local box the +// host using it is the person's own. A coding agent is not the person: it runs // whatever a prompt or a repository talks it into, so this proxy reaches the // public internet and nothing else — not this machine, not its LAN, not the // cloud metadata address, nor any address this machine takes as its own (a @@ -23,7 +23,6 @@ package main import ( "bufio" "context" - "errors" "net" "os" "path/filepath" @@ -50,35 +49,12 @@ var ( coderDialOut egressDialer = (&net.Dialer{Timeout: egressDialTimeout}).DialContext ) -// coderInternalIP is internalIP (egress.go) plus the IPv6 forms that carry an -// IPv4 address and are not covered there: IPv4-compatible ::a.b.c.d (::/96), -// IPv4-translated ::ffff:0:a.b.c.d, and the deprecated site-local fec0::/10. -// IPv4-mapped ::ffff:a.b.c.d is covered by internalIP through To4. +// coderInternalIP is internalIP (egress.go): a coder run's proxy and the +// backup lanes refuse the same address classes as the egress lane, from the +// one classifier (classifyIP), the IPv4-compatible, IPv4-translated and +// site-local fec0::/10 forms included. func coderInternalIP(ip net.IP) bool { - if internalIP(ip) { - return true - } - if ip.To4() != nil || len(ip) != net.IPv6len { - return false - } - zero := true - for _, b := range ip[:12] { - if b != 0 { - zero = false - break - } - } - if zero { - return true // ::/96 — never a public host - } - if ip[0] == 0 && ip[1] == 0 && ip[2] == 0 && ip[3] == 0 && ip[4] == 0 && ip[5] == 0 && - ip[6] == 0 && ip[7] == 0 && ip[8] == 0xff && ip[9] == 0xff && ip[10] == 0 && ip[11] == 0 { - return true // ::ffff:0:0/96 - } - if ip[0] == 0xfe && ip[1]&0xc0 == 0xc0 { - return true // fec0::/10 - } - return false + return internalIP(ip) } // coderLANAllowed: the owner's allow_lan names this requested host (an entry @@ -245,23 +221,7 @@ func handleCoderEgressConn(client net.Conn, allowLAN []string, refused *int64) { coderRefuse(client, why) return } - var upstream net.Conn - err = errors.New("no address") - for i, a := range dialAddrs { - // Each address gets an even share of what is left of the dial budget, - // so one that black-holes cannot use it all. - actx, acancel := ctx, context.CancelFunc(func() {}) - if left := len(dialAddrs) - i; left > 1 { - if dl, ok := ctx.Deadline(); ok { - actx, acancel = context.WithTimeout(ctx, time.Until(dl)/time.Duration(left)) - } - } - upstream, err = coderDialOut(actx, "tcp", a) - acancel() - if err == nil { - break - } - } + upstream, err := dialEach(ctx, coderDialOut, dialAddrs) if err != nil { _, _ = client.Write([]byte("HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")) return diff --git a/client-go/computer_tree_test.go b/client-go/computer_tree_test.go index d7e44f4..853eed4 100644 --- a/client-go/computer_tree_test.go +++ b/client-go/computer_tree_test.go @@ -47,7 +47,7 @@ func TestBoundTreeRecountsAndCaps(t *testing.T) { t.Fatalf("full tree: count=%d trunc=%v %+v", count, trunc, out) } // the cap holds on THIS side even if a helper lied about truncation - out, count, trunc = boundTree([]*computerNode{win}, 3) + _, count, trunc = boundTree([]*computerNode{win}, 3) if count != 3 || !trunc { t.Fatalf("capped: count=%d trunc=%v", count, trunc) } diff --git a/client-go/config.go b/client-go/config.go index 4e3dcb7..ebb1655 100644 --- a/client-go/config.go +++ b/client-go/config.go @@ -76,12 +76,20 @@ type Config struct { // that its file does not hold yet (rotate.go, setUnsaved; guarded by // rotation.mu). Unexported: never in the file. savePending bool + // invalid is why loadPairings found this pairing's file unusable (an + // endpoint that is not one host:port), naming the file; "" for a usable + // one. Such a pairing is listed, so status shows it and unpair removes it, + // but it never serves: active() is false. Unexported: never in the file. + invalid string } const ( pairingActive = "active" pairingPending = "pending" pairingRevoked = "revoked" + // pairingInvalid is reported, never stored: the file holds a value this + // build refuses (Config.invalid). + pairingInvalid = "invalid" // transportUnix is Config.Transport for a Host on this machine. transportUnix = "unix" @@ -103,11 +111,15 @@ func (c *Config) localTransport() bool { return c.unixTransport() || c.tcpTransp // active is the ONE place the state field is interpreted: "" and "active" serve, // anything else does not. run.go, the heartbeat and the mgmt lane all ask this. func (c *Config) active() bool { - return c.State == "" || c.State == pairingActive + return c.invalid == "" && (c.State == "" || c.State == pairingActive) } -// stateName is the state as reported: the empty legacy value reads "active". +// stateName is the state as reported: the empty legacy value reads "active", +// and a pairing whose file is unusable reads "invalid" whatever its state. func (c *Config) stateName() string { + if c.invalid != "" { + return pairingInvalid + } if c.State == "" { return pairingActive } @@ -316,17 +328,36 @@ func (c *Config) updateOnDiskIf(apply func(disk *Config) error) error { // updatePairingOnDisk is updateOnDisk for a caller that only has the host // name (the mgmt handlers). Returns the pairing as written, or nil with // errPairingGone when no such file exists. +// +// The copy is checked as loadPairings checks a file, so a caller that starts +// the pairing from it never serves one whose file this build refuses. func updatePairingOnDisk(host string, apply func(disk *Config)) (*Config, error) { + pd, err := pairingsDir() + if err != nil { + return nil, err + } c := &Config{HostName: host} var out *Config - err := c.updateOnDisk(func(d *Config) { + err = c.updateOnDisk(func(d *Config) { apply(d) cp := *d + cp.invalid = invalidReason(&cp, filepath.Join(pd, host+".json")) out = &cp }) return out, err } +// invalidReason is why the pairing c, read from file, is never served, naming +// the file, or "" when it may be served. It is the check the pairing response +// and the rotate directive get, for a file an older build saved from either. +func invalidReason(c *Config, file string) string { + if err := checkEndpoints(append([]string{c.Endpoint}, c.Endpoints...)...); err != nil { + return fmt.Sprintf("pairing %s: %v — it is not served; pair it again (`privatae-bastion pair --replace `) or remove it (`privatae-bastion unpair %s`)", + file, err, c.HostName) + } + return "" +} + // unpairHost tears down and deletes one pairing: its file and its grant file. // The context store and DCN dir are the USER's data and are left in place — // deleting a pairing must not delete what the user put on this machine. @@ -495,31 +526,15 @@ func loadPairings() ([]*Config, error) { if err := json.Unmarshal(raw, &c); err != nil { return nil, fmt.Errorf("pairing %s: %w", e.Name(), err) } + // One such file must not stop every other pairing, status or unpair: + // it loads, never serves. + c.invalid = invalidReason(&c, filepath.Join(pd, e.Name())) cfgs = append(cfgs, &c) } sort.Slice(cfgs, func(i, j int) bool { return cfgs[i].HostName < cfgs[j].HostName }) return cfgs, nil } -// loadConfig keeps single-pairing call sites working: with exactly one pairing it -// returns it. With several it REFUSES — a straggler caller must name a host and -// fail safe, not silently act on an arbitrary one (that ambiguity is the same -// family of bug as the single-config unpair). -func loadConfig() (*Config, error) { - cfgs, err := loadPairings() - if err != nil { - return nil, err - } - switch len(cfgs) { - case 0: - return nil, fmt.Errorf("no pairings") - case 1: - return cfgs[0], nil - default: - return nil, fmt.Errorf("%d hosts are paired — this operation must name one (see --host)", len(cfgs)) - } -} - // hostNames lists the paired host names (for error messages). func hostNames(cfgs []*Config) []string { names := make([]string, 0, len(cfgs)) diff --git a/client-go/contain.go b/client-go/contain.go index 67fb496..d519aeb 100644 --- a/client-go/contain.go +++ b/client-go/contain.go @@ -135,7 +135,7 @@ func maybeContain(pairing, dcnDir string, heartbeatResponse []byte) { // already passed the agent checks above, and an agent whose own // fingerprint is in the accounting (one running as a service, say) // would otherwise be spared in silence — killTree returns 0 for a - // spared root and a zero kill is not recorded (review of #100). + // spared root and a zero kill is not recorded. root := pid killed := killTree(pid, func(p int) bool { if p == root { diff --git a/client-go/containfinding.go b/client-go/containfinding.go index fc87d08..d3b3c22 100644 --- a/client-go/containfinding.go +++ b/client-go/containfinding.go @@ -104,7 +104,7 @@ func recordContainResult(pairing string, o findingOrder, outcome string, stopped // "Carried out once" held only while this process lived: a bastion that // restarted after a kill and before the beat carried the answer (a self-update, // a crash) would receive the order again, run it again, and report -// not_a_finding — or kill a restarted copy (review of #100). So every result +// not_a_finding — or kill a restarted copy. So every result // is written to the config dir as it is recorded and read back on first use. var containResultsLoaded bool @@ -165,7 +165,7 @@ func containResultsSaveLocked() { } if b, err := json.Marshal(disk); err == nil { // Atomic: a crash mid-write must not corrupt the record and make the - // box forget answers it gave (review of #100). + // box forget answers it gave. _ = atomicWrite(p, b) } } @@ -231,7 +231,7 @@ type findingView struct { protected map[int]bool // identity: each finding pid's identity read straight after the walk. The // kill checks it before touching the root, so a pid reused between the walk - // and the kill is never signalled (review of #100). + // and the kill is never signalled. identity map[int]string } @@ -441,9 +441,9 @@ func maybeContainFindings(pairing, dcnDir string, heartbeatResponse []byte) { // process in the tree, not only the ones an order names — an accounting entry // (accountingPIDs), an app the user is running, anything this bastion started, // init, the bastion. killTree does not signal a protected process or walk -// beneath it (review of #100: stopping a replaced sshd would otherwise have -// taken every ssh session with it). It also returns the bastion's ANCESTORS — -// its launch chain — which the finding path refuses by name: if any process in +// beneath it: without that, stopping a replaced sshd would take every ssh +// session with it. It also returns the bastion's ANCESTORS — its launch +// chain — which the finding path refuses by name: if any process in // a tree were an ancestor of the bastion, the tree's root would be one too, so // the named check always fires first and a tree-level check could never be // reached. diff --git a/client-go/containfinding_test.go b/client-go/containfinding_test.go index a510f2d..e267987 100644 --- a/client-go/containfinding_test.go +++ b/client-go/containfinding_test.go @@ -290,7 +290,7 @@ func TestProcessFindingsFromTheRealComparison(t *testing.T) { } } -// THE GUARDS HOLD ACROSS THE WHOLE TREE (review of #100, blocking). A finding +// THE GUARDS HOLD ACROSS THE WHOLE TREE. A finding // whose tree holds an accounting entry, an app, or something the bastion // started: those processes and everything beneath them are spared, the rest // of the tree is stopped. @@ -334,8 +334,8 @@ func TestContainFindingRefusesTheBastionsOwnAncestors(t *testing.T) { } // THE REFUSALS liveProcessFindings MAKES, through the real function with only -// the machine replaced (review of #100: every other test replaced the whole -// function, so none of these was tested). +// the machine replaced. Every other test replaces the whole function, so +// this is the one that tests these refusals. // // MUTATION: drop the root check. RESULT: red (no_authority not returned). // MUTATION: drop !digestOK. RESULT: red (an edited accounting is used). @@ -375,7 +375,7 @@ func TestLiveProcessFindingsRefusesWhatItCannotTrust(t *testing.T) { } } -// RESULTS SURVIVE A RESTART (review of #100): an answer recorded before the +// RESULTS SURVIVE A RESTART: an answer recorded before the // process died is still there — and still stops a re-run — after it starts. // // MUTATION: skip containResultsSaveLocked. RESULT: red. @@ -413,7 +413,7 @@ func TestContainFindingRunsAnOrderOnce(t *testing.T) { } } -// THE REAL FILTER IS WIRED IN (review of #100): with containFilter left at its +// THE REAL FILTER IS WIRED IN: with containFilter left at its // default, a new process the bastion itself started is filtered by the same // deltaFilter the pass uses, and so is never containable; the same process // started by anyone else is. @@ -422,7 +422,7 @@ func TestContainFindingRunsAnOrderOnce(t *testing.T) { func TestContainFilterIsTheDeltaPassFilter(t *testing.T) { // The process's AGE is supplied (an hour), not read from a real process: // a real pid's age depends on where the suite runs — pid 1 in a fresh PID - // namespace is seconds old and reads as short-lived (review of #100). + // namespace is seconds old and reads as short-lived. self := os.Getpid() prevTree, prevAge := processTree, deltaAgeFn t.Cleanup(func() { processTree, deltaAgeFn = prevTree, prevAge }) @@ -446,7 +446,7 @@ func TestContainFilterIsTheDeltaPassFilter(t *testing.T) { } } -// THE ROOT'S IDENTITY IS CARRIED FROM THE WALK (review of #100): a pid that +// THE ROOT'S IDENTITY IS CARRIED FROM THE WALK: a pid that // changed hands between the finding walk and the kill is not signalled at all. // // MUTATION: killTree(pid, protected) instead of killTreeExpect with the view's @@ -471,7 +471,7 @@ func TestContainFindingDoesNotKillARootThatChangedHandsAfterTheWalk(t *testing.T } } -// THE AGENT PATH HAS THE SAME TREE GUARD (review of #100): an accounting entry +// THE AGENT PATH HAS THE SAME TREE GUARD: an accounting entry // or an app inside an agent's tree is spared with its subtree. // // MUTATION: killTree(pid, spareFromContain) in maybeContain. RESULT: red. @@ -497,9 +497,9 @@ func TestAgentContainSparesProtectedProcessesInTheTree(t *testing.T) { } } -// AN AGENT WHOSE OWN FINGERPRINT IS IN THE ACCOUNTING IS STILL STOPPED (review -// of #100, round 3): the tree guard applies BENEATH the agent, never to the -// agent the host named. Before this, such an agent was spared in silence. +// AN AGENT WHOSE OWN FINGERPRINT IS IN THE ACCOUNTING IS STILL STOPPED: the +// tree guard applies BENEATH the agent, never to the agent the host named. +// Before this, such an agent was spared in silence. // // MUTATION: killTree(pid, guard) — the guard on the root too. RESULT: red. func TestAgentContainStopsAnAgentThatIsItselfInTheAccounting(t *testing.T) { diff --git a/client-go/contextsock.go b/client-go/contextsock.go index aed434a..9eaab4c 100644 --- a/client-go/contextsock.go +++ b/client-go/contextsock.go @@ -200,10 +200,10 @@ func contextAskViaSock(root, name string) (sent bool, host, note string, availab return true, rep.Host, "", true } -// contextSockUnsupported is the note used where the platform has no +// errContextSockUnsupported is the note used where the platform has no // implementation yet, so the reason a person sees names the platform rather // than pretending the feature is merely broken. -var contextSockUnsupported = errors.New("peer identity over a local socket is not " + +var errContextSockUnsupported = errors.New("peer identity over a local socket is not " + "implemented for this platform yet") func contextSockFmtUser(uid uint32, name string) string { diff --git a/client-go/contextsock_other.go b/client-go/contextsock_other.go index c100694..ea011c1 100644 --- a/client-go/contextsock_other.go +++ b/client-go/contextsock_other.go @@ -7,4 +7,4 @@ package main // to tell who is on the other end: contextPeerUser then fails, and // contextSockServe answers every connection with that reason and forwards // nothing. An ask that cannot be attributed is refused, never let through. -func contextPeerUID(uintptr) (uint32, error) { return 0, contextSockUnsupported } +func contextPeerUID(uintptr) (uint32, error) { return 0, errContextSockUnsupported } diff --git a/client-go/contextsock_windows.go b/client-go/contextsock_windows.go index 77ad350..d4bdfec 100644 --- a/client-go/contextsock_windows.go +++ b/client-go/contextsock_windows.go @@ -33,11 +33,11 @@ const contextSockDir = `C:\ProgramData\privatae-bastion` func contextSockDialable(string) string { return "" } func contextSockListen(path string) (net.Listener, error) { - return nil, contextSockUnsupported + return nil, errContextSockUnsupported } func contextPeerUser(c net.Conn) (string, error) { - return "", contextSockUnsupported + return "", errContextSockUnsupported } var _ = os.Remove diff --git a/client-go/contextstt.go b/client-go/contextstt.go index a3e81e0..fca744f 100644 --- a/client-go/contextstt.go +++ b/client-go/contextstt.go @@ -859,10 +859,7 @@ func sttIsNonSpeechMark(text string) bool { // ONE group, not two. "(yes) and (no)" ends in a bracket and is speech; the // opening bracket must be closed only at the very end. inner := t[1 : len(t)-1] - if strings.IndexByte(inner, closer) >= 0 { - return false - } - return true + return strings.IndexByte(inner, closer) < 0 } type sttSegment struct { diff --git a/client-go/ctxcap_measure_test.go b/client-go/ctxcap_measure_test.go index eb35771..2154429 100644 --- a/client-go/ctxcap_measure_test.go +++ b/client-go/ctxcap_measure_test.go @@ -26,9 +26,7 @@ import ( "github.com/gorilla/websocket" ) -// uvicornWSMaxSize is the host's WebSocket frame ceiling: the server default, -// which nothing on the host overrides, so this is the live value. -const uvicornWSMaxSize = 16 << 20 +// uvicornWSMaxSize (harness.go) is the host's WebSocket frame ceiling. func TestMeasureCtxPutCapacity(t *testing.T) { if os.Getenv("MEASURE_CTX_PUT") == "" { diff --git a/client-go/device.go b/client-go/device.go index 13585e9..2d2594e 100644 --- a/client-go/device.go +++ b/client-go/device.go @@ -296,13 +296,6 @@ func nonceSave(m map[string]string) error { return atomicWrite(p, raw) } -func nonceSeen(nonce string) bool { - deviceMu.Lock() - defer deviceMu.Unlock() - _, ok := nonceLoad()[nonce] - return ok -} - // burnNonce records the nonce until its approval's expiry (after which a // replay fails on expiry anyway). func burnNonce(a *deviceApproval) { @@ -331,17 +324,6 @@ func approvalOK(dir, approval, purpose string) error { return stepupCheckApproval(dir, approval, purpose) } -// approvalKind names how an approval was given, for the audit line. -func approvalKind(approval string) string { - switch { - case strings.TrimSpace(approval) == "": - return "" - case isDeviceApproval(approval): - return "device" - } - return "console" -} - // approvalRequired: does a change on this box need the person's approval? // Always on an anchored box (writes.go); on a local box only when the person // enrolled a factor here — the rule the DCN lanes have had since stepup.go. diff --git a/client-go/egress.go b/client-go/egress.go index 057be4a..5fd3948 100644 --- a/client-go/egress.go +++ b/client-go/egress.go @@ -3,6 +3,7 @@ package main import ( "bufio" "context" + "errors" "io" "net" "strings" @@ -63,12 +64,12 @@ func handleEgressConn(client net.Conn, dialOut egressDialer, self string) { defer cancel() if strings.EqualFold(method, "CONNECT") { - dialAddr, why := egressVettedAddr(ctx, ensurePort(target, "443")) + dialAddrs, why := egressVettedAddrs(ctx, ensurePort(target, "443")) if why != "" { _, _ = client.Write([]byte("HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\nX-Privatae-Refused: " + why + "\r\n\r\n")) return } - upstream, err := dialOut(ctx, "tcp", dialAddr) + upstream, err := dialEach(ctx, dialOut, dialAddrs) if err != nil { _, _ = client.Write([]byte("HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\n\r\n")) return @@ -94,12 +95,12 @@ func handleEgressConn(client net.Conn, dialOut egressDialer, self string) { if host == "" { return } - dialAddr, why := egressVettedAddr(ctx, ensurePort(host, "80")) + dialAddrs, why := egressVettedAddrs(ctx, ensurePort(host, "80")) if why != "" { _, _ = client.Write([]byte("HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\nX-Privatae-Refused: " + why + "\r\n\r\n")) return } - upstream, err := dialOut(ctx, "tcp", dialAddr) + upstream, err := dialEach(ctx, dialOut, dialAddrs) if err != nil { _, _ = client.Write([]byte("HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\n\r\n")) return @@ -145,47 +146,65 @@ func splice(clientReader io.Reader, client, upstream net.Conn) { _ = client.Close() } -// egressInternalRefusal: on an ANCHORED box the egress lane reaches the public -// internet and nothing else. Loopback, link-local and private addresses — this -// machine's own services, the LAN it sits on, the mesh — are refused before -// any dial, whether named by address or by a name that resolves to one. The -// lane is USE of the machine's connection (writes.go), and that holds only +// egressInternalRefusal is the refusal egressVettedAddrs gives, or "". +// +// The lane is USE of the machine's connection (writes.go), and that holds only // while it cannot be turned inward: `POST http://127.0.0.1:11434/api/pull` -// down this lane would reach ollama around the forward's filter, and anything -// on the LAN around every other gate. So is any address the machine takes as -// its own (ownaddr.go): a public one on its NIC, or a range a `local` route -// delivers here. A local box keeps the lane it had. -// Resolution failures refuse too — a name this box cannot resolve is not one +// down this lane would reach ollama around the forward's filter and around the +// `models` switch, and this machine's other services around theirs. So on +// every box the lane refuses this machine: `localhost` names, loopback and +// unspecified addresses, link-local ones (the cloud metadata address +// included), and any address the machine takes as its own (ownaddr.go), such +// as a public one on its NIC or a range a `local` route delivers here. +// +// What differs is the network around it. On an ANCHORED box the lane reaches +// the public internet and nothing else: private, CGNAT and reserved addresses +// (the LAN, the mesh) are refused too. On a LOCAL box the LAN stays reachable: +// the host there is the one the person paired, and their own network is +// theirs to reach. +// +// Resolution failures refuse too: a name this box cannot resolve is not one // it can vouch for. // -// THE ADDRESS THE CHECK VETTED IS THE ADDRESS DIALLED. egressVettedAddr -// returns "ip:port" on an anchored box, and the dial takes that, never the -// name: a name resolved once for the check and again for the dial is a -// rebinding hole (a public answer for the check, 127.0.0.1 for the dial). +// THE ADDRESS THE CHECK VETTED IS THE ADDRESS DIALLED. egressVettedAddrs +// returns "ip:port" forms, and the dial takes those, never the name: a name +// resolved once for the check and again for the dial is a rebinding hole (a +// public answer for the check, 127.0.0.1 for the dial). func egressInternalRefusal(ctx context.Context, hostport string) string { - _, why := egressVettedAddr(ctx, hostport) + _, why := egressVettedAddrs(ctx, hostport) return why } -// egressResolve is how the anchored lane resolves a name. A seam so a test can -// answer for a name without DNS. +// egressResolve is how the lane resolves a name. A seam so a test can answer +// for a name without DNS. var egressResolve = net.DefaultResolver.LookupIPAddr -// egressVettedAddr: (the address to dial, "") or ("", the refusal). On a -// local box the input is returned as it came (the OS resolves at the dial, -// as before); on an anchored box every address the name resolves to is -// checked and the FIRST one is what the dial gets, as "ip:port". +// egressVettedAddr: (the first address to dial, "") or ("", the refusal). func egressVettedAddr(ctx context.Context, hostport string) (string, string) { - if currentAnchor() == nil { - return hostport, "" + all, why := egressVettedAddrs(ctx, hostport) + if why != "" { + return "", why } + return all[0], "" +} + +// egressVettedAddrs: every address the name resolved to, as "ip:port" in the +// resolver's order, or (nil, the refusal). Any one refused address refuses the +// name. The dial tries them in order (dialEach), so a dual-stack site still +// answers from a machine whose IPv6 goes nowhere, as it did when the OS dialled +// the name. +func egressVettedAddrs(ctx context.Context, hostport string) ([]string, string) { + anchored := currentAnchor() != nil host, port, err := net.SplitHostPort(hostport) if err != nil { host, port = hostport, "" } host = strings.Trim(strings.TrimSpace(host), "[]") if strings.EqualFold(host, "localhost") || strings.HasSuffix(strings.ToLower(host), ".localhost") { - return "", "refused: egress to " + host + " — this lane reaches the public internet only on an anchored machine" + if anchored { + return nil, "refused: egress to " + host + " — this lane reaches the public internet only on an anchored machine" + } + return nil, "refused: egress to " + host + " — this lane never reaches this machine" } var addrs []net.IP if ip := net.ParseIP(host); ip != nil { @@ -193,50 +212,150 @@ func egressVettedAddr(ctx context.Context, hostport string) (string, string) { } else { resolved, err := egressResolve(ctx, host) if err != nil || len(resolved) == 0 { - return "", "refused: egress to " + host + " — could not resolve it on this machine" + return nil, "refused: egress to " + host + " — could not resolve it on this machine" } for _, r := range resolved { addrs = append(addrs, r.IP) } } for _, ip := range addrs { - if internalIP(ip) { - return "", "refused: egress to " + host + " (" + ip.String() + ") — this lane reaches the public " + + if anchored && internalIP(ip) { + return nil, "refused: egress to " + host + " (" + ip.String() + ") — this lane reaches the public " + "internet only on an anchored machine, never this machine, its network or the mesh" } + if !anchored && thisMachineOrLinkIP(ip) { + return nil, "refused: egress to " + host + " (" + ip.String() + ") — this lane never reaches " + + "this machine or a link-local address" + } } - // This machine is not only loopback: a public address on its NIC, or a - // range the kernel delivers locally through a `local` route, reaches its - // 0.0.0.0 services too (ownaddr.go). This lane has no allow list, so the - // refusal has no exception; a local box returned above and is unchanged. + // This machine is not only loopback: a public address on its NIC, a LAN + // address on it, or a range the kernel delivers locally through a `local` + // route, reaches its 0.0.0.0 services too, and a container's or VM's + // address on a network made here reaches a service on it (ownaddr.go). This + // lane has no allow list, so the refusal has no exception on either kind of + // box. own, err := ownDestination(addrs) if err != nil { - return "", "refused: egress to " + host + " — this machine could not tell whether that is itself, so it cannot be vetted" + return nil, "refused: egress to " + host + " — this machine could not tell whether that is itself, so it cannot be vetted" } if own { - return "", "refused: egress to " + host + " — that is this machine; this lane reaches the public " + - "internet only on an anchored machine, never this machine, its network or the mesh" + if anchored { + return nil, "refused: egress to " + host + " — that is this machine; this lane reaches the public " + + "internet only on an anchored machine, never this machine, its network or the mesh" + } + return nil, "refused: egress to " + host + " — that is this machine; this lane never reaches this machine" } - return net.JoinHostPort(addrs[0].String(), port), "" + out := make([]string, 0, len(addrs)) + for _, ip := range addrs { + out = append(out, net.JoinHostPort(ip.String(), port)) + } + return out, "" } -// internalIP: loopback, link-local, private (RFC 1918, ULA), CGNAT, unspecified. -func internalIP(ip net.IP) bool { - if ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || - ip.IsPrivate() || ip.IsUnspecified() || ip.IsMulticast() { - return true +// dialEachStagger is how long an attempt has to answer before the next +// address is tried beside it: the connection attempt delay of Happy Eyeballs +// (RFC 8305 recommends 250 ms), which the OS dialler used when it was given +// the name. +var dialEachStagger = 300 * time.Millisecond + +// dialEach dials vetted addresses, in order, and returns the first that +// answers. The next address starts as soon as an attempt fails, or when the +// attempts in flight have not answered within dialEachStagger, so an address +// whose packets go nowhere costs a fraction of a second, not the dial budget. +// Only these addresses are ever dialled. The attempts still in flight when one +// answers are cancelled, and any that connects anyway is closed. +func dialEach(ctx context.Context, dialOut egressDialer, addrs []string) (net.Conn, error) { + if len(addrs) == 0 { + return nil, errors.New("no address") + } + ctx, cancel := context.WithCancel(ctx) + defer cancel() + type attempt struct { + c net.Conn + err error + } + results := make(chan attempt, len(addrs)) + next, inFlight := 0, 0 + start := func() { + a := addrs[next] + next++ + inFlight++ + go func() { + c, err := dialOut(ctx, "tcp", a) + results <- attempt{c, err} + }() + } + stagger := time.NewTimer(dialEachStagger) + defer stagger.Stop() + start() + var err error + for inFlight > 0 { + var tick <-chan time.Time + if next < len(addrs) { + tick = stagger.C + } + select { + case r := <-results: + inFlight-- + if r.err == nil { + go func(n int) { + for ; n > 0; n-- { + if late := <-results; late.c != nil { + _ = late.c.Close() + } + } + }(inFlight) + return r.c, nil + } + err = r.err + if next < len(addrs) { + start() + stagger.Reset(dialEachStagger) + } + case <-tick: + start() + stagger.Reset(dialEachStagger) + } + } + return nil, err +} + +// ipClass is what an address reaches, read by every lane that dials for +// someone else. The address rules live here once: internalIP is "anything but +// public", and the local egress lane refuses the classes that are this machine +// or its link. +type ipClass int + +const ( + ipPublic ipClass = iota // the public internet + ipThisMachine // loopback, unspecified, 0.0.0.0/8 + ipLink // link-local unicast and multicast (the cloud metadata address) + ipNetwork // private (RFC 1918, ULA), CGNAT, multicast, benchmarking, reserved + ipUnreadable // a translated form whose IPv4 address this box cannot read + ipObsolete // IPv4-compatible ::/96, IPv4-translated ::ffff:0:0/96, site-local fec0::/10 +) + +func classifyIP(ip net.IP) ipClass { + switch { + case ip.IsLoopback(), ip.IsUnspecified(): + return ipThisMachine + case ip.IsLinkLocalUnicast(), ip.IsLinkLocalMulticast(): + return ipLink + case ip.IsPrivate(), ip.IsMulticast(): + return ipNetwork } if v4 := ip.To4(); v4 != nil { - // 100.64.0.0/10 (carrier NAT) is as internal as 10/8 for this purpose. - if v4[0] == 100 && v4[1]&0xc0 == 64 { - return true + // 0.0.0.0/8 is "this host on this network". + if v4[0] == 0 { + return ipThisMachine } - // 0.0.0.0/8 ("this network"), 198.18.0.0/15 (benchmarking) and - // 240.0.0.0/4 (reserved) are never a public host. - if v4[0] == 0 || (v4[0] == 198 && v4[1]&0xfe == 18) || v4[0] >= 240 { - return true + // 100.64.0.0/10 (carrier NAT) is as internal as 10/8 for this purpose; + // 198.18.0.0/15 (benchmarking) and 240.0.0.0/4 (reserved) are never a + // public host. + if (v4[0] == 100 && v4[1]&0xc0 == 64) || (v4[0] == 198 && v4[1]&0xfe == 18) || v4[0] >= 240 { + return ipNetwork } - return false + return ipPublic } // IPv6 forms that carry an IPv4 address inside them reach that address, so // they are judged by it: NAT64 64:ff9b::/96 (the last four bytes — how an @@ -245,15 +364,51 @@ func internalIP(ip net.IP) bool { if len(ip) == net.IPv6len { if ip[0] == 0x00 && ip[1] == 0x64 && ip[2] == 0xff && ip[3] == 0x9b { if ip[4]|ip[5]|ip[6]|ip[7]|ip[8]|ip[9]|ip[10]|ip[11] == 0 { - return internalIP(net.IPv4(ip[12], ip[13], ip[14], ip[15])) + return classifyIP(net.IPv4(ip[12], ip[13], ip[14], ip[15])) } // The rest of 64:ff9b::/32 (local-use translators, RFC 8215) maps - // to IPv4 in ways this box cannot read: refused. - return true + // to IPv4 in ways this box cannot read. + return ipUnreadable } if ip[0] == 0x20 && ip[1] == 0x02 { - return internalIP(net.IPv4(ip[2], ip[3], ip[4], ip[5])) + return classifyIP(net.IPv4(ip[2], ip[3], ip[4], ip[5])) } + // Deprecated forms that are no destination of their own: IPv4-compatible + // ::a.b.c.d (RFC 4291 2.5.5.1), IPv4-translated ::ffff:0:a.b.c.d (SIIT) + // and site-local fec0::/10 (RFC 3879). Where each is delivered depends on + // the platform, so no lane dials them. (:: and ::1 are matched above.) + if isZeros(ip[0:12]) || + (isZeros(ip[0:8]) && ip[8] == 0xff && ip[9] == 0xff && ip[10] == 0 && ip[11] == 0) || + (ip[0] == 0xfe && ip[1]&0xc0 == 0xc0) { + return ipObsolete + } + } + return ipPublic +} + +func isZeros(b []byte) bool { + for _, c := range b { + if c != 0 { + return false + } + } + return true +} + +// internalIP: loopback, link-local, private (RFC 1918, ULA), CGNAT, +// unspecified, multicast and reserved, also inside a NAT64 or 6to4 address, +// and the deprecated IPv4-compatible, IPv4-translated and site-local forms. +func internalIP(ip net.IP) bool { + return classifyIP(ip) != ipPublic +} + +// thisMachineOrLinkIP: what the local egress lane refuses by address class. +// An address it cannot read is refused, since it could be either, and so is a +// deprecated form, which is no destination of its own. +func thisMachineOrLinkIP(ip net.IP) bool { + switch classifyIP(ip) { + case ipThisMachine, ipLink, ipUnreadable, ipObsolete: + return true } return false } diff --git a/client-go/egress_dial_test.go b/client-go/egress_dial_test.go new file mode 100644 index 0000000..93ffa00 --- /dev/null +++ b/client-go/egress_dial_test.go @@ -0,0 +1,122 @@ +package main + +// dialEach, which both egress lanes dial through: an address whose packets go +// nowhere must not hold up the next one for its share of the dial budget, the +// way the OS dialler's Happy Eyeballs never did when it was given the name. + +import ( + "context" + "errors" + "net" + "reflect" + "sync" + "sync/atomic" + "testing" + "time" +) + +type closeRecordingConn struct { + net.Conn + closed atomic.Bool +} + +func (c *closeRecordingConn) Close() error { + c.closed.Store(true) + return c.Conn.Close() +} + +// MUTATION: dial the addresses one after another, each with an even share of +// what is left of the budget. +// RESULT: red — the second address waits out the first one's share (2 s here). +func TestDialEachDoesNotWaitOutAnAddressThatGoesNowhere(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 4*time.Second) + defer cancel() + var mu sync.Mutex + var dialled []string + abandoned := make(chan struct{}) + dial := func(ctx context.Context, _, addr string) (net.Conn, error) { + mu.Lock() + dialled = append(dialled, addr) + mu.Unlock() + if addr == "[2001:db8::30]:443" { + <-ctx.Done() // no answer and no error: the packets go nowhere + close(abandoned) + return nil, ctx.Err() + } + a, b := net.Pipe() + go func() { _ = b.Close() }() + return a, nil + } + began := time.Now() + c, err := dialEach(ctx, dial, []string{"[2001:db8::30]:443", "198.51.100.30:443"}) + took := time.Since(began) + if err != nil { + t.Fatalf("the second address answers: %v", err) + } + _ = c.Close() + if took > 1500*time.Millisecond { + t.Fatalf("the second address waited %v for the first", took) + } + select { + case <-abandoned: + case <-time.After(2 * time.Second): + t.Fatal("the attempt left in flight must be cancelled once another answers") + } + mu.Lock() + defer mu.Unlock() + if want := []string{"[2001:db8::30]:443", "198.51.100.30:443"}; !reflect.DeepEqual(dialled, want) { + t.Fatalf("only the vetted addresses, in order: got %v, want %v", dialled, want) + } +} + +// A connection that answers after another already won is closed, not leaked; +// an address that fails at once hands over to the next without waiting; and +// when every address fails the last error is returned. +func TestDialEachClosesALateWinnerAndReturnsTheLastError(t *testing.T) { + late := make(chan *closeRecordingConn, 1) + dial := func(ctx context.Context, _, addr string) (net.Conn, error) { + switch addr { + case "198.51.100.31:443": + time.Sleep(2 * dialEachStagger) // answers, but after the second address did + a, b := net.Pipe() + go func() { _ = b.Close() }() + rc := &closeRecordingConn{Conn: a} + late <- rc + return rc, nil + case "198.51.100.32:443": + a, b := net.Pipe() + go func() { _ = b.Close() }() + return a, nil + } + return nil, errors.New("refused: " + addr) + } + c, err := dialEach(context.Background(), dial, []string{"198.51.100.31:443", "198.51.100.32:443"}) + if err != nil { + t.Fatal(err) + } + _ = c.Close() + select { + case rc := <-late: + deadline := time.Now().Add(2 * time.Second) + for !rc.closed.Load() && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + } + if !rc.closed.Load() { + t.Fatal("a connection that answered after the winner must be closed") + } + case <-time.After(3 * time.Second): + t.Fatal("the slow attempt never answered") + } + + began := time.Now() + _, err = dialEach(context.Background(), dial, []string{"198.51.100.33:443", "198.51.100.34:443"}) + if err == nil || err.Error() != "refused: 198.51.100.34:443" { + t.Fatalf("every address failed, so the last error: %v", err) + } + if took := time.Since(began); took >= dialEachStagger { + t.Fatalf("a failed address hands over at once, not after the stagger: %v", took) + } + if _, err := dialEach(context.Background(), dial, nil); err == nil { + t.Fatal("no address is an error") + } +} diff --git a/client-go/egress_local_test.go b/client-go/egress_local_test.go new file mode 100644 index 0000000..5306267 --- /dev/null +++ b/client-go/egress_local_test.go @@ -0,0 +1,228 @@ +//go:build !windows + +// Tagged !windows: it borrows setupBox (anchor_test.go) and stubOwnAddrs +// (ownaddr_test.go). + +package main + +// The egress lane on a local (unanchored) box. It refuses this machine and +// its link-local addresses, so a host cannot reach this machine's services +// (ollama above all) around the switches that govern them, and it keeps the +// person's LAN. An anchored box is unchanged: the public internet only. + +import ( + "bufio" + "context" + "errors" + "fmt" + "io" + "net" + "reflect" + "strings" + "sync/atomic" + "testing" + "time" +) + +// answerEgressNames answers the egress lane's name lookups from m. +func answerEgressNames(t *testing.T, m map[string][]string) { + t.Helper() + old := egressResolve + t.Cleanup(func() { egressResolve = old }) + egressResolve = func(_ context.Context, host string) ([]net.IPAddr, error) { + ips, ok := m[strings.ToLower(host)] + if !ok { + return nil, errors.New("no such host") + } + var out []net.IPAddr + for _, s := range ips { + out = append(out, net.IPAddr{IP: net.ParseIP(s)}) + } + return out, nil + } +} + +// A machine with a LAN address of its own and an AnyIP range, and names that +// answer with this machine, with the LAN and with the public internet. +func localEgressFixture(t *testing.T) { + t.Helper() + t.Setenv("XDG_CONFIG_HOME", t.TempDir()) + if currentAnchor() != nil || boxMode() != modeLocal { + t.Fatal("the fixture must be a local, unanchored box") + } + stubOwnAddrs(t, []string{"127.0.0.1/8", "::1/128", "192.168.50.7/24", "fe80::7/64"}, localRange("203.0.113.0/24")) + answerEgressNames(t, map[string][]string{ + "ollama.example.test": {"127.0.0.1"}, + "mixed.example.test": {"198.51.100.8", "127.0.0.1"}, + "self.example.test": {"192.168.50.7"}, + "nas.example.test": {"192.168.50.30"}, + "dual.example.test": {"2001:db8::8", "198.51.100.8"}, + }) +} + +// MUTATION: restore the early return for an unanchored box in egressVettedAddrs. +// RESULT: red — 127.0.0.1:11434 is handed to the dial, and a name is dialled +// as a name instead of the address that was vetted. +func TestLocalEgressRefusesThisMachineAndKeepsTheLAN(t *testing.T) { + localEgressFixture(t) + ctx := context.Background() + for _, hp := range []string{ + "127.0.0.1:11434", "127.9.9.9:80", "localhost:11434", "ollama.localhost:80", "[::1]:11434", + "0.0.0.0:11434", "0.1.2.3:80", "[::]:80", + "[::ffff:127.0.0.1]:11434", "[64:ff9b::7f00:1]:11434", "[2002:7f00:1::]:80", "[64:ff9b:1::5]:80", + "169.254.169.254:80", "[fe80::1]:80", + "192.168.50.7:11434", // this machine's own LAN address + "203.0.113.5:80", // a range a `local` route delivers here + "ollama.example.test:11434", "mixed.example.test:443", "self.example.test:11434", + "unknown.example.test:443", + } { + if addr, why := egressVettedAddr(ctx, hp); why == "" || addr != "" { + t.Errorf("%s must be refused on a local box, got dial %q", hp, addr) + } + } + for hp, want := range map[string][]string{ + "192.168.50.20:8080": {"192.168.50.20:8080"}, + "10.1.2.3:443": {"10.1.2.3:443"}, + "172.16.4.4:22": {"172.16.4.4:22"}, + "[fd12:3456::9]:80": {"[fd12:3456::9]:80"}, + "100.64.0.9:80": {"100.64.0.9:80"}, + "198.51.100.8:443": {"198.51.100.8:443"}, + "nas.example.test:443": {"192.168.50.30:443"}, + "dual.example.test:443": {"[2001:db8::8]:443", "198.51.100.8:443"}, + } { + got, why := egressVettedAddrs(ctx, hp) + if why != "" || !reflect.DeepEqual(got, want) { + t.Errorf("%s is the LAN or the internet, so a local box dials %v: got %v %q", hp, want, got, why) + } + } +} + +// The anchored lane keeps its own wording and still refuses the LAN that a +// local box reaches. +func TestAnchoredEgressStillRefusesTheLAN(t *testing.T) { + setupBox(t) + stubOwnAddrs(t, []string{"127.0.0.1/8", "::1/128", "192.168.50.7/24"}, localRange("203.0.113.0/24")) + ctx := context.Background() + for _, hp := range []string{"192.168.50.20:8080", "10.1.2.3:443", "[fd12:3456::9]:80", "100.64.0.9:80", "127.0.0.1:11434"} { + if addr, why := egressVettedAddr(ctx, hp); addr != "" || !strings.Contains(why, "only on an anchored machine") { + t.Errorf("%s must be refused on an anchored box as before, got %q %q", hp, addr, why) + } + } + if addr, why := egressVettedAddr(ctx, "198.51.100.8:443"); why != "" || addr != "198.51.100.8:443" { + t.Fatalf("the public internet passes: %q %q", addr, why) + } +} + +// Through the lane itself, with a real dialer: a request for this machine is +// answered 403 and never reaches the listener; a LAN request is dialled at its +// vetted address; a dual-stack name falls back to its second address. +func TestLocalEgressLaneNeverDialsThisMachine(t *testing.T) { + localEgressFixture(t) + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer ln.Close() + var hits atomic.Int32 + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + hits.Add(1) + c.Close() + } + }() + ask := func(dial egressDialer, req string) string { + srv, cli := net.Pipe() + go handleEgressConn(srv, dial, "home.host.example") + _ = cli.SetDeadline(time.Now().Add(5 * time.Second)) + _, _ = io.WriteString(cli, req) + line, _ := bufio.NewReader(cli).ReadString('\n') + cli.Close() + return line + } + netDial := (&net.Dialer{Timeout: 2 * time.Second}).DialContext + target := ln.Addr().String() + for _, req := range []string{ + "CONNECT " + target + " HTTP/1.1\r\n\r\n", + "GET http://" + target + "/api/tags HTTP/1.1\r\nHost: " + target + "\r\n\r\n", + "CONNECT localhost:" + strings.Split(target, ":")[1] + " HTTP/1.1\r\n\r\n", + } { + if line := ask(netDial, req); !strings.HasPrefix(line, "HTTP/1.1 403") { + t.Errorf("a local box must refuse %q, answered %q", req, line) + } + } + time.Sleep(50 * time.Millisecond) + if n := hits.Load(); n != 0 { + t.Fatalf("the lane reached this machine's listener %d times", n) + } + + var asked []string + recording := func(_ context.Context, _, addr string) (net.Conn, error) { + asked = append(asked, addr) + return nil, errors.New("stop here") + } + if line := ask(recording, "CONNECT nas.example.test:443 HTTP/1.1\r\n\r\n"); !strings.HasPrefix(line, "HTTP/1.1 502") { + t.Fatalf("a LAN destination is dialled (and this dial fails): %q", line) + } + if !reflect.DeepEqual(asked, []string{"192.168.50.30:443"}) { + t.Fatalf("the vetted LAN address, never the name: %v", asked) + } + + asked = nil + fallback := func(_ context.Context, _, addr string) (net.Conn, error) { + asked = append(asked, addr) + if strings.HasPrefix(addr, "[") { + return nil, errors.New("network is unreachable") + } + a, b := net.Pipe() + go func() { _, _ = io.Copy(b, b) }() + return a, nil + } + if line := ask(fallback, "CONNECT dual.example.test:443 HTTP/1.1\r\n\r\n"); !strings.HasPrefix(line, "HTTP/1.1 200") { + t.Fatalf("a dual-stack name answers from its second address: %q", line) + } + if want := []string{"[2001:db8::8]:443", "198.51.100.8:443"}; !reflect.DeepEqual(asked, want) { + t.Fatalf("each vetted address in turn: got %v, want %v", asked, want) + } +} + +// What docs/models.md tells a person with a local box to do: switch `models` +// off for the host. The forward then filters store-changing requests, and the +// egress lane is no way around it, since it never reaches this machine's +// ollama, on loopback or on the machine's own LAN address. +func TestLocalModelsSwitchLeavesNoEgressRouteToOllama(t *testing.T) { + localEgressFixture(t) + const host = "home.host.example" + writePairingState(t, host, pairingActive) + if inferenceFilterNeeded(host, laneOllama, pairingModelScope(host)) { + t.Fatal("an unscoped local pairing is a raw splice before the switch, so this test would prove nothing") + } + if err := localDenySet(host, "models", true); err != nil { + t.Fatal(err) + } + if !inferenceFilterNeeded(host, laneOllama, pairingModelScope(host)) { + t.Fatal("with `models` off the ollama forward must filter /api/pull") + } + var dialled []string + dial := func(_ context.Context, _, addr string) (net.Conn, error) { + dialled = append(dialled, addr) + return nil, errors.New("stop here") + } + for _, dest := range []string{"127.0.0.1:11434", "localhost:11434", "192.168.50.7:11434", "[::1]:11434"} { + srv, cli := net.Pipe() + go handleEgressConn(srv, dial, host) + _ = cli.SetDeadline(time.Now().Add(5 * time.Second)) + fmt.Fprintf(cli, "POST http://%s/api/pull HTTP/1.1\r\nHost: %s\r\n\r\n", dest, dest) + line, _ := bufio.NewReader(cli).ReadString('\n') + cli.Close() + if !strings.HasPrefix(line, "HTTP/1.1 403") { + t.Errorf("egress to %s must be refused: %q", dest, line) + } + } + if len(dialled) != 0 { + t.Fatalf("nothing for this machine may be dialled: %v", dialled) + } +} diff --git a/client-go/egress_obsolete_test.go b/client-go/egress_obsolete_test.go new file mode 100644 index 0000000..2e4d875 --- /dev/null +++ b/client-go/egress_obsolete_test.go @@ -0,0 +1,80 @@ +//go:build !windows + +// Tagged !windows: it borrows setupBox (anchor_test.go) and stubOwnAddrs +// (ownaddr_test.go). + +package main + +// The deprecated IPv6 forms are no destination of their own, and where each +// is delivered depends on the platform, so every lane refuses them: the +// anchored egress lane, the local one, and a coder run's proxy, which refused +// them first. + +import ( + "context" + "testing" +) + +var obsoleteForms = []string{ + "[::c633:6408]:443", // IPv4-compatible ::198.51.100.8 + "[::7f00:1]:11434", // IPv4-compatible ::127.0.0.1 + "[::c0a8:3214]:80", // IPv4-compatible ::192.168.50.20, a LAN address + "[::ffff:0:c633:6408]:443", // IPv4-translated ::ffff:0:198.51.100.8 + "[::ffff:0:7f00:1]:80", // IPv4-translated ::ffff:0:127.0.0.1 + "[fec0::5]:80", // site-local + "[feff:1::9]:443", // the top of fec0::/10 + "compat.example.test:443", // a name with one such answer among public ones + "sitelocal.example.test:80", +} + +func answerObsoleteNames(t *testing.T) { + answerEgressNames(t, map[string][]string{ + "compat.example.test": {"198.51.100.8", "::c633:6408"}, + "sitelocal.example.test": {"fec0::5"}, + "public.example.test": {"198.51.100.8", "2001:db8::8"}, + }) +} + +// MUTATION: classify the three forms as public in classifyIP. +// RESULT: red — both egress lanes hand them to the dial. +func TestEveryLaneRefusesTheDeprecatedIPv6Forms(t *testing.T) { + ctx := context.Background() + refusedOn := func(t *testing.T, lane string) { + t.Helper() + for _, hp := range obsoleteForms { + if addrs, why := egressVettedAddrs(ctx, hp); why == "" || addrs != nil { + t.Errorf("%s must be refused on the %s egress lane, got dial %v", hp, lane, addrs) + } + } + for _, hp := range []string{"198.51.100.8:443", "public.example.test:443"} { + if _, why := egressVettedAddrs(ctx, hp); why != "" { + t.Errorf("%s is the public internet and passes the %s egress lane: %s", hp, lane, why) + } + } + } + + t.Run("anchored", func(t *testing.T) { + setupBox(t) + stubOwnAddrs(t, loopbackOnly, localRange()) + answerObsoleteNames(t) + refusedOn(t, "anchored") + }) + + t.Run("local", func(t *testing.T) { + localEgressFixture(t) + answerObsoleteNames(t) + refusedOn(t, "local") + }) + + t.Run("a coder run's proxy agrees", func(t *testing.T) { + stubOwnAddrs(t, loopbackOnly, localRange()) + for _, hp := range obsoleteForms[:7] { + if addrs, why := coderVettedAddrs(ctx, hp, nil); why == "" || addrs != nil { + t.Errorf("%s must be refused on a coder run's proxy, got dial %v", hp, addrs) + } + } + if _, why := coderVettedAddrs(ctx, "198.51.100.8:443", nil); why != "" { + t.Errorf("the public internet passes a coder run's proxy: %s", why) + } + }) +} diff --git a/client-go/egress_test.go b/client-go/egress_test.go index cdd076d..6c3d77c 100644 --- a/client-go/egress_test.go +++ b/client-go/egress_test.go @@ -38,6 +38,16 @@ func dialHandler(t *testing.T, dialOut egressDialer) (net.Conn, func()) { return cli, func() { cli.Close(); ln.Close() } } +// redirectDial stands in for the user's internet: the lane asks for a public +// destination, and this dials the loopback backend instead, recording what the +// lane asked for. The lane never reaches loopback itself. +func redirectDial(backend string, asked *[]string) egressDialer { + return func(ctx context.Context, network, addr string) (net.Conn, error) { + *asked = append(*asked, addr) + return (&net.Dialer{}).DialContext(ctx, network, backend) + } +} + // CONNECT establishes a raw tunnel; bytes flow both ways through it. func TestEgressConnConnectTunnel(t *testing.T) { backend, err := net.Listen("tcp", "127.0.0.1:0") @@ -54,9 +64,10 @@ func TestEgressConnConnectTunnel(t *testing.T) { _, _ = io.Copy(c, c) // echo }() - cli, cleanup := dialHandler(t, (&net.Dialer{}).DialContext) + var asked []string + cli, cleanup := dialHandler(t, redirectDial(backend.Addr().String(), &asked)) defer cleanup() - fmt.Fprintf(cli, "CONNECT %s HTTP/1.1\r\nHost: %s\r\n\r\n", backend.Addr(), backend.Addr()) + fmt.Fprint(cli, "CONNECT 198.51.100.20:443 HTTP/1.1\r\nHost: 198.51.100.20:443\r\n\r\n") br := bufio.NewReader(cli) status, err := br.ReadString('\n') if err != nil || !strings.Contains(status, "200") { @@ -82,6 +93,9 @@ func TestEgressConnConnectTunnel(t *testing.T) { if string(buf) != want { t.Fatalf("tunnel echo mismatch: got %q", string(buf)) } + if len(asked) != 1 || asked[0] != "198.51.100.20:443" { + t.Fatalf("the lane must dial the address it vetted: %v", asked) + } } func TestEgressConnPlainHTTP(t *testing.T) { @@ -89,15 +103,18 @@ func TestEgressConnPlainHTTP(t *testing.T) { fmt.Fprint(w, "plain-http-ok") })) defer backend.Close() - host := strings.TrimPrefix(backend.URL, "http://") - cli, cleanup := dialHandler(t, (&net.Dialer{}).DialContext) + var asked []string + cli, cleanup := dialHandler(t, redirectDial(strings.TrimPrefix(backend.URL, "http://"), &asked)) defer cleanup() - fmt.Fprintf(cli, "GET http://%s/ HTTP/1.1\r\nHost: %s\r\nConnection: close\r\n\r\n", host, host) + fmt.Fprint(cli, "GET http://198.51.100.21/ HTTP/1.1\r\nHost: 198.51.100.21\r\nConnection: close\r\n\r\n") body, _ := io.ReadAll(cli) if !strings.Contains(string(body), "plain-http-ok") { t.Fatalf("plain HTTP forward: got %q", string(body)) } + if len(asked) != 1 || asked[0] != "198.51.100.21:80" { + t.Fatalf("the lane must dial the address it vetted: %v", asked) + } } func TestEgressConnDialFailure(t *testing.T) { @@ -106,7 +123,7 @@ func TestEgressConnDialFailure(t *testing.T) { } cli, cleanup := dialHandler(t, failDial) defer cleanup() - fmt.Fprint(cli, "CONNECT unreachable.example:443 HTTP/1.1\r\n\r\n") + fmt.Fprint(cli, "CONNECT 198.51.100.22:443 HTTP/1.1\r\n\r\n") status, _ := bufio.NewReader(cli).ReadString('\n') if !strings.Contains(status, "502") { t.Fatalf("expected 502 on dial failure, got %q", status) diff --git a/client-go/endpoint_invalid_test.go b/client-go/endpoint_invalid_test.go new file mode 100644 index 0000000..cf7612e --- /dev/null +++ b/client-go/endpoint_invalid_test.go @@ -0,0 +1,230 @@ +//go:build !windows + +// Uses fakeToken from pairing_redeem_test.go and setupBox/driveOp from +// anchor_test.go, both unix-only. + +package main + +// A pairing whose file holds an endpoint this build refuses loads as invalid +// and is never served (endpoint_test.go). These are the paths that could +// still act on it: anchoring the box to it, an approval from the anchor host, +// what the hosts are told about it, and the two ways a person clears it. + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// unservedEndpoint is one the endpoint check refuses (badTunnelEndpoints). +const unservedEndpoint = "203.0.113.7:51820\nallowed_ip=::/0" + +// writeUnservedPairing writes a pairing file in the given state whose +// endpoint this build refuses, and returns its path. +func writeUnservedPairing(t *testing.T, host, state string) string { + t.Helper() + pd, err := pairingsDir() + if err != nil { + t.Fatal(err) + } + b, _ := json.Marshal(Config{HostName: host, WGPrivateKey: "k", BastionTunnelIP: "fd12:9::a:1", + State: state, Endpoint: unservedEndpoint}) + p := filepath.Join(pd, host+".json") + if err := os.WriteFile(p, b, 0o600); err != nil { + t.Fatal(err) + } + return p +} + +// fileState is the state field as written in host's pairing file. +func fileState(t *testing.T, host string) string { + t.Helper() + pd, _ := pairingsDir() + raw, err := os.ReadFile(filepath.Join(pd, host+".json")) + if err != nil { + t.Fatal(err) + } + var c Config + if err := json.Unmarshal(raw, &c); err != nil { + t.Fatal(err) + } + return c.State +} + +// claim-anchor names the reason and writes nothing: the pairing keeps its +// state, and the box is neither anchored nor claimed. As the sole pairing and +// named with --host among others, in the legacy active state and pending. +func TestClaimAnchorRefusesAPairingThatIsNotServed(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("claim-anchor needs machine authority, which needs root") + } + for _, c := range []struct { + name string + others []string + host string + }{{"sole pairing", nil, ""}, {"named with --host", []string{tAnchor}, tMember}} { + for _, st := range []string{"", pairingPending} { + t.Run(c.name+"/"+st, func(t *testing.T) { + t.Setenv("XDG_CONFIG_HOME", t.TempDir()) + withMarkerAt(t, filepath.Join(t.TempDir(), "machine.json")) + if err := setBoxMode(modeNetworked); err != nil { + t.Fatal(err) + } + for _, o := range c.others { + writePairing(t, o) + } + p := writeUnservedPairing(t, tMember, st) + err := cmdClaimAnchor("", false, "", c.host) + if err == nil || !contains(err.Error(), "not a host:port") || !contains(err.Error(), p) { + t.Fatalf("claim-anchor must refuse the pairing, naming its file and why: %v", err) + } + if a := currentAnchor(); a != nil { + t.Fatalf("a refused claim anchored the box: %+v", a) + } + if oc := currentOwnerClaim(); oc != nil { + t.Fatalf("a refused claim wrote the owner claim: %+v", oc) + } + if got := fileState(t, tMember); got != st { + t.Fatalf("a refused claim changed the pairing's state from %q to %q", st, got) + } + }) + } + } +} + +// The anchor host's approval of such a pairing is refused: it is not made +// active, no grant is kept, and the refusal names no file of this machine. +// The copy updatePairingOnDisk hands back carries the same mark, so a caller +// that starts a pairing from it starts nothing. +func TestAnApprovalOfAPairingThatIsNotServedIsRefused(t *testing.T) { + priv := setupBox(t) + writeUnservedPairing(t, tMember, pairingPending) + future := time.Now().Add(time.Hour).UTC().Format(time.RFC3339) + terms, sig := signTerms(t, priv, grantTerms(tMember, []string{"observe"}, future)) + _, errOut, code := driveOp(t, tAnchor, "pairing_grant_sync", []string{terms, sig}) + if code != 126 || !contains(errOut, "not served") { + t.Fatalf("the approval must be refused (126, not served): %d %s", code, errOut) + } + pd, _ := pairingsDir() + if contains(errOut, pd) { + t.Fatalf("the refusal sent to the host names a file of this machine: %s", errOut) + } + if got := fileState(t, tMember); got != pairingPending { + t.Fatalf("a refused approval changed the state to %q", got) + } + if _, err := loadPairingGrant(tMember); err == nil { + t.Fatal("a refused approval kept the grant") + } + + c, err := updatePairingOnDisk(tMember, func(d *Config) { d.State = pairingActive }) + if err != nil { + t.Fatal(err) + } + if c.invalid == "" || c.active() { + t.Fatalf("the copy written back must still be invalid and never active: invalid=%q active=%v", c.invalid, c.active()) + } + if startPairing(c) { + t.Fatal("a pairing that is not served was started") + } +} + +// What the hosts hear: neither the peer list every host gets, nor the +// pairing list the anchor host gets (in the heartbeat and from pairing_list), +// names such a pairing, and every state they do carry is one a host knows. +// The machine's own status still shows it invalid (endpoint_test.go). +func TestNoHostIsToldOfAPairingThatIsNotServed(t *testing.T) { + setupBox(t) + const beta, pend = "beta.host.privatae.ai", "pend.host.privatae.ai" + writePairing(t, beta) + writeUnservedPairing(t, tMember, "") + writeUnservedPairing(t, pend, pairingPending) + known := map[string]bool{pairingActive: true, pairingPending: true, pairingRevoked: true} + + peers := peersFor(tAnchor) + if len(peers) != 1 || peers[0].HostName != beta { + t.Fatalf("the peers must be only %s, got %+v", beta, peers) + } + for _, p := range peers { + if !known[p.State] { + t.Fatalf("a peer was sent with a state a host does not know: %+v", p) + } + } + list := pairingsReport() + var names []string + for _, e := range list { + names = append(names, e.HostName) + if !known[e.State] { + t.Fatalf("a pairing was sent with a state a host does not know: %+v", e) + } + } + if got := strings.Join(names, ","); got != tAnchor+","+beta { + t.Fatalf("the anchor host's list must be %s,%s, got %s", tAnchor, beta, got) + } + out, errOut, code := driveOp(t, tAnchor, "pairing_list", nil) + if code != 0 || contains(out, tMember) || contains(out, pend) || contains(out, pairingInvalid) { + t.Fatalf("pairing_list must leave the pairing out: %d %s %s", code, out, errOut) + } + if u := unapprovedAttempts(time.Now().Add(24 * time.Hour)); len(u) != 0 { + t.Fatalf("a pairing that is not served is no unapproved attempt: %+v", u) + } + cfgs, err := loadPairings() + if err != nil { + t.Fatal(err) + } + if c := pairingFor(cfgs, tMember); c == nil || c.stateName() != pairingInvalid { + t.Fatalf("this machine still lists it invalid: %+v", c) + } +} + +// pair --replace over such a pairing pairs it again, served; unpair removes +// one. Both through the commands a person runs. +func TestPairReplaceAndUnpairClearAPairingThatIsNotServed(t *testing.T) { + uiTestConfigDir(t) + withMarkerAt(t, filepath.Join(t.TempDir(), "machine.json")) + prev := unpairNeedsRoot + unpairNeedsRoot = func() bool { return false } + t.Cleanup(func() { unpairNeedsRoot = prev }) + const host, good = "acme.host.privatae.ai", "203.0.113.7:51820" + redeem := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _ = json.NewEncoder(w).Encode(map[string]any{ + "operator_crt": "CERT", "ca_crt": "CA", "bearer": "b", + "endpoint": good, "endpoints": []string{good}, + "endpoint_pubkey": testWGKey(t), "bastion_tunnel_ip": "2001:db8::a:1", + "host_tunnel_ip": "2001:db8::e:3", "host_name": host, + }) + })) + defer redeem.Close() + old := redeemURLBase + redeemURLBase = redeem.URL + t.Cleanup(func() { redeemURLBase = old }) + + writeUnservedPairing(t, host, "") + if err := cmdPair(fakeToken("acme"), "", "", true); err != nil { + t.Fatalf("pair --replace over a pairing that is not served: %v", err) + } + cfgs, err := loadPairings() + if err != nil || len(cfgs) != 1 { + t.Fatalf("after pair --replace one pairing loads: %v %v", cfgs, err) + } + if c := cfgs[0]; c.HostName != host || c.invalid != "" || !c.active() || c.Endpoint != good { + t.Fatalf("pair --replace must leave a served pairing: %q invalid=%q active=%v endpoint=%q", + c.HostName, c.invalid, c.active(), c.Endpoint) + } + + writeUnservedPairing(t, host, "") + if err := cmdUnpair(host); err != nil { + t.Fatalf("unpair of a pairing that is not served: %v", err) + } + if cfgs, err := loadPairings(); err != nil || len(cfgs) != 0 { + t.Fatalf("after unpair nothing is left: %v %v", cfgs, err) + } + pd, _ := pairingsDir() + if _, err := os.Stat(filepath.Join(pd, host+".json")); !os.IsNotExist(err) { + t.Fatalf("the pairing file is still there: %v", err) + } +} diff --git a/client-go/endpoint_test.go b/client-go/endpoint_test.go new file mode 100644 index 0000000..56ecc11 --- /dev/null +++ b/client-go/endpoint_test.go @@ -0,0 +1,207 @@ +//go:build !windows + +// Uses fakeToken from pairing_redeem_test.go, unix-only. + +package main + +// A tunnel endpoint arrives from the host three ways: in the pairing +// response, in the rotate directive on a heartbeat answer, and from the +// pairing file those were saved to. Each one refuses an endpoint that is +// not one host:port, the file by never serving that pairing (the +// device-level checks are in claims_tunnel_test.go). + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" +) + +// A pairing response whose endpoint, or any entry of its endpoints list, is +// not one host:port is refused and nothing is saved. An IPv4, an IPv6 and a +// host name endpoint pair. +func TestAPairingResponseWithAnUnsafeEndpointSavesNothing(t *testing.T) { + cfgHome := uiTestConfigDir(t) + withMarkerAt(t, filepath.Join(t.TempDir(), "machine.json")) + var endpoint, hostName string + var endpoints []string + redeem := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _ = json.NewEncoder(w).Encode(map[string]any{ + "operator_crt": "CERT", "ca_crt": "CA", "bearer": "b", + "endpoint": endpoint, "endpoints": endpoints, + "endpoint_pubkey": testWGKey(t), "bastion_tunnel_ip": "2001:db8::a:1", + "host_tunnel_ip": "2001:db8::e:3", "host_name": hostName, + }) + })) + defer redeem.Close() + old := redeemURLBase + redeemURLBase = redeem.URL + t.Cleanup(func() { redeemURLBase = old }) + + hostName = "acme.host.privatae.ai" + for _, bad := range badTunnelEndpoints { + if bad == "" { + continue // a missing endpoint has its own refusal + } + for _, c := range []struct { + endpoint string + list []string + }{{bad, nil}, {"203.0.113.7:51820", []string{"[2001:db8::7]:51820", bad}}} { + endpoint, endpoints = c.endpoint, c.list + if _, _, err := pairWithToken(fakeToken("acme"), false, false); err == nil || !strings.Contains(err.Error(), "not a host:port") { + t.Errorf("%q in %+v: the pairing must be refused for its endpoint, got %v", bad, c, err) + } + } + } + _ = filepath.Walk(cfgHome, func(path string, fi os.FileInfo, err error) error { + if err == nil && !fi.IsDir() { + t.Errorf("a refused pairing left a file behind: %s", path) + } + return nil + }) + + for _, good := range goodTunnelEndpoints { + uiTestConfigDir(t) // a local bastion serves one host: a fresh box each + withMarkerAt(t, filepath.Join(t.TempDir(), "machine.json")) + endpoint, endpoints = good, []string{good} + cfg, _, err := pairWithToken(fakeToken("acme"), false, false) + if err != nil { + t.Fatalf("%q must pair: %v", good, err) + } + if cfg.Endpoint != good || len(cfg.Endpoints) != 1 || cfg.Endpoints[0] != good { + t.Fatalf("%q: saved %q %q", good, cfg.Endpoint, cfg.Endpoints) + } + } +} + +// A pairing file that already holds an unsafe endpoint (written by an older +// build, or by hand) loads as invalid, naming the file and why: it is never +// served, while every other pairing still loads, status shows it, and unpair +// removes it. A machine that upgrades is never stopped by one such file. +func TestAPairingFileWithAnUnsafeEndpointLoadsButIsNeverServed(t *testing.T) { + uiTestConfigDir(t) + withMarkerAt(t, filepath.Join(t.TempDir(), "machine.json")) + supervised(t) + prev := unpairNeedsRoot + unpairNeedsRoot = func() bool { return false } + t.Cleanup(func() { unpairNeedsRoot = prev }) + pd, err := pairingsDir() + if err != nil { + t.Fatal(err) + } + write := func(c *Config) string { + t.Helper() + b, _ := json.Marshal(c) + p := filepath.Join(pd, c.HostName+".json") + if err := os.WriteFile(p, b, 0o600); err != nil { + t.Fatal(err) + } + return p + } + const host, other = "acme.host.privatae.ai", "other.host.privatae.ai" + write(&Config{HostName: other, Endpoint: goodTunnelEndpoints[0]}) + for _, bad := range badTunnelEndpoints { + if bad == "" { + continue // a pairing over a socket has no endpoint + } + for _, c := range []*Config{ + {HostName: host, Endpoint: bad}, + {HostName: host, Endpoint: "203.0.113.7:51820", Endpoints: []string{bad}}, + } { + p := write(c) + cfgs, err := loadPairings() + if err != nil || len(cfgs) != 2 { + t.Fatalf("%q: one bad file must not fail the load: %v %v", bad, cfgs, err) + } + got := pairingFor(cfgs, host) + if got.active() || got.stateName() != "invalid" { + t.Errorf("%q: the pairing must load invalid: active=%v %q", bad, got.active(), got.stateName()) + } + if out := statusOutput(t); !strings.Contains(out, host+" [invalid]") || + !strings.Contains(out, "not a host:port") || !strings.Contains(out, p) { + t.Errorf("%q: status must show the pairing invalid, naming its file and why:\n%s", bad, out) + } + if !pairingFor(cfgs, other).active() { + t.Errorf("%q: the other pairing must still be served", bad) + } + if startPairing(got) { + t.Errorf("%q: an invalid pairing was started", bad) + } + } + } + // The supervisor starts nothing for it either (the other pairing is + // removed first: starting it would raise a real tunnel). + if err := os.Remove(filepath.Join(pd, other+".json")); err != nil { + t.Fatal(err) + } + superviseOnce() + supervisor.Lock() + running := len(supervisor.runs) + supervisor.Unlock() + if running != 0 { + t.Fatalf("the supervisor started %d run(s) for an invalid pairing", running) + } + + if h, err := unpairHost(host); err != nil || h != host { + t.Fatalf("unpair must remove an invalid pairing: %q %v", h, err) + } + if cfgs, err := loadPairings(); err != nil || len(cfgs) != 0 { + t.Fatalf("after unpair nothing is left: %v %v", cfgs, err) + } + + for _, good := range goodTunnelEndpoints { + write(&Config{HostName: host, Endpoint: good, Endpoints: goodTunnelEndpoints}) + if cfgs, err := loadPairings(); err != nil || len(cfgs) != 1 || cfgs[0].Endpoint != good || !cfgs[0].active() { + t.Fatalf("%q must load and be served: %v %v", good, cfgs, err) + } + } + write(&Config{HostName: host, Transport: transportUnix}) + if cfgs, err := loadPairings(); err != nil || len(cfgs) != 1 || !cfgs[0].active() { + t.Fatalf("a pairing with no endpoint must load and be served: %v %v", cfgs, err) + } +} + +// statusOutput is what `status` prints. +func statusOutput(t *testing.T) string { + t.Helper() + r, w, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + stdout := os.Stdout + os.Stdout = w + serr := cmdStatus() + os.Stdout = stdout + _ = w.Close() + out, _ := io.ReadAll(r) + if serr != nil { + t.Fatalf("status: %v", serr) + } + return string(out) +} + +// A rotate directive that names any unsafe endpoint is not acted on; one +// that names only host:port entries is. +func TestARotateDirectiveWithAnUnsafeEndpointIsIgnored(t *testing.T) { + for _, bad := range badTunnelEndpoints { + if bad == "" { + continue // an empty entry is no entry + } + for _, d := range [][]byte{ + directive(bad, nil, "PK="), + directive("203.0.113.7:51820", []string{bad, "[2001:db8::7]:51820"}, "PK="), + } { + if c, pk := parseRotateTarget(d); len(c) != 0 || pk != "" { + t.Errorf("%q: the directive must be ignored, got %q %q", bad, c, pk) + } + } + } + c, pk := parseRotateTarget(directive("203.0.113.7:51820", goodTunnelEndpoints, "PK=")) + if pk != "PK=" || !sameList(c, goodTunnelEndpoints) { + t.Fatalf("a directive of host:port entries must be taken: %q %q", c, pk) + } +} diff --git a/client-go/freeagent.go b/client-go/freeagent.go index d0af532..22a6d0e 100644 --- a/client-go/freeagent.go +++ b/client-go/freeagent.go @@ -60,6 +60,27 @@ type freeagentConfig struct { Cert string // PEM: the client identity this box presents Key string CA string // PEM: the root the local agent's server cert must chain to + // Mode is the sanitize mode this bastion wrote into the agent's manifest + // (faManagedSanitizeMode for every agent it deploys), or "" for an agent + // someone runs by hand, whose manifest this bastion did not write. + Mode string +} + +// faManagedSanitizeMode is the sanitize mode the bastion writes into every +// manifest it deploys (faManifestText). Flag mode is the agent's rules pass +// only: it replaces e-mail addresses, phone numbers, card, bank and ID +// numbers, IP and MAC addresses and keys, and does not find names, street +// addresses or dates of birth. +const faManagedSanitizeMode = "flag" + +// faSanitizeNotes says, for each mode a reply can carry, what personal data +// may still be in the text, in the words the host relays. A mode with no entry +// here is reported as not known to remove personal data. +var faSanitizeNotes = map[string]string{ + "flag": "rules only: e-mail addresses, phone numbers, card, bank and ID numbers, IP and MAC addresses " + + "and keys were replaced; names, street addresses and dates of birth were not", + "sterilize": "", + "none": "the agent's manifest has no sanitize stage, so nothing in the text was replaced", } func freeagentEnv() (freeagentConfig, bool) { @@ -252,6 +273,16 @@ func freeagentProbe() (bool, bool) { // local agent's own 403 (grant/mask) and 503 (manifest not // serving) are refusals with the reason it gave; 404 on read_range is gone. func freeagentMap(op string, status int, body []byte, err error) (map[string]any, string, bool) { + return freeagentMapMode(op, "", status, body, err) +} + +// freeagentMapMode is freeagentMap for an agent whose manifest this bastion +// wrote with sanitize mode `mode` ("" when it did not write it). A successful +// reply says how its text was sanitized: `sanitize_mode` is the mode this +// bastion wrote, else the one the agent reports, else "unknown"; `sanitized` +// is true only for "sterilize", the mode that looks for names; and +// `sanitize_note` says what the mode leaves in. +func freeagentMapMode(op, mode string, status int, body []byte, err error) (map[string]any, string, bool) { if err != nil { return map[string]any{"state": "offline", "detail": "freeagent: " + err.Error()}, "", true } @@ -267,7 +298,7 @@ func freeagentMap(op string, status int, body []byte, err error) (map[string]any return nil, "freeagent: malformed reply", false case status == http.StatusOK: payload["state"] = "ok" - payload["sanitized"] = true + faStampSanitize(payload, mode) return payload, "", true case status == http.StatusNotFound && op == "read_range": return map[string]any{"state": "gone", "detail": reason}, "", true @@ -286,10 +317,33 @@ func freeagentMap(op string, status int, body []byte, err error) (map[string]any } } +// faStampSanitize writes the three sanitize fields onto a successful reply. +// The mode this bastion wrote wins over the one the reply carries: a reply +// cannot make text it never sterilized read as sterilized. +func faStampSanitize(payload map[string]any, mode string) { + if mode == "" { + mode, _ = payload["sanitize_mode"].(string) + } + if mode == "" { + mode = "unknown" + } + note, known := faSanitizeNotes[mode] + if !known { + note = "the local agent did not say how this text was sanitized; it may contain personal data" + } + payload["sanitize_mode"] = mode + payload["sanitized"] = mode == "sterilize" + delete(payload, "sanitize_note") + if note != "" { + payload["sanitize_note"] = note + } +} + // handleDcnSearch — op `dcn_search`: args = . Gated // by the synced grant with the 'search' mask, then forwarded to the -// local agent's /search. Hits are sanitized windows with document ids the host -// feeds back to read_range. +// local agent's /search. Hits are windows of the agent's sanitized text, with +// document ids the host feeds back to read_range; faStampSanitize says which +// sanitize mode produced them. func handleDcnSearch(ws *websocket.Conn, dir string, args []string) { storeID, a, ok := dcnParseArgs(ws, args) if !ok { @@ -319,7 +373,7 @@ func handleDcnSearch(ws *websocket.Conn, dir string, args []string) { } status, body, err := freeagentCall(c, http.MethodPost, "/search", map[string]any{"query": q, "k": k}) - payload, why, ok := freeagentMap("search", status, body, err) + payload, why, ok := freeagentMapMode("search", c.Mode, status, body, err) if !ok { dcnRefuse(ws, dir, "search", storeID, "", why) return @@ -394,7 +448,7 @@ func handleDcnCatalog(ws *websocket.Conn, dir string, args []string) { "state": "offline", "detail": "no catalog yet — the local agent's first sweep has not finished; try again later"}) return } - payload, why, ok := freeagentMap("catalog", status, body, err) + payload, why, ok := freeagentMapMode("catalog", c.Mode, status, body, err) if !ok { dcnRefuse(ws, dir, "catalog", storeID, "kind:"+kind, why) return @@ -427,7 +481,7 @@ func handleDcnReadRangeDoc(ws *websocket.Conn, dir string, storeID int64, a *dcn } status, body, err := freeagentCall(c, http.MethodPost, "/read_range", map[string]any{"doc": a.Doc, "start": start, "end": end}) - payload, why, ok := freeagentMap("read_range", status, body, err) + payload, why, ok := freeagentMapMode("read_range", c.Mode, status, body, err) if !ok { dcnRefuse(ws, dir, "read_range", storeID, "doc:"+a.Doc, why) return diff --git a/client-go/freeagent_drop.go b/client-go/freeagent_drop.go index 1f68555..58fa5f7 100644 --- a/client-go/freeagent_drop.go +++ b/client-go/freeagent_drop.go @@ -81,11 +81,19 @@ type faOwner struct { gid uint32 name string home string - // plat is what this platform needs beyond the above: nothing on unix, the + // plat is what this platform needs beyond the above: nothing on unix (an + // owner there is fully described by its uid, gid, name and home), the // owner's SID on Windows (freeagent_drop_windows.go). plat faOwnerPlat } +// faOwnerPlat holds the owner's SID, as text, and their bare user name. Only +// Windows fills it; it stays zero on unix. +type faOwnerPlat struct { + sid string + user string +} + // faResolveOwner decides the identity for a store's agent. Where this // platform drops and the daemon holds machine authority it is the granted // folder's owner, refused for the machine's own accounts and for anyone who diff --git a/client-go/freeagent_drop_unix.go b/client-go/freeagent_drop_unix.go index f97bbc9..dac7787 100644 --- a/client-go/freeagent_drop_unix.go +++ b/client-go/freeagent_drop_unix.go @@ -20,9 +20,6 @@ func faDropSupported() bool { return true } // own 0700 leaf. func faRootStateBase() string { return "/var/lib/privatae/freeagent" } -// faOwnerPlat: a unix owner is fully described by its uid, gid, name and home. -type faOwnerPlat struct{} - // faHandsOverByChown: on unix every file root writes for the child is handed // over by changing its owner. const faHandsOverByChown = true diff --git a/client-go/freeagent_drop_windows.go b/client-go/freeagent_drop_windows.go index 6dd20ac..bce2c6e 100644 --- a/client-go/freeagent_drop_windows.go +++ b/client-go/freeagent_drop_windows.go @@ -71,12 +71,6 @@ func faRootStateBase() string { return filepath.Join(d, "freeagent-state") } -// faOwnerPlat holds the owner's SID, as text, and their bare user name. -type faOwnerPlat struct { - sid string - user string -} - // faHandsOverByChown: on Windows the hand-over is a permission entry on the // state leaf, inherited by everything later created in it. const faHandsOverByChown = false @@ -430,12 +424,6 @@ func (c *winChild) Wait() error { return nil } -func faPathOwner(string) (uint32, uint32, error) { - return 0, 0, errors.New("ownership is not read as a uid on windows") -} - -func faChown(string, uint32, uint32) error { return nil } - // sysProcAttr: the Windows drop does not go through a credential; faStartChild // starts the child under the owner's token itself. func (o *faOwner) sysProcAttr() *syscall.SysProcAttr { return nil } @@ -457,5 +445,4 @@ func faFdSane(f *os.File) error { return nil } -func faInfoOwner(os.FileInfo) (uint32, uint32, bool) { return 0, 0, false } -func faInfoLinks(os.FileInfo) (uint64, bool) { return 0, false } +func faInfoLinks(os.FileInfo) (uint64, bool) { return 0, false } diff --git a/client-go/freeagent_managed.go b/client-go/freeagent_managed.go index 864b190..999a28f 100644 --- a/client-go/freeagent_managed.go +++ b/client-go/freeagent_managed.go @@ -500,7 +500,7 @@ func faManifestText(id, cn string, terms *dcnTerms, sweep, recipient, embedModel fmt.Fprintf(&b, " exclude: [%s]\n", strings.Join(ex, ", ")) } fmt.Fprintf(&b, "mask: [%s]\nstages: [%s]\n", strings.Join(mask, ", "), strings.Join(stages, ", ")) - b.WriteString("sanitize:\n mode: flag\n") + fmt.Fprintf(&b, "sanitize:\n mode: %s\n", faManagedSanitizeMode) if containsStr(stages, "embed") { fmt.Fprintf(&b, "embed:\n model: %s\n dim: %d\n chunk: {size_tokens: 512, overlap_tokens: 64}\n", yamlStr(embedModel), embedDim) } @@ -677,7 +677,8 @@ func faManagedConfig(storeID int64) (freeagentConfig, bool) { if err != nil || port <= 0 { return freeagentConfig{}, false } - return freeagentConfig{Addr: "127.0.0.1:" + strconv.Itoa(port), Cert: inst.Cert, Key: inst.Key, CA: inst.Cert}, true + return freeagentConfig{Addr: "127.0.0.1:" + strconv.Itoa(port), Cert: inst.Cert, Key: inst.Key, CA: inst.Cert, + Mode: faManagedSanitizeMode}, true } // ── ensure / reconcile ────────────────────────────────────────────────────── diff --git a/client-go/freeagent_test.go b/client-go/freeagent_test.go index e9c022f..4a1e99a 100644 --- a/client-go/freeagent_test.go +++ b/client-go/freeagent_test.go @@ -190,8 +190,11 @@ func TestFreeagentProbeAndEnv(t *testing.T) { func TestFreeagentMap(t *testing.T) { m, why, ok := freeagentMap("search", 200, []byte(`{"hits":[]}`), nil) - if !ok || m["state"] != "ok" || m["sanitized"] != true || why != "" { - t.Fatalf("200 → ok/sanitized, got %v %q", m, why) + if !ok || m["state"] != "ok" || why != "" { + t.Fatalf("200 → ok, got %v %q", m, why) + } + if m["sanitized"] != false || m["sanitize_mode"] != "unknown" || m["sanitize_note"] == nil { + t.Fatalf("a reply that names no mode is not called sanitized, got %v", m) } m, _, ok = freeagentMap("read_range", 404, []byte(`{"error":"no such document"}`), nil) if !ok || m["state"] != "gone" { @@ -298,3 +301,45 @@ func TestDcnOpArgsCarryFreeagentFields(t *testing.T) { t.Fatalf("args not carried: %+v", a) } } + +// The sanitize fields say what was done to the text. An agent this bastion +// deployed runs in flag mode, which leaves names in: its replies are never +// called sanitized, whatever the reply itself claims. An agent run by hand is +// reported in the mode it names, and only sterilize counts as sanitized. +func TestASanitizeStampNamesTheModeThatRan(t *testing.T) { + for _, c := range []struct { + name, mode, body, wantMode string + wantSanitized, wantNote bool + }{ + {"deployed, reply silent", faManagedSanitizeMode, `{"hits":[]}`, "flag", false, true}, + {"deployed, reply claims sterilize", faManagedSanitizeMode, `{"hits":[],"sanitize_mode":"sterilize","sanitized":true}`, "flag", false, true}, + {"by hand, flag", "", `{"hits":[],"sanitize_mode":"flag"}`, "flag", false, true}, + {"by hand, sterilize", "", `{"hits":[],"sanitize_mode":"sterilize","sanitize_note":"x"}`, "sterilize", true, false}, + {"by hand, silent", "", `{"hits":[],"sanitized":true}`, "unknown", false, true}, + {"by hand, no sanitize stage", "", `{"hits":[],"sanitize_mode":"none"}`, "none", false, true}, + {"by hand, a mode this build does not know", "", `{"hits":[],"sanitize_mode":"other"}`, "other", false, true}, + } { + for _, op := range []string{"search", "read_range", "catalog"} { + m, why, ok := freeagentMapMode(op, c.mode, 200, []byte(c.body), nil) + if !ok || why != "" || m["state"] != "ok" { + t.Fatalf("%s/%s: %v %q", c.name, op, m, why) + } + note, hasNote := m["sanitize_note"].(string) + if m["sanitize_mode"] != c.wantMode || m["sanitized"] != c.wantSanitized || hasNote != c.wantNote || (hasNote && note == "") { + t.Errorf("%s/%s: got mode %v sanitized %v note %q", c.name, op, m["sanitize_mode"], m["sanitized"], note) + } + } + } + if !strings.Contains(faSanitizeNotes["flag"], "names") { + t.Fatal("the flag note must say names are left in") + } +} + +// The mode the stamp reports for a deployed agent is the mode the bastion +// writes into its manifest. +func TestTheDeployedManifestUsesTheStampedMode(t *testing.T) { + text := faManifestText("s1", "cn", &dcnTerms{Root: "/tmp/x", Mask: []string{"search"}}, "", "", "m", 8) + if !strings.Contains(text, "sanitize:\n mode: "+faManagedSanitizeMode+"\n") { + t.Fatalf("manifest sanitize block:\n%s", text) + } +} diff --git a/client-go/go.mod b/client-go/go.mod index f2db8a7..b368e2d 100644 --- a/client-go/go.mod +++ b/client-go/go.mod @@ -1,14 +1,14 @@ module git.privatae.ai/privatae/privatae-bastion -go 1.25.0 +go 1.26.0 require ( filippo.io/age v1.3.2 fyne.io/systray v1.12.2 github.com/creack/pty v1.1.24 github.com/gorilla/websocket v1.5.3 - golang.org/x/crypto v0.55.0 - golang.org/x/sys v0.47.0 + golang.org/x/crypto v0.58.0 + golang.org/x/sys v0.49.0 golang.zx2c4.com/wireguard v0.0.0-20260522210424-ecfc5a8d5446 gopkg.in/yaml.v3 v3.0.1 rsc.io/qr v0.2.0 @@ -18,7 +18,7 @@ require ( filippo.io/hpke v0.4.0 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/google/btree v1.1.2 // indirect - golang.org/x/net v0.57.0 // indirect + golang.org/x/net v0.61.0 // indirect golang.org/x/time v0.7.0 // indirect golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 // indirect gvisor.dev/gvisor v0.0.0-20250503011706-39ed1f5ac29c // indirect diff --git a/client-go/go.sum b/client-go/go.sum index 42f76c0..a38df58 100644 --- a/client-go/go.sum +++ b/client-go/go.sum @@ -14,14 +14,14 @@ github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= -golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= -golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= -golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= -golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= -golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= -golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= +golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54= +golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w= +golang.org/x/net v0.61.0 h1:RR+6j/BTBCrggOLugxa+66R8FvoN6MHlzDlZIbN9YFw= +golang.org/x/net v0.61.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= +golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA= +golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/term v0.47.0 h1:cpyOwQkr2Nzxe2p6KnrTG7Ei2wtzjsuSjhpXXepZ/BM= +golang.org/x/term v0.47.0/go.mod h1:efAfAuG6RKABvg1ArGebRkvX6LXul14SM67o2e4mUCY= golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ= golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 h1:B82qJJgjvYKsXS9jeunTOisW56dUokqW/FOteYJJ/yg= diff --git a/client-go/harness.go b/client-go/harness.go index 81065fd..981da78 100644 --- a/client-go/harness.go +++ b/client-go/harness.go @@ -9,14 +9,17 @@ import ( "encoding/base64" "encoding/hex" "encoding/json" + "errors" "fmt" "io" + "io/fs" "net" "net/http" "net/url" "os" "os/exec" "path/filepath" + "runtime" "sort" "strconv" "strings" @@ -55,10 +58,9 @@ type harnessSpec struct { // push — and spends the person's approval (writes.go), bound to the // harness id, mode, image, repo and branch of THIS run. Approval string `json:"approval,omitempty"` - // mode=lab (the Lab Harness): build a sealed lab of N hardened - // hosts on THIS box's lab cluster, put the app under test in each, drive the test runbook with - // ONE coding-harness pod inside the lab, collect, tear down. Only a lab bastion answers it. - Hosts int `json:"hosts"` // how many lab hosts (Entity CRs) to build + // mode=lab (the lab harness): test tooling built only with -tags lab + // (lab.go). A release build refuses it with exit 126 (lab_off.go). + Hosts int `json:"hosts"` // how many lab hosts to build Envs map[string]string `json:"envs"` // app settings written to every lab host (.env) Tests []string `json:"tests"` // the runbook: ordered steps, one governed /run each Engine string `json:"engine"` // which coding harness drives the tests (PRIVATAE_APP_ID) @@ -342,9 +344,10 @@ func handleHarnessConn(ws *websocket.Conn, self, dcnDir string) { for { select { case r := <-resCh: - fmt.Printf("harness: job %s done exit=%d\n", spec.HarnessID, r.code) - sendHarness(ws, 'J', r.body) - sendHarnessExit(ws, r.code) + body, code := fitLane(r.body, r.code) + fmt.Printf("harness: job %s done exit=%d\n", spec.HarnessID, code) + sendHarness(ws, 'J', body) + sendHarnessExit(ws, code) return case <-ticker.C: sendHarness(ws, 'O', []byte("running\n")) // keepalive for long ops (worktree/git/lab runs) @@ -467,7 +470,15 @@ func runWorktree(spec harnessSpec) ([]byte, int) { return b, 1 } defer os.RemoveAll(dir) // collect-out is over the lane; the box keeps nothing - if err := extractTar(tarIn, dir); err != nil { + // The tree is a folder inside dir, as runGit's is: dir stays 0700, so the + // tree opened for the agent is out of every other local user's reach. + work := filepath.Join(dir, "work") + if err := os.Mkdir(work, 0o700); err != nil { + res["ok"], res["error"] = false, "tempdir: "+err.Error() + b, _ := json.Marshal(res) + return b, 1 + } + if err := extractTar(tarIn, work); err != nil { res["ok"], res["error"] = false, "extract: "+err.Error() b, _ := json.Marshal(res) return b, 1 @@ -485,7 +496,7 @@ func runWorktree(spec harnessSpec) ([]byte, int) { // Drive the harness image's REAL agent (its /run sidecar) with the prompt. The agent works // cwd in the bind-mounted /work, so the worktree is modified in place and collected below // (tar-out). - agentRes, agentErr := driveHarnessAgent(rt, dir, spec, maxMin) + agentRes, agentErr := driveHarnessAgent(rt, work, spec, maxMin) if agentErr != nil { res["ok"], res["error"] = false, "agent run: "+agentErr.Error() b, _ := json.Marshal(res) @@ -495,13 +506,22 @@ func runWorktree(spec harnessSpec) ([]byte, int) { res["transcript"] = agentRes["transcript"] res["agent_diff"] = agentRes["diff"] res["usage"] = agentRes["usage"] - tarOut, err := makeTar(dir) + tarOut, skips, err := packWorktreeSkips(work) + if errors.Is(err, errTreeTooLarge) { + // No tree at all rather than part of one: a shorter tar would read as + // the agent's whole work. + res["ok"], res["worktree_too_large"] = false, true + res["error"] = "the tree the agent left is not returned: " + err.Error() + ", the most this lane carries back" + b, _ := json.Marshal(res) + return b, 1 + } if err != nil { res["ok"], res["error"] = false, "repack: "+err.Error() b, _ := json.Marshal(res) return b, 1 } res["worktree_b64"] = base64.StdEncoding.EncodeToString(tarOut) + skips.report(res) res["meter_s"] = round2(time.Since(start).Seconds()) res["ok"] = true b, _ := json.Marshal(res) @@ -512,64 +532,154 @@ func runWorktree(spec harnessSpec) ([]byte, int) { // real harness agent against it, then commit + push to a // branch. The diff rides git — lane-light, no large tar. spec.RepoURL carries the per-run minted // token (the host mints it, dispatches, and revokes it after the run). +// +// The agent's tree, its .git included, is the agent's to write, and this +// process may be root: a hook, core.fsmonitor, a filter or a credential +// helper set in that .git would run here, and a changed remote would take the +// push and its token elsewhere. So git never runs in that tree after the +// agent has had it. The commit is made in a second clone, in a folder only +// this process can enter and that is mounted nowhere, which gets the agent's +// plain files and folders (mirrorTree) and nothing of its .git. The branch +// gets one commit holding the agent's final files. func runGit(spec harnessSpec) ([]byte, int) { res := map[string]interface{}{"mode": "git", "ran_on": orUnknown(hostnameSafe())} + fail := func(msg string) ([]byte, int) { + res["ok"], res["error"] = false, msg + b, _ := json.Marshal(res) + return b, 1 + } rt := detectRuntime() res["runtime"] = rt if rt == "" { - res["ok"], res["error"] = false, "no container runtime" - b, _ := json.Marshal(res) - return b, 1 + return fail("no container runtime") } if spec.Image == "" || spec.RepoURL == "" { - res["ok"], res["error"] = false, "image and repo_url required for mode=git" - b, _ := json.Marshal(res) - return b, 1 + return fail("image and repo_url required for mode=git") } dir, err := os.MkdirTemp("", "harness-git-") if err != nil { - res["ok"], res["error"] = false, "tempdir: "+err.Error() - b, _ := json.Marshal(res) - return b, 1 + return fail("tempdir: " + err.Error()) } defer os.RemoveAll(dir) + // The push side: the clean clone and the empty HOME every git call runs + // with. MkdirTemp makes it 0700, and it is never mounted. + own, err := os.MkdirTemp("", "harness-push-") + if err != nil { + return fail("tempdir: " + err.Error()) + } + defer os.RemoveAll(own) + home := filepath.Join(own, "home") + if err := os.Mkdir(home, 0o700); err != nil { + return fail("tempdir: " + err.Error()) + } + // Every git call names the repository by its URL without the token and + // gets the token from its environment, so neither clone's config holds it + // and no process argument shows it. + bareURL, auth, err := repoGitAuth(spec.RepoURL) + if err != nil { + return fail("repo_url: " + err.Error()) + } + git := func(timeout time.Duration, in string, args ...string) (string, error) { + return harnessGitAuth(timeout, home, auth, in, args...) + } repoDir := filepath.Join(dir, "repo") start := time.Now() - if out, err := runCmd(120*time.Second, "git", "clone", "--", spec.RepoURL, repoDir); err != nil { - res["ok"], res["error"] = false, "clone: "+truncate(scrubURL(out, spec.RepoURL), 220) - b, _ := json.Marshal(res) - return b, 1 + if out, err := git(120*time.Second, "", "clone", "--", bareURL, repoDir); err != nil { + return fail("clone: " + truncate(scrubURL(out, spec.RepoURL), 220)) + } + // The agent gets no push access of its own: the run's commit is made from + // the clean clone below. Its clone was made with no token in its URL, and + // is refused if the token is in any file of its .git all the same. + if err := dropRepoCredential(git, repoDir, spec.RepoURL); err != nil { + return fail("clone: " + truncate(scrubURL(err.Error(), spec.RepoURL), 220)) } branch := spec.Branch if branch == "" { branch = "harness/" + spec.HarnessID } - _, _ = runCmd(15*time.Second, "git", "-C", repoDir, "checkout", "-b", branch) - _, _ = runCmd(15*time.Second, "git", "-C", repoDir, "config", "user.email", "harness@privatae.ai") - _, _ = runCmd(15*time.Second, "git", "-C", repoDir, "config", "user.name", "harness") + // The commit the agent starts from, read before it has the tree: the + // clean clone commits on top of the same one. An empty repository has + // none, and the run's commit is its first. + base := "" + if out, err := git(10*time.Second, repoDir, "rev-parse", "--verify", "-q", "HEAD^{commit}"); err == nil { + base = strings.TrimSpace(out) + } + _, _ = git(15*time.Second, repoDir, "checkout", "-b", branch) + _, _ = git(15*time.Second, repoDir, "config", "user.email", harnessGitEmail) + _, _ = git(15*time.Second, repoDir, "config", "user.name", harnessGitName) maxMin := spec.MaxMinutes if maxMin <= 0 { maxMin = 10 } // Drive the harness image's REAL agent against the cloned worktree (its /run sidecar); - // the agent modifies /work in place, committed + pushed below. + // the agent modifies /work in place, copied into the clean clone below. agentRes, agentErr := driveHarnessAgent(rt, repoDir, spec, maxMin) if agentErr != nil { - res["ok"], res["error"] = false, "agent run: "+agentErr.Error() - b, _ := json.Marshal(res) - return b, 1 + return fail("agent run: " + agentErr.Error()) } res["run_ok"] = agentRes["ok"] == true res["transcript"] = agentRes["transcript"] - _, _ = runCmd(15*time.Second, "git", "-C", repoDir, "add", "-A") - commitOut, _ := runCmd(20*time.Second, "git", "-C", repoDir, "commit", "-m", "harness run: "+spec.HarnessID) - res["commit_out"] = truncate(commitOut, 200) - if out, err := runCmd(60*time.Second, "git", "-C", repoDir, "push", "origin", branch); err != nil { - res["ok"], res["error"] = false, "push: "+truncate(scrubURL(out, spec.RepoURL), 220) - b, _ := json.Marshal(res) - return b, 1 + + clean := filepath.Join(own, "repo") + if out, err := git(120*time.Second, "", "clone", "--no-checkout", "--", bareURL, clean); err != nil { + return fail("clone for the push: " + truncate(scrubURL(out, spec.RepoURL), 220)) } - sha, _ := runCmd(10*time.Second, "git", "-C", repoDir, "rev-parse", "HEAD") + checkout := []string{"checkout", "-b", branch} + if base != "" { + checkout = append(checkout, base) + } + if out, err := git(30*time.Second, clean, checkout...); err != nil { + return fail("clone for the push: could not check out the commit the run started from (" + orUnknown(base) + "): " + + truncate(scrubURL(out, spec.RepoURL), 160)) + } + _, _ = git(15*time.Second, clean, "config", "user.email", harnessGitEmail) + _, _ = git(15*time.Second, clean, "config", "user.name", harnessGitName) + skips, err := mirrorTree(repoDir, clean, maxCopiedTreeBytes) + if errors.Is(err, errTreeTooLarge) { + res["tree_too_large"] = true + return fail("nothing was pushed: " + err.Error() + ", the most a run copies (each file and folder counts 512 bytes and its name besides its content)") + } + if err != nil { + return fail("copy the agent's tree: " + err.Error()) + } + skips.report(res) + if out, err := git(60*time.Second, clean, "add", "-A"); err != nil { + return fail("nothing was pushed: add: " + truncate(scrubURL(out, spec.RepoURL), 220)) + } + // A run that changed no file has nothing to commit and pushes nothing: + // ok, with changed false and no branch, since none was made. + dout, derr := git(60*time.Second, clean, "diff", "--cached", "--quiet") + var exitErr *exec.ExitError + switch { + case derr == nil: + res["changed"] = false + res["note"] = "the agent changed no file; nothing was committed or pushed" + res["meter_s"] = round2(time.Since(start).Seconds()) + res["ok"] = true + b, _ := json.Marshal(res) + return b, 0 + case !errors.As(derr, &exitErr) || exitErr.ExitCode() != 1: + return fail("nothing was pushed: diff: " + truncate(scrubURL(dout, spec.RepoURL), 220)) + } + commitOut, err := git(20*time.Second, clean, "commit", "-m", "harness run: "+spec.HarnessID) + res["commit_out"] = truncate(scrubURL(commitOut, spec.RepoURL), 200) + if err != nil { + return fail("nothing was pushed: commit: " + truncate(scrubURL(commitOut, spec.RepoURL), 220)) + } + res["changed"] = true + // Hooks never run, so git-lfs's pre-push hook does not either: the large + // files the commit points to are sent first, to the repository's own LFS + // address, whatever a .lfsconfig in the agent's tree names. + if harnessLFSConfig() != nil { + args := append(lfsEndpointPins(bareURL), "lfs", "push", "origin", branch) + if out, err := git(10*time.Minute, clean, args...); err != nil { + return fail("nothing was pushed: lfs push: " + truncate(scrubURL(out, spec.RepoURL), 220)) + } + } + if out, err := git(60*time.Second, clean, "push", "origin", branch); err != nil { + return fail("push: " + truncate(scrubURL(out, spec.RepoURL), 220)) + } + sha, _ := git(10*time.Second, clean, "rev-parse", "HEAD") res["branch"], res["commit"] = branch, strings.TrimSpace(sha) res["meter_s"] = round2(time.Since(start).Seconds()) res["ok"] = true @@ -577,13 +687,189 @@ func runGit(spec harnessSpec) ([]byte, int) { return b, 0 } +// harnessGitSafe goes on every git the harness runs: no hook, no file +// system monitor and no ext:: transport, whatever a repository's config says. +var harnessGitSafe = []string{"-c", "core.hooksPath=" + os.DevNull, "-c", "core.fsmonitor=false", + "-c", "protocol.ext.allow=never"} + +// harnessGitAuth runs git (in the folder in, if not "") with harnessGitSafe and +// an environment that brings no config of this machine's: no system file, no +// global file, HOME and XDG_CONFIG_HOME at home (an empty folder), and no +// GIT_ variable of this process's. auth (repoGitAuth's environment) is added +// to the environment, so a credential is never in a process argument, where +// any local user can read it. Output as runCmd gives it. +func harnessGitAuth(timeout time.Duration, home string, auth []string, in string, args ...string) (string, error) { + a := append(append([]string{}, harnessGitSafe...), harnessLFSConfig()...) + if in != "" { + a = append(a, "-C", in) + } + a = append(a, args...) + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + cmd := exec.CommandContext(ctx, "git", a...) + hideConsole(cmd) + cmd.Env = append(harnessGitEnv(home), auth...) + var out, errb bytes.Buffer + cmd.Stdout, cmd.Stderr = &out, &errb + if err := cmd.Run(); err != nil { + return strings.TrimSpace(out.String() + " " + errb.String()), err + } + return out.String(), nil +} + +func harnessGitEnv(home string) []string { + env := []string{} + for _, kv := range os.Environ() { + k, _, _ := strings.Cut(kv, "=") + switch k = strings.ToUpper(k); { + case strings.HasPrefix(k, "GIT_"), k == "HOME", k == "XDG_CONFIG_HOME": + continue + } + env = append(env, kv) + } + // The harness identity on every call, so git never makes one from this + // machine's user and host name: that costs a host name lookup per call + // (seconds where there is no resolver) and writes the name into the + // reflog of the clone the agent is handed. + return append(env, "HOME="+home, "XDG_CONFIG_HOME="+home, "GIT_CONFIG_NOSYSTEM=1", + "GIT_CONFIG_GLOBAL="+os.DevNull, "GIT_TERMINAL_PROMPT=0", + "GIT_AUTHOR_NAME="+harnessGitName, "GIT_AUTHOR_EMAIL="+harnessGitEmail, + "GIT_COMMITTER_NAME="+harnessGitName, "GIT_COMMITTER_EMAIL="+harnessGitEmail) +} + +// The name and address a harness run's commits carry. +const ( + harnessGitName = "harness" + harnessGitEmail = "harness@privatae.ai" +) + +// harnessLFSConfig is the git-lfs filter, as `git lfs install` writes it, +// when git-lfs is installed here. No global config is read, so a repository +// that keeps files in LFS would otherwise be cloned as pointer files and its +// changed large files committed whole. It comes from this process only: a +// filter the repository or the agent names under any other name has no +// command, so it never runs. +func harnessLFSConfig() []string { + if _, err := exec.LookPath("git-lfs"); err != nil { + return nil + } + return []string{"-c", "filter.lfs.clean=git-lfs clean -- %f", "-c", "filter.lfs.smudge=git-lfs smudge -- %f", + "-c", "filter.lfs.process=git-lfs filter-process", "-c", "filter.lfs.required=true"} +} + +// lfsEndpointPins sets every LFS address git-lfs reads to the one it derives +// from repoURL itself, so a .lfsconfig in the agent's tree cannot send the +// push anywhere else. git config wins over .lfsconfig. repoURL is the URL +// without its credential (repoGitAuth sends that, to this repository only). +// Only an http(s) address is pinned; for any other, git-lfs is left to derive +// the address as it does. +func lfsEndpointPins(repoURL string) []string { + u, err := url.Parse(repoURL) + if err != nil || (u.Scheme != "https" && u.Scheme != "http") { + return nil + } + ep := strings.TrimSuffix(repoURL, "/") + if !strings.HasSuffix(ep, ".git") { + ep += ".git" + } + ep += "/info/lfs" + var a []string + for _, k := range []string{"lfs.url", "lfs.pushurl", "remote.origin.lfsurl", "remote.origin.lfspushurl"} { + a = append(a, "-c", k+"="+ep) + } + return a +} + +// repoURLParts is repoURL without its user and password, and the secret it +// carried (the password, or a user name standing alone, which is how some +// forges take a token). A URL with no user part comes back as it is. +func repoURLParts(repoURL string) (bare, secret string, err error) { + u, perr := url.Parse(repoURL) + if perr != nil { + if strings.Contains(repoURL, "://") { + return "", "", errors.New("the repository URL does not parse") + } + return repoURL, "", nil // a local path or user@host:path, which holds no token + } + if u.User == nil { + return repoURL, "", nil + } + secret = u.User.Username() + if p, ok := u.User.Password(); ok && p != "" { + secret = p + } + u.User = nil + return u.String(), secret, nil +} + +// repoGitAuth is how git reaches repoURL with no credential in a process +// argument or a config file: the URL without its user and password, and the +// environment that sends them as an Authorization header to that URL and the +// addresses under it only (http..extraHeader, which git and git-lfs both +// read, set through GIT_CONFIG_COUNT as the mirror's push sets its own). A +// URL with no user part needs no environment. +func repoGitAuth(repoURL string) (bare string, env []string, err error) { + bare, secret, err := repoURLParts(repoURL) + if err != nil || secret == "" { + return bare, nil, err + } + u, err := url.Parse(repoURL) + if err != nil { + return "", nil, err + } + pass, _ := u.User.Password() + cred := base64.StdEncoding.EncodeToString([]byte(u.User.Username() + ":" + pass)) + return bare, []string{"GIT_CONFIG_COUNT=1", "GIT_CONFIG_KEY_0=http." + bare + ".extraHeader", + "GIT_CONFIG_VALUE_0=Authorization: Basic " + cred}, nil +} + +// dropRepoCredential makes the clone at repoDir name its origin without the +// token repoURL carries, then refuses it if the token is still in any file of +// its .git (objects aside, which hold only the repository's content). +func dropRepoCredential(git func(time.Duration, string, ...string) (string, error), repoDir, repoURL string) error { + bare, secret, err := repoURLParts(repoURL) + if err != nil || secret == "" { + return err + } + if out, err := git(15*time.Second, repoDir, "remote", "set-url", "origin", bare); err != nil { + return fmt.Errorf("remove the token from the agent's clone: %s", out) + } + if len(secret) < 8 { + return nil // too short to be a token, and would match ordinary text + } + gitDir := filepath.Join(repoDir, ".git") + return filepath.WalkDir(gitDir, func(p string, d fs.DirEntry, werr error) error { + if werr != nil { + return werr + } + if d.IsDir() { + if p == filepath.Join(gitDir, "objects") || p == filepath.Join(gitDir, "lfs") { + return filepath.SkipDir + } + return nil + } + if !d.Type().IsRegular() { + return nil + } + b, err := os.ReadFile(p) + if err != nil { + return err + } + if bytes.Contains(b, []byte(secret)) { + rel, _ := filepath.Rel(repoDir, p) + return fmt.Errorf("the token is still in the agent's clone, in %s", filepath.ToSlash(rel)) + } + return nil + }) +} + // driveHarnessAgent runs the harness IMAGE as a sidecar server (PORT=8844, /work bind-mounted) // and drives its REAL agent via POST /run with the prompt. The // agent works cwd in /work, so the worktree is modified in place (the caller collects it). The agent's // OWN per-run credential (mint->use->revoke, never baked into the image) is delivered via // spec.AgentCreds; absent => the image's baked-in auth (the test path). Always rm's the container. // Returns the harness /run JSON ({ok, exit_code, transcript, diff, usage, ...}). -func driveHarnessAgent(rt, dir string, spec harnessSpec, maxMin int) (map[string]interface{}, error) { +func driveHarnessAgent(rt, dir string, spec harnessSpec, maxMin int) (agentRes map[string]interface{}, runErr error) { const port = 8844 // Run the SAME image the harness-pool Deployment runs, with the SAME bind env it // passes — not invented flags. The sidecar's bind defaults to 127.0.0.1 @@ -597,16 +883,15 @@ func driveHarnessAgent(rt, dir string, spec harnessSpec, maxMin int) (map[string // (not via a platform proxy) — the one intentional difference from the pool env. // // The non-root harness (uid 10005) must READ AND WRITE the whole bind-mounted /work - // tree — its cwd is /work and it edits files in place. The worktree dir is 0700 - // root-owned (MkdirTemp) with 0644/0755 root-owned contents, so the harness can't - // even enter it: POST /run dies "Permission denied: /work" in seconds. Open the - // ephemeral per-run tree the same way as the creds dir below (it's removed after). - _ = filepath.Walk(dir, func(p string, _ os.FileInfo, werr error) error { - if werr == nil { - _ = os.Chmod(p, 0o777) - } - return nil - }) + // tree — its cwd is /work and it edits files in place. Its contents are 0644/0755 + // and this process's own, so the harness could not even enter it: POST /run dies + // "Permission denied: /work" in seconds. So the ephemeral per-run tree is opened to + // every user (it's removed after), and it must sit in a folder no other user can + // enter: that folder is what keeps the rest of this machine out of the run's tree. + if err := harnessTreeEnclosed(dir); err != nil { + return nil, err + } + openHarnessTree(dir) // Both spellings of the sidecar's two variables: the harness image is // published on its own schedule, so a box run must work against one that // reads either name. The older pair comes out once none does. @@ -621,25 +906,15 @@ func driveHarnessAgent(rt, dir string, spec harnessSpec, maxMin int) (map[string // sidecar reads + self-refreshes the FILE, ignoring CLAUDE_CODE_OAUTH_TOKEN). Mount a temp // .claude dir holding it, then DELETE the dir after the run so nothing durable is left on the // box. The host pulls these creds from the entity vault per run. - var credDir string if spec.AgentCreds != "" { - if d, derr := os.MkdirTemp("", "harness-claude-*"); derr == nil { - credDir = d - // MkdirTemp is 0700 + root-owned; the harness container runs as a NON-root - // user (uid 10005) that must both READ the token AND WRITE in this dir — the - // bundled Claude Code SDK creates .claude/session-env for its Bash tool and the - // in-container token refresh rewrites .credentials.json. 0755 only lets it read, - // so Bash dies ("could not create session-env") and refresh can't persist. Make - // the per-run temp dir world-writable (it's ephemeral, on the user's own box, - // removed right after the run); the creds file itself stays 0644. - _ = os.Chmod(d, 0o777) - if werr := os.WriteFile(filepath.Join(d, ".credentials.json"), []byte(spec.AgentCreds), 0o644); werr == nil { - args = append(args, "-v", credDir+":/home/privatae/.claude") - } + credDir, credRemove, err := writeHarnessCreds(spec.AgentCreds) + if credRemove != "" { + defer os.RemoveAll(credRemove) } - } - if credDir != "" { - defer os.RemoveAll(credDir) + if err != nil { + return nil, fmt.Errorf("agent credential: %w", err) + } + args = append(args, "-v", credDir+":/home/privatae/.claude") } args = append(args, spec.Image) out, err := runCmd(60*time.Second, rt, args...) @@ -647,7 +922,17 @@ func driveHarnessAgent(rt, dir string, spec harnessSpec, maxMin int) (map[string return nil, fmt.Errorf("harness server start: %s", truncate(out, 200)) } cid := strings.TrimSpace(out) - defer func() { _, _ = runCmd(20*time.Second, rt, "rm", "-f", cid) }() + // The caller reads the tree once this returns, so the agent must be gone + // by then: one still running could go on writing it while it is read. A + // container whose removal is not confirmed fails the run, and nothing of + // its tree is used. + defer func() { + if out, err := runCmd(20*time.Second, rt, "rm", "-f", cid); err != nil { + agentRes = nil + runErr = fmt.Errorf("the agent's container %s could not be removed, so nothing of its tree was used: %s", + shortID(cid), truncate(strings.TrimSpace(out), 160)) + } + }() ip := containerIP(rt, cid) if ip == "" { @@ -674,6 +959,104 @@ func driveHarnessAgent(rt, dir string, spec harnessSpec, maxMin int) (map[string return res, nil } +// harnessAgentUID is the user the harness image runs its agent as, the owner +// of its home /home/privatae, where the credential folder is mounted. +const harnessAgentUID = 10005 + +// writeHarnessCreds puts the agent's credential in a new folder for one run, +// and returns the folder to mount and the folder to remove after the run (the +// second even with an error). The agent must read the file and write beside +// it (its Bash tool makes session-env there, and a token refresh rewrites the +// file). As root the folder (0700) and the file (0600) are handed to the +// agent's user. A process that is not root cannot do that, so it opens them +// to every user (0777, 0666) as it opens the run's tree; either way the +// folder sits in one only this process can enter, so no other local user can +// reach the credential. +func writeHarnessCreds(creds string) (mount, remove string, err error) { + remove, err = os.MkdirTemp("", "harness-claude-*") + if err != nil { + return "", "", err + } + d := filepath.Join(remove, "claude") + f := filepath.Join(d, ".credentials.json") + dirMode, fileMode := os.FileMode(0o777), os.FileMode(0o666) + if os.Geteuid() == 0 { + dirMode, fileMode = 0o700, 0o600 + } + if err := os.Mkdir(d, 0o700); err != nil { + return "", remove, err + } + if err := os.WriteFile(f, []byte(creds), 0o600); err != nil { + return "", remove, err + } + if os.Geteuid() == 0 { + for _, p := range []string{f, d} { + if err := os.Lchown(p, harnessAgentUID, harnessAgentUID); err != nil { + return "", remove, err + } + } + } + if err := os.Chmod(f, fileMode); err != nil { + return "", remove, err + } + if err := os.Chmod(d, dirMode); err != nil { + return "", remove, err + } + return d, remove, nil +} + +// harnessTreeEnclosed refuses a run tree whose folder another local user +// could enter: that folder must be a plain folder (never a link) and, where +// the system has file modes, have no access for group or others and, where +// it has owners, be this process's own. The tree itself is opened to every +// user for the agent (openHarnessTree), so this folder is all that keeps the +// rest of the machine out of it. +func harnessTreeEnclosed(dir string) error { + parent := filepath.Dir(filepath.Clean(dir)) + fi, err := os.Lstat(parent) + if err != nil { + return fmt.Errorf("run tree: %w", err) + } + if !fi.IsDir() { + return fmt.Errorf("run tree: %s is not a folder (a link is never followed)", parent) + } + if runtime.GOOS == "windows" { + return nil + } + if perm := fi.Mode().Perm(); perm&0o077 != 0 { + return fmt.Errorf("run tree: %s is open to other users (%04o); the tree is opened to every user, so it must sit in a folder only this process can enter", parent, perm) + } + if uid, ok := coderFileOwner(fi); ok && uid != os.Geteuid() { + return fmt.Errorf("run tree: %s belongs to uid %d, not to this process", parent, uid) + } + return nil +} + +// openHarnessTree gives every user read and write on the run tree, for the +// agent's uid: folders 0777, files 0666, or 0777 for a file that was +// executable, so a run commits no mode change of its own. A link is left as +// it is: chmod follows it, and the tree comes from the repository or the +// host, so a link in it may name any file on this machine. +func openHarnessTree(dir string) { + _ = filepath.WalkDir(dir, func(p string, d fs.DirEntry, werr error) error { + if werr != nil { + return nil + } + mode := os.FileMode(0o777) + switch { + case d.IsDir(): + case d.Type().IsRegular(): + if fi, err := d.Info(); err != nil || fi.Mode().Perm()&0o111 == 0 { + mode = 0o666 + } + default: + return nil // a link, a socket, a device: not opened, never followed + } + _ = os.Chmod(p, mode) + return nil + }) +} + // scrubURL removes a credential-bearing URL (and its token) from output, so a clone/push error // never echoes the per-run token back over the lane. func scrubURL(s, url string) string { @@ -728,53 +1111,413 @@ func extractTar(data []byte, dir string) error { } } -// makeTar packs dir (recursively; files + dirs, skipping .git) into a tar byte stream. +// makeTar packs dir (recursively; plain files and folders, skipping .git) into +// a tar byte stream. packWorktree also says what it left out. func makeTar(dir string) ([]byte, error) { + b, _, err := packWorktree(dir) + return b, err +} + +// uvicornWSMaxSize is the largest WebSocket message the host takes in: the +// uvicorn default, which the host's server does not override. A harness +// result is one such message, so this is the most a run can return. +const uvicornWSMaxSize = 16 << 20 + +// maxReturnedTreeBytes is the largest tar a run sends back. It rides base64 +// in the result (4/3 of its size), so 9 MiB is 12 MiB on the lane and leaves +// 4 MiB of the 16 MiB message for the transcript, the diff and the rest. +var maxReturnedTreeBytes int64 = 9 << 20 + +// maxCopiedTreeBytes is the most a git run copies from the agent's tree into +// the clone it pushes. That push goes over git, not the lane, and the copy +// carries the whole tree, so the lane's limit does not fit it; it is set at +// what a large forge takes in one push (GitHub refuses a push over 2 GB), +// which also bounds the disk this process writes for a run. +var maxCopiedTreeBytes int64 = 2 << 30 + +// errTreeTooLarge is a tree with more in it than the run may carry. +var errTreeTooLarge = errors.New("too large") + +// treeTooLarge says which entry took the tree over its limit. +func treeTooLarge(rel string, limit int64) error { + return fmt.Errorf("%s takes the tree over %d MiB: %w", rel, limit>>20, errTreeTooLarge) +} + +// packWorktree packs dir into a tar byte stream and lists, by their path in +// the tree, the entries it left out. +// +// A run's tree is written by the agent, which may still be writing it while +// it is packed, and this process may be root: a name seen as a plain file can +// be a link to any file on this machine by the time it is opened, and a folder +// seen as a folder can be a link to any folder. So the tree is walked by open +// folder (packTree, per system): each name is opened relative to the folder +// already open above it, a link is never followed, and what goes in is what +// was opened. Only a plain file or a folder goes in; a link, a pipe, a device, +// a socket, or a name that changed between the look and the open is left out +// and listed. +// +// Files are streamed from the opened file, never read whole, and the tar +// stops at maxReturnedTreeBytes: a tree larger than that is an error, never +// a shorter tar. +func packWorktree(dir string) ([]byte, []string, error) { + b, skips, err := packWorktreeSkips(dir) + return b, skips.names, err +} + +// packWorktreeSkips is packWorktree with the count of left-out entries. +func packWorktreeSkips(dir string) ([]byte, treeSkips, error) { var buf bytes.Buffer - tw := tar.NewWriter(&buf) - root := filepath.Clean(dir) - err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - rel, err := filepath.Rel(root, path) - if err != nil { - return err - } - if rel == "." { - return nil - } - if info.IsDir() && info.Name() == ".git" { - return filepath.SkipDir - } - hdr, err := tar.FileInfoHeader(info, "") - if err != nil { - return err - } - hdr.Name = filepath.ToSlash(rel) - if err := tw.WriteHeader(hdr); err != nil { - return err - } - if info.Mode().IsRegular() { - f, err := os.Open(path) - if err != nil { - return err + cw := &treeCapWriter{w: &buf, limit: maxReturnedTreeBytes} + sink := &tarSink{tw: tar.NewWriter(cw), cw: cw} + if err := packTree(sink, filepath.Clean(dir)); err != nil { + return nil, treeSkips{}, err + } + if err := sink.tw.Close(); err != nil { + return nil, treeSkips{}, err + } + return buf.Bytes(), sink.treeSkips, nil +} + +// treeSink takes what packTree opened, a folder before what is in it. rel is +// the entry's path in the tree with / between names. listed hears each name +// as the walk reads it from its folder, in batches of treeListBatch, and +// refuses it once the tree holds more entries than the sink takes; folder +// hears all the names in a folder, rel, once they are read and before any of +// them is put; skip hears each entry the walk leaves out. +type treeSink interface { + listed(rel string) error + put(f *os.File, fi os.FileInfo, rel string) error + folder(rel string, names []string) error + skip(rel string) +} + +// treeListBatch is how many names the walk reads from a folder at a time, so +// a folder of any size is refused after at most this many names past the +// limit, never read whole first. +const treeListBatch = 512 + +// readTreeNames reads the names in the open folder d (rel in the tree, with / +// between names) treeListBatch at a time, handing each to sink.listed as it +// is read, and returns them sorted. +func readTreeNames(sink treeSink, d *os.File, rel string) ([]string, error) { + var names []string + for { + batch, err := d.Readdirnames(treeListBatch) + for _, n := range batch { + p := n + if rel != "" { + p = rel + "/" + n } - _, cerr := io.Copy(tw, f) - f.Close() - if cerr != nil { - return cerr + if lerr := sink.listed(p); lerr != nil { + return nil, lerr } } + names = append(names, batch...) + if err == io.EOF || (err == nil && len(batch) == 0) { + break + } + if err != nil { + return nil, fmt.Errorf("read folder %q: %w", rel, err) + } + } + sort.Strings(names) + return names, nil +} + +// treeSkips counts the entries a walk left out and keeps the first +// maxSkippedReported of their paths, so a tree of any number of links holds +// no more than that in memory. +type treeSkips struct { + names []string + count int +} + +func (s *treeSkips) skip(rel string) { + s.count++ + if len(s.names) < maxSkippedReported { + s.names = append(s.names, rel) + } +} + +// report puts the left-out entries in a run's result: the count in full and +// the names kept, sorted. +func (s treeSkips) report(res map[string]interface{}) { + if s.count == 0 { + return + } + names := append([]string(nil), s.names...) + sort.Strings(names) + res["worktree_skipped_count"], res["worktree_skipped"] = s.count, names +} + +// treeEntryCost is what an entry costs a copy beside its content: a tar +// header block, as each entry costs the tar a worktree run returns, and its +// name. It bounds how many entries a tree may hold, empty files included. +func treeEntryCost(rel string) int64 { return 512 + int64(len(rel)) } + +// treeCapWriter passes writes on until limit bytes have gone through, then +// refuses them with errTreeTooLarge (contextextract.go's capWriter drops +// them instead). +type treeCapWriter struct { + w io.Writer + n int64 + limit int64 +} + +func (c *treeCapWriter) Write(p []byte) (int, error) { + if c.n+int64(len(p)) > c.limit { + return 0, errTreeTooLarge + } + n, err := c.w.Write(p) + c.n += int64(n) + return n, err +} + +// tarSink writes each entry into a tar. Every entry pays its header in the +// tar itself; listed also charges each name read against the tar's limit, +// so the names the walk holds stay within it. +type tarSink struct { + tw *tar.Writer + cw *treeCapWriter + listCost int64 + treeSkips +} + +func (s *tarSink) listed(rel string) error { + if s.listCost += treeEntryCost(rel); s.listCost > s.cw.limit { + return treeTooLarge(rel, s.cw.limit) + } + return nil +} + +func (s *tarSink) folder(string, []string) error { return nil } + +// put writes the header of an entry packTree opened and, for a file, its +// bytes, copied from the opened file. The size in the header is the opened +// file's size; a file over what is left of the limit is refused before a +// byte of it is read, and one that gets shorter while it is copied is an +// error, never a short entry. +func (s *tarSink) put(f *os.File, fi os.FileInfo, rel string) error { + hdr, err := tar.FileInfoHeader(fi, "") + if err != nil { + return err + } + hdr.Name = rel + if fi.IsDir() { + if err := s.tw.WriteHeader(hdr); err != nil { + return s.sized(rel, err) + } return nil - }) + } + // A header block, the content padded to a block, and the two blocks that + // end the tar. + if size := fi.Size(); size > s.cw.limit-s.cw.n-3*512 { + return treeTooLarge(rel, s.cw.limit) + } + if err := s.tw.WriteHeader(hdr); err != nil { + return s.sized(rel, err) + } + n, err := io.CopyN(s.tw, f, hdr.Size) + if err == io.EOF { + return fmt.Errorf("%s got shorter while it was packed (%d of %d bytes)", rel, n, hdr.Size) + } if err != nil { - return nil, err + return s.sized(rel, fmt.Errorf("read %s: %w", rel, err)) } - if err := tw.Close(); err != nil { - return nil, err + return nil +} + +func (s *tarSink) sized(rel string, err error) error { + if errors.Is(err, errTreeTooLarge) { + return treeTooLarge(rel, s.cw.limit) } - return buf.Bytes(), nil + return err +} + +// copySink copies each entry into the folder root, the clone a git run +// pushes. It writes only plain files and folders it makes itself: whatever is +// at a name in root before (a file, a folder, a link the repository holds) is +// removed first and never written through. A name .git in any letter case, +// at any depth, is left out, so nothing the agent wrote can stand in for the +// clone's own repository or a nested one. +// +// Every entry is charged treeEntryCost besides its bytes, and nothing is kept +// per entry, so neither the disk a copy writes nor the memory it holds grows +// past the limit however many files the tree has. +type copySink struct { + root string + limit int64 + total int64 + treeSkips +} + +// inDotGit says whether any name in rel is one dotGitName refuses. +func inDotGit(rel string) bool { + for _, name := range strings.Split(rel, "/") { + if dotGitName(name) { + return true + } + } + return false +} + +// charge takes size bytes of rel's content from what is left of the limit. +func (s *copySink) charge(rel string, size int64) error { + if size > s.limit-s.total { + return treeTooLarge(rel, s.limit) + } + s.total += size + return nil +} + +// listed charges each name the walk reads its treeEntryCost, whether or not +// it is then copied. +func (s *copySink) listed(rel string) error { + return s.charge(rel, treeEntryCost(rel)) +} + +// folder removes from the clone's folder rel every name the agent's folder +// does not hold, except the clone's own .git. A name the agent's folder holds +// but the copy leaves out (a link, a special file, a folder too deep) keeps +// what the clone has, the repository's version or nothing. It runs before any +// entry of the folder is copied, so a name that differs only in letter case +// on a system that ignores case is removed before the new one is written. +func (s *copySink) folder(rel string, names []string) error { + if rel != "" && inDotGit(rel) { + return nil + } + dir := filepath.Join(s.root, filepath.FromSlash(rel)) + ents, err := os.ReadDir(dir) + if err != nil { + return err + } + in := make(map[string]bool, len(names)) + for _, n := range names { + in[n] = true + } + for _, e := range ents { + if in[e.Name()] || (rel == "" && e.Name() == ".git") { + continue + } + if err := os.RemoveAll(filepath.Join(dir, e.Name())); err != nil { + return err + } + } + return nil +} + +func (s *copySink) put(f *os.File, fi os.FileInfo, rel string) error { + if inDotGit(rel) { + s.skip(rel) + return nil + } + dst := filepath.Join(s.root, filepath.FromSlash(rel)) + if fi.IsDir() { + if st, err := os.Lstat(dst); err == nil && !st.IsDir() { + if err := os.RemoveAll(dst); err != nil { + return err + } + } + if err := os.Mkdir(dst, 0o755); err != nil && !errors.Is(err, fs.ErrExist) { + return err + } + return nil + } + size := fi.Size() + if err := s.charge(rel, size); err != nil { + return err + } + if err := os.RemoveAll(dst); err != nil { + return err + } + mode := os.FileMode(0o644) + if fi.Mode().Perm()&0o111 != 0 { + mode = 0o755 + } + // O_EXCL: the name was just emptied, so this makes a new file and opens + // no link. + out, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_EXCL, mode) + if err != nil { + return err + } + n, err := io.CopyN(out, f, size) + if cerr := out.Close(); err == nil { + err = cerr + } + if err == io.EOF { + return fmt.Errorf("%s got shorter while it was copied (%d of %d bytes)", rel, n, size) + } + if err != nil { + return fmt.Errorf("copy %s: %w", rel, err) + } + return os.Chmod(dst, mode) // the mode as asked, whatever the umask +} + +// dotGitName says whether a file system may open name as .git: any letter +// case; on Windows also with dots or spaces after it, or a stream after a +// colon, or as its short name GIT~1; on HFS+ also with the code points it +// ignores in a name. These are the names git itself refuses in a tree +// (is_ntfs_dotgit, is_hfs_dotgit), checked on every system, since a name the +// agent made on one is copied by a bastion that may run on another. +func dotGitName(name string) bool { + name = strings.Map(func(r rune) rune { + switch { + case r >= 0x200c && r <= 0x200f, r >= 0x202a && r <= 0x202e, r >= 0x206a && r <= 0x206f, r == 0xfeff: + return -1 + } + return r + }, name) + if i := strings.IndexByte(name, ':'); i >= 0 { + name = name[:i] + } + name = strings.TrimRight(name, ". ") + return strings.EqualFold(name, ".git") || strings.EqualFold(name, "git~1") +} + +// mirrorTree makes the clone at dst hold the agent's tree at src: every +// plain file and folder in src is copied in, and whatever the clone has that +// src does not is deleted, except the clone's own .git. An entry the copy +// leaves out (a link, a special file, a .git) keeps what the clone has at +// that name, the repository's own version or nothing, so a link the agent +// made is never committed and one the repository holds is not deleted by a +// run that could not copy it (copySink.folder). Returns the entries left out. +func mirrorTree(src, dst string, limit int64) (treeSkips, error) { + sink := ©Sink{root: dst, limit: limit} + err := packTree(sink, filepath.Clean(src)) + return sink.treeSkips, err +} + +// packTreeBeforeOpen, when set, runs between packTree's look at an entry and +// its open: the gap a concurrent writer aims for. Only tests set it. +var packTreeBeforeOpen func(rel string) + +// maxTreeDepth bounds how deep packTree goes, since every level holds one +// folder open. A folder deeper than this is left out and listed. +const maxTreeDepth = 128 + +// maxSkippedReported bounds the left-out names a run reports; the count is +// always reported in full. +const maxSkippedReported = 100 + +// fitLane is the result a harness job sends: body, or, when body is more than +// one message on the lane can carry (uvicornWSMaxSize, less the frame's tag), +// a refusal that says so. The host would drop an oversized message and the +// run would read as lost. +func fitLane(body []byte, code int) ([]byte, int) { + if len(body) < uvicornWSMaxSize { + return body, code + } + var head struct { + Mode string `json:"mode"` + } + _ = json.Unmarshal(body, &head) + b, _ := json.Marshal(map[string]any{"ok": false, "mode": head.Mode, "result_too_large": true, + "error": fmt.Sprintf("the run's result is %d MiB, more than the %d MiB the lane to the host carries; nothing from it was returned", + len(body)>>20, uvicornWSMaxSize>>20)}) + if code == 0 { + code = 1 + } + return b, code } // detectRuntime returns the first working container runtime, or "" if none. diff --git a/client-go/harness_creds_linux_test.go b/client-go/harness_creds_linux_test.go new file mode 100644 index 0000000..4f116e9 --- /dev/null +++ b/client-go/harness_creds_linux_test.go @@ -0,0 +1,44 @@ +package main + +// The agent's credential for a harness run is a file in a folder mounted into +// the container. No other local user may reach either: as root the bastion +// hands both to the container's user, owner-only; as anyone else it opens them +// for that user, inside a folder only the bastion can enter. + +import ( + "os" + "path/filepath" + "strconv" + "strings" + "testing" +) + +func TestTheHarnessCredentialIsReadableOnlyByTheContainersUser(t *testing.T) { + // A stand-in runtime: on `run` it records the mode and owner of the folder + // the credential folder sits in, of that folder and of the file, then + // fails, so the run stops there. + bin := t.TempDir() + record := filepath.Join(bin, "record") + rt := filepath.Join(bin, "runtime") + script := "#!/bin/sh\n[ \"$1\" = run ] || exit 1\nfor a; do case \"$a\" in *:/home/privatae/.claude) d=${a%:/home/privatae/.claude};; esac; done\n" + + "[ -n \"$d\" ] && stat -c '%a %u %g' \"$(dirname \"$d\")\" \"$d\" \"$d/.credentials.json\" > " + record + "\nexit 1\n" + if err := os.WriteFile(rt, []byte(script), 0o755); err != nil { + t.Fatal(err) + } + spec := harnessSpec{HarnessID: "h1", Image: "harness:test", AgentCreds: `{"claudeAiOauth":{}}`} + if _, err := driveHarnessAgent(rt, t.TempDir(), spec, 1); err == nil { + t.Fatal("the stand-in runtime fails the start; the run must report it") + } + got, err := os.ReadFile(record) + if err != nil { + t.Fatalf("the credential folder was not mounted: %v", err) + } + self := strconv.Itoa(os.Geteuid()) + " " + strconv.Itoa(os.Getegid()) + want := "700 " + self + "\n777 " + self + "\n666 " + self + if os.Geteuid() == 0 { + want = "700 " + self + "\n700 10005 10005\n600 10005 10005" // the harness image's user + } + if strings.TrimSpace(string(got)) != want { + t.Fatalf("enclosing folder, folder, then file: want %q, got %q", want, strings.TrimSpace(string(got))) + } +} diff --git a/client-go/harness_gitrun_bounds_linux_test.go b/client-go/harness_gitrun_bounds_linux_test.go new file mode 100644 index 0000000..339c56c --- /dev/null +++ b/client-go/harness_gitrun_bounds_linux_test.go @@ -0,0 +1,154 @@ +package main + +// A git run's copy is bounded by how many entries a tree holds as well as by +// its bytes; it never hands git the run's token in an argument or a config +// file; and it uses nothing of the agent's tree while the agent's container +// may still be running. + +import ( + "encoding/base64" + "net/http" + "net/http/cgi" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// Many empty files cost nothing in bytes; each still counts against the +// copy, so a tree of them is refused and nothing is pushed. +func TestAGitRunRefusesATreeOfTooManyEntries(t *testing.T) { + f := newGitRunFixture(t) + f.setAgent(t, `cd "$1" +i=0; while [ $i -lt 40 ]; do mkdir d$i; j=0; while [ $j -lt 100 ]; do : > d$i/f$j; j=$((j+1)); done; i=$((i+1)); done +`) + saved := maxCopiedTreeBytes + maxCopiedTreeBytes = 1 << 20 // 4000 entries at 512 bytes each are 2 MB + t.Cleanup(func() { maxCopiedTreeBytes = saved }) + res, code := f.run(t) + if code == 0 || res["ok"] != false { + t.Fatalf("a tree of 4000 empty files was copied under a 1 MiB limit: %v", res) + } + if e, _ := res["error"].(string); !strings.Contains(e, "too large") { + t.Errorf("the error does not say the tree is too large: %q", e) + } + if branchPushed(f) { + t.Error("the branch was pushed") + } +} + +// A container the runtime does not confirm removed fails the run before any +// of its tree is copied. +func TestAGitRunFailsWhenTheAgentsContainerIsNotRemoved(t *testing.T) { + f := newGitRunFixture(t) + f.setAgent(t, "cd \"$1\" && echo new > edit.txt\n") + docker := filepath.Join(filepath.Dir(f.agentFile), "docker") + b, err := os.ReadFile(docker) + if err != nil { + t.Fatal(err) + } + failing := strings.Replace(string(b), "ps|pull|rm) exit 0;;", "ps|pull) exit 0;;\nrm) echo 'Error: removal stuck' >&2; exit 1;;", 1) + if failing == string(b) { + t.Fatal("the stand-in runtime changed shape") + } + if err := os.WriteFile(docker, []byte(failing), 0o755); err != nil { + t.Fatal(err) + } + res, code := f.run(t) + if code == 0 || res["ok"] != false { + t.Fatalf("a run whose container was not removed reads ok: %v", res) + } + if e, _ := res["error"].(string); !strings.Contains(e, "could not be removed") || !strings.Contains(e, "removal stuck") { + t.Errorf("the error does not say the container was not removed: %q", e) + } + if branchPushed(f) { + t.Error("the branch was pushed") + } +} + +// Against a server that needs the token, the run clones and pushes with the +// token in no git argument and in no clone's config. +func TestAGitRunKeepsTheTokenOutOfEveryArgumentAndConfig(t *testing.T) { + f := newGitRunFixture(t) + f.setAgent(t, "cd \"$1\" && echo new > edit.txt\n") + realGit, err := exec.LookPath("git") + if err != nil { + t.Skip("no git") + } + want := "Basic " + base64.StdEncoding.EncodeToString([]byte("harness:"+gitRunToken)) + backend := &cgi.Handler{Path: realGit, Args: []string{"http-backend"}, Root: "/git", + Env: []string{"GIT_PROJECT_ROOT=" + filepath.Dir(f.origin), "GIT_HTTP_EXPORT_ALL=1", + "REMOTE_USER=harness", "GIT_CONFIG_NOSYSTEM=1", "HOME=" + t.TempDir()}} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != want { + w.Header().Set("WWW-Authenticate", `Basic realm="test"`) + http.Error(w, "auth", http.StatusUnauthorized) + return + } + backend.ServeHTTP(w, r) + })) + t.Cleanup(srv.Close) + + logs := t.TempDir() + argvLog, cfgLog := filepath.Join(logs, "argv"), filepath.Join(logs, "config") + bin := t.TempDir() + wrapper := "#!/bin/sh\nprintf '%s\\n' \"$*\" >> " + argvLog + "\n" + + "prev=; for a; do if [ \"$prev\" = -C ] && [ -f \"$a/.git/config\" ]; then cat \"$a/.git/config\" >> " + cfgLog + "; fi; prev=$a; done\n" + + "exec " + realGit + " \"$@\"\n" + if err := os.WriteFile(filepath.Join(bin, "git"), []byte(wrapper), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH")) + + repoURL := strings.Replace(srv.URL, "http://", "http://harness:"+gitRunToken+"@", 1) + "/git/" + filepath.Base(f.origin) + b, code := runGit(harnessSpec{Mode: "git", HarnessID: "h1", Image: "harness:test", + RepoURL: repoURL, Branch: "agent-work", MaxMinutes: 1}) + if code != 0 { + t.Fatalf("the run failed: %s", b) + } + if !branchPushed(f) { + t.Fatal("the branch is not in the origin") + } + argv, _ := os.ReadFile(argvLog) + if !strings.Contains(string(argv), "push origin agent-work") { + t.Fatalf("the stand-in git did not see the push:\n%s", argv) + } + cfg, _ := os.ReadFile(cfgLog) + if len(cfg) == 0 { + t.Fatal("no clone's config was read") + } + for name, body := range map[string][]byte{"an argument": argv, "a clone's config": cfg} { + for _, secret := range []string{gitRunToken, strings.TrimPrefix(want, "Basic ")} { + if strings.Contains(string(body), secret) { + t.Errorf("the token is in %s:\n%s", name, body) + } + } + } +} + +// Every git call carries the harness identity: nothing of this machine's user +// or host name reaches the clone the agent is handed, and the commit is the +// harness's. +func TestAGitRunsGitCarriesTheHarnessIdentity(t *testing.T) { + f := newGitRunFixture(t) + reflog := filepath.Join(t.TempDir(), "reflog") + f.setAgent(t, "cp \"$1/.git/logs/HEAD\" "+reflog+" && cd \"$1\" && echo new > edit.txt\n") + res, code := f.run(t) + if code != 0 { + t.Fatalf("the run failed: %v", res) + } + b, err := os.ReadFile(reflog) + if err != nil { + t.Fatal(err) + } + host, _ := os.Hostname() + if !strings.Contains(string(b), " harness ") || (host != "" && strings.Contains(string(b), host)) { + t.Errorf("the agent's clone records another identity:\n%s", b) + } + who := strings.TrimSpace(testGit(t, f.origin, "log", "-1", "--format=%an <%ae> / %cn <%ce>", "agent-work")) + if who != "harness / harness " { + t.Errorf("the commit is by %q", who) + } +} diff --git a/client-go/harness_gitrun_fixes_linux_test.go b/client-go/harness_gitrun_fixes_linux_test.go new file mode 100644 index 0000000..9a2f669 --- /dev/null +++ b/client-go/harness_gitrun_fixes_linux_test.go @@ -0,0 +1,360 @@ +package main + +// What a git run hands the agent, how it ends when its commit cannot be made, +// and the git-lfs filter it supplies itself. + +import ( + "bufio" + "bytes" + "fmt" + "io" + "io/fs" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" +) + +const gitRunToken = "tok3n0f7hisrun5a1e" + +// The agent's clone names the repository without the run's token, and the +// token is in no file of the tree it is handed; the run still pushes. +func TestAGitRunHandsTheAgentNoToken(t *testing.T) { + f := newGitRunFixture(t) + snap := filepath.Join(t.TempDir(), "agent-saw") + f.setAgent(t, "cp -a \"$1\" "+snap+" && cd \"$1\" && echo new > edit.txt\n") + b, code := runGit(harnessSpec{Mode: "git", HarnessID: "h1", Image: "harness:test", + RepoURL: "file://harness:" + gitRunToken + "@" + f.origin, Branch: "agent-work", MaxMinutes: 1}) + if code != 0 { + t.Fatalf("the run failed: %s", b) + } + if bytes.Contains(b, []byte(gitRunToken)) { + t.Errorf("the result carries the token: %s", b) + } + n := 0 + err := filepath.WalkDir(snap, func(p string, d fs.DirEntry, err error) error { + if err != nil || !d.Type().IsRegular() { + return err + } + n++ + body, err := os.ReadFile(p) + if err == nil && bytes.Contains(body, []byte(gitRunToken)) { + t.Errorf("the token is in the agent's tree, in %s", p[len(snap):]) + } + return err + }) + if err != nil || n == 0 { + t.Fatalf("the agent's tree was not read (%d files): %v", n, err) + } + if got := testGit(t, f.origin, "show", "agent-work:edit.txt"); got != "new\n" { + t.Errorf("edit.txt = %q", got) + } +} + +func TestRepoURLPartsAndLFSPins(t *testing.T) { + bare, secret, err := repoURLParts("https://host-user:" + gitRunToken + "@git.example.invalid/o/r.git") + if err != nil || bare != "https://git.example.invalid/o/r.git" || secret != gitRunToken { + t.Errorf("parts = %q, %q, %v", bare, secret, err) + } + if bare, secret, err := repoURLParts("/srv/x/origin.git"); err != nil || bare != "/srv/x/origin.git" || secret != "" { + t.Errorf("a path = %q, %q, %v", bare, secret, err) + } + pins := strings.Join(lfsEndpointPins("https://u:"+gitRunToken+"@git.example.invalid/o/r"), " ") + for _, k := range []string{"lfs.url", "lfs.pushurl", "remote.origin.lfsurl", "remote.origin.lfspushurl"} { + if !strings.Contains(pins, k+"=https://u:"+gitRunToken+"@git.example.invalid/o/r.git/info/lfs") { + t.Errorf("%s is not pinned to the repository's own LFS address: %s", k, pins) + } + } + if p := lfsEndpointPins("/srv/x/origin.git"); p != nil { + t.Errorf("a path got pins: %v", p) + } +} + +// standInGit puts a git on PATH that fails on the subcommand fail and is the +// real git otherwise. +func standInGit(t *testing.T, fail string) { + t.Helper() + real, err := exec.LookPath("git") + if err != nil { + t.Skip("no git") + } + bin := t.TempDir() + script := "#!/bin/sh\nfor a; do [ \"$a\" = " + fail + " ] && { echo 'stand-in: " + fail + " failed' >&2; exit 128; }; done\nexec " + real + " \"$@\"\n" + if err := os.WriteFile(filepath.Join(bin, "git"), []byte(script), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH")) +} + +func branchPushed(f gitRunFixture) bool { + return exec.Command("git", "--git-dir", f.origin, "rev-parse", "--verify", "-q", "agent-work").Run() == nil +} + +// A commit that fails is the run failing: nothing is pushed. +func TestAGitRunWhoseCommitFailsPushesNothing(t *testing.T) { + f := newGitRunFixture(t) + f.setAgent(t, "cd \"$1\" && echo new > edit.txt\n") + standInGit(t, "commit") + res, code := f.run(t) + if code == 0 || res["ok"] != false { + t.Errorf("a run whose commit failed reads ok: code %d, %v", code, res) + } + if e, _ := res["error"].(string); !strings.Contains(e, "commit") || !strings.Contains(e, "nothing was pushed") { + t.Errorf("the error does not say the commit failed: %q", e) + } + if branchPushed(f) { + t.Error("the branch was pushed") + } +} + +// A run that changed nothing is ok, says so, and pushes no branch. +func TestAGitRunThatChangesNothingPushesNothing(t *testing.T) { + f := newGitRunFixture(t) + f.setAgent(t, "true\n") + res, code := f.run(t) + if code != 0 || res["ok"] != true || res["changed"] != false { + t.Errorf("want ok with changed false: code %d, %v", code, res) + } + if _, ok := res["branch"]; ok { + t.Errorf("the result names a branch: %v", res["branch"]) + } + if branchPushed(f) { + t.Error("a branch was pushed for a run that changed nothing") + } +} + +// Names a file system may open as .git are left out of the copy, and the +// rest of the run goes on. +func TestAGitRunLeavesOutEveryNameThatOpensAsDotGit(t *testing.T) { + f := newGitRunFixture(t) + names := []string{".git.", ".GIT ", "git~1", "GIT~1. ", ".g\u200cit", ".git::$INDEX_ALLOCATION"} + script := "cd \"$1\" && echo new > edit.txt\n" + for _, n := range names { + script += "mkdir '" + n + "' && echo '[core]' > '" + n + "/config'\n" + } + f.setAgent(t, script) + res, code := f.run(t) + if code != 0 || res["ok"] != true { + t.Fatalf("the run failed: code %d, %v", code, res) + } + files := testGit(t, f.origin, "ls-tree", "-r", "--name-only", "-z", "agent-work") + for _, n := range names { + if strings.Contains(files, n+"/") { + t.Errorf("%q is in the commit", n) + } + } + skipped, _ := res["worktree_skipped"].([]any) + listed := map[string]bool{} + for _, s := range skipped { + listed[fmt.Sprint(s)] = true + } + for _, n := range names { + if !listed[n] { + t.Errorf("%q is not listed as left out: %v", n, skipped) + } + } + for _, n := range []string{".git", ".Git", ".git.", ".git . .", "GiT~1", ".gi\ufefft", ".git:x"} { + if !dotGitName(n) { + t.Errorf("dotGitName(%q) = false", n) + } + } + for _, n := range []string{".gitignore", ".gitattributes", "git", ".github", "git~2", "x.git"} { + if dotGitName(n) { + t.Errorf("dotGitName(%q) = true", n) + } + } +} + +// With git-lfs installed, the bastion's own git calls clean and smudge +// through it and push its objects; the agent's .git cannot replace it, and a +// filter of another name the agent's .gitattributes names runs nothing. +func TestAGitRunUsesTheLFSFilterOnlyFromThisProcess(t *testing.T) { + f := newGitRunFixture(t) + seed := filepath.Join(t.TempDir(), "s") + testGit(t, "", "clone", "-q", f.origin, seed) + if err := os.WriteFile(filepath.Join(seed, ".gitattributes"), []byte("*.bin filter=lfs -text\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(seed, "big.bin"), []byte("v1\n"), 0o644); err != nil { + t.Fatal(err) + } + testGit(t, seed, "add", "-A") + testGit(t, seed, "commit", "-q", "-m", "lfs") + testGit(t, seed, "push", "-q", "origin", "main") + + bin := t.TempDir() + lfsLog := filepath.Join(t.TempDir(), "lfs.log") + wrapper := "#!/bin/sh\nGO_HELPER_GITLFS=1 GO_HELPER_GITLFS_LOG=" + lfsLog + " exec " + os.Args[0] + + " -test.run='^TestHelperGitLFS$' -- \"$@\"\n" + if err := os.WriteFile(filepath.Join(bin, "git-lfs"), []byte(wrapper), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH")) + + mark := func(name string) string { return "touch " + filepath.Join(f.marks, name) } + f.setAgent(t, `set -e +cd "$1" +git config filter.lfs.process '`+mark("agent-lfs-process")+`; exit 1' +git config filter.lfs.clean '`+mark("agent-lfs-clean")+`; cat' +git config filter.evil.clean '`+mark("evil")+`; cat' +printf '*.bin filter=lfs -text\n*.txt filter=evil\n' > .gitattributes +echo v2 > big.bin +echo new > edit.txt +`) + res, code := f.run(t) + if code != 0 || res["ok"] != true { + t.Fatalf("the run failed: code %d, %v", code, res) + } + ents, _ := os.ReadDir(f.marks) + for _, e := range ents { + t.Errorf("the agent's %s ran as this process", e.Name()) + } + logged, _ := os.ReadFile(lfsLog) + for _, want := range []string{"smudge big.bin", "clean big.bin", "push origin agent-work"} { + if !strings.Contains(string(logged), want+"\n") { + t.Errorf("git-lfs did not %s; it logged:\n%s", want, logged) + } + } + for _, line := range strings.Split(string(logged), "\n") { + if strings.HasSuffix(line, ".txt") { + t.Errorf("git-lfs ran for a file of another filter: %s", line) + } + } + if got := testGit(t, f.origin, "show", "agent-work:big.bin"); got != "v2\n" { + t.Errorf("big.bin = %q", got) + } +} + +// TestHelperGitLFS is the stand-in git-lfs: run by the wrapper above, it is +// an identity filter that logs what it was asked to do. +func TestHelperGitLFS(t *testing.T) { + if os.Getenv("GO_HELPER_GITLFS") != "1" { + return + } + args := os.Args + for i, a := range args { + if a == "--" { + args = args[i+1:] + break + } + } + logf, err := os.OpenFile(os.Getenv("GO_HELPER_GITLFS_LOG"), os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0o644) + if err != nil { + os.Exit(2) + } + switch { + case len(args) > 0 && args[0] == "filter-process": + if err := helperFilterProcess(logf); err != nil { + fmt.Fprintln(os.Stderr, "stand-in git-lfs:", err) + os.Exit(1) + } + case len(args) > 0 && (args[0] == "clean" || args[0] == "smudge"): + fmt.Fprintf(logf, "%s %s\n", args[0], args[len(args)-1]) + _, _ = io.Copy(os.Stdout, os.Stdin) + default: + fmt.Fprintln(logf, strings.Join(args, " ")) + } + logf.Close() + os.Exit(0) +} + +func pktRead(r *bufio.Reader) ([]byte, bool, error) { + var h [4]byte + if _, err := io.ReadFull(r, h[:]); err != nil { + return nil, false, err + } + n, err := strconv.ParseUint(string(h[:]), 16, 16) + if err != nil { + return nil, false, err + } + if n == 0 { + return nil, true, nil + } + b := make([]byte, n-4) + _, err = io.ReadFull(r, b) + return b, false, err +} + +func pktList(r *bufio.Reader) ([]string, error) { + var out []string + for { + b, flush, err := pktRead(r) + if err != nil || flush { + return out, err + } + out = append(out, strings.TrimSuffix(string(b), "\n")) + } +} + +func pktWrite(w io.Writer, lines ...string) { + for _, l := range lines { + fmt.Fprintf(w, "%04x%s", len(l)+4, l) + } + _, _ = w.Write([]byte("0000")) +} + +// helperFilterProcess speaks git's long-running filter protocol, version 2, +// returning every file as it came. +func helperFilterProcess(logf io.Writer) error { + r, w := bufio.NewReader(os.Stdin), bufio.NewWriter(os.Stdout) + if _, err := pktList(r); err != nil { + return err + } + pktWrite(w, "git-filter-server\n", "version=2\n") + if err := w.Flush(); err != nil { + return err + } + if _, err := pktList(r); err != nil { + return err + } + pktWrite(w, "capability=clean\n", "capability=smudge\n") + if err := w.Flush(); err != nil { + return err + } + for { + head, err := pktList(r) + if err == io.EOF { + return nil + } + if err != nil { + return err + } + cmd, path := "", "" + for _, kv := range head { + k, v, _ := strings.Cut(kv, "=") + switch k { + case "command": + cmd = v + case "pathname": + path = v + } + } + var body bytes.Buffer + for { + b, flush, err := pktRead(r) + if err != nil { + return err + } + if flush { + break + } + body.Write(b) + } + fmt.Fprintf(logf, "%s %s\n", cmd, path) + pktWrite(w, "status=success\n") + data := body.Bytes() + for len(data) > 0 { + n := min(len(data), 65516) + fmt.Fprintf(w, "%04x", n+4) + _, _ = w.Write(data[:n]) + data = data[n:] + } + _, _ = w.Write([]byte("0000")) + pktWrite(w) + if err := w.Flush(); err != nil { + return err + } + } +} diff --git a/client-go/harness_gitrun_linux_test.go b/client-go/harness_gitrun_linux_test.go new file mode 100644 index 0000000..8e53c8d --- /dev/null +++ b/client-go/harness_gitrun_linux_test.go @@ -0,0 +1,355 @@ +package main + +// A git run's tree is the agent's, .git included. These tests stand in for an +// agent that uses that .git against the bastion: hooks, a file system +// monitor, a filter, a signing program, a changed remote and a link to a file +// outside the tree, besides ordinary edits. The container runtime is a +// stand-in script that does the agent's work on the mounted tree; the sidecar +// it would start is a local HTTP server on the sidecar's port. + +import ( + "encoding/json" + "net" + "net/http" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "testing" + "time" +) + +const gitRunOutsideSecret = "outside-the-tree-7c41" + +// gitRunFixture is an origin with one commit, a decoy repository, a file +// outside the tree, a folder the agent's code marks when it runs, and a +// stand-in runtime that runs agent (a shell script, given the tree's path as +// $1) when the run starts its container. +type gitRunFixture struct { + origin, decoy, outside, marks, agentFile string +} + +// setAgent writes the script the stand-in runtime runs as the agent. +func (f gitRunFixture) setAgent(t *testing.T, script string) { + t.Helper() + if err := os.WriteFile(f.agentFile, []byte(script), 0o755); err != nil { + t.Fatal(err) + } +} + +func testGit(t *testing.T, dir string, args ...string) string { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %v: %v: %s", args, err, out) + } + return string(out) +} + +func newGitRunFixture(t *testing.T) gitRunFixture { + t.Helper() + if _, err := exec.LookPath("git"); err != nil { + t.Skip("no git") + } + tmp := t.TempDir() + // The test's own git, and git as the code before this change ran it, read + // no config of this machine's. + home := filepath.Join(tmp, "home") + if err := os.Mkdir(home, 0o700); err != nil { + t.Fatal(err) + } + t.Setenv("HOME", home) + t.Setenv("XDG_CONFIG_HOME", home) + t.Setenv("GIT_CONFIG_NOSYSTEM", "1") + t.Setenv("GIT_AUTHOR_NAME", "seed") + t.Setenv("GIT_AUTHOR_EMAIL", "seed@example.invalid") + t.Setenv("GIT_COMMITTER_NAME", "seed") + t.Setenv("GIT_COMMITTER_EMAIL", "seed@example.invalid") + f := gitRunFixture{ + origin: filepath.Join(tmp, "origin.git"), + decoy: filepath.Join(tmp, "decoy.git"), + outside: filepath.Join(tmp, "outside", "secret"), + marks: filepath.Join(tmp, "marks"), + } + for _, d := range []string{filepath.Dir(f.outside), f.marks} { + if err := os.Mkdir(d, 0o755); err != nil { + t.Fatal(err) + } + } + if err := os.WriteFile(f.outside, []byte(gitRunOutsideSecret), 0o600); err != nil { + t.Fatal(err) + } + testGit(t, tmp, "init", "-q", "--bare", "-b", "main", f.origin) + testGit(t, tmp, "init", "-q", "--bare", "-b", "main", f.decoy) + seed := filepath.Join(tmp, "seed") + testGit(t, tmp, "init", "-q", "-b", "main", seed) + for name, body := range map[string]string{"keep.txt": "keep\n", "edit.txt": "old\n", "gone.txt": "bye\n"} { + if err := os.WriteFile(filepath.Join(seed, name), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + if err := os.WriteFile(filepath.Join(seed, "tool.sh"), []byte("#!/bin/sh\n"), 0o755); err != nil { + t.Fatal(err) + } + testGit(t, seed, "add", "-A") + testGit(t, seed, "commit", "-q", "-m", "seed") + testGit(t, seed, "push", "-q", f.origin, "main") + + bin := t.TempDir() + f.agentFile = filepath.Join(bin, "agent.sh") + agentFile := f.agentFile + runtime := "#!/bin/sh\ncase \"$1\" in\nps|pull|rm) exit 0;;\ninspect) echo 127.0.0.1; exit 0;;\n" + + "run) for a; do case \"$a\" in *:/work) d=${a%:/work};; esac; done\n" + + " /bin/sh " + agentFile + " \"$d\" > " + filepath.Join(bin, "agent.log") + " 2>&1 || exit 1\n" + + " echo standin0123456789; exit 0;;\nesac\nexit 1\n" + if err := os.WriteFile(filepath.Join(bin, "docker"), []byte(runtime), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH")) + t.Cleanup(func() { + if t.Failed() { + if b, err := os.ReadFile(filepath.Join(bin, "agent.log")); err == nil { + t.Logf("agent log:\n%s", b) + } + } + }) + + // The sidecar the run drives: it answers, and /run reports success. + ln, err := net.Listen("tcp", "127.0.0.1:8844") + if err != nil { + t.Fatalf("the sidecar's port is taken: %v", err) + } + srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/run" { + _, _ = w.Write([]byte(`{"ok":true,"transcript":"done"}`)) + } + })} + go func() { _ = srv.Serve(ln) }() + t.Cleanup(func() { _ = srv.Close() }) + return f +} + +func (f gitRunFixture) run(t *testing.T) (map[string]any, int) { + t.Helper() + b, code := runGit(harnessSpec{Mode: "git", HarnessID: "h1", Image: "harness:test", + RepoURL: f.origin, Branch: "agent-work", MaxMinutes: 1}) + var res map[string]any + if err := json.Unmarshal(b, &res); err != nil { + t.Fatalf("result: %v: %s", err, b) + } + return res, code +} + +// theAgent turns its tree's .git against whoever runs git in it next, and +// makes ordinary changes to the files. +func theAgent(f gitRunFixture) string { + mark := func(name string) string { return "touch " + filepath.Join(f.marks, name) } + return `set -e +d="$1"; cd "$d" +g="$d/.git" +printf '#!/bin/sh\n` + mark("pre-commit") + `\n' > "$g/hooks/pre-commit" +printf '#!/bin/sh\n` + mark("post-commit") + `\n' > "$g/hooks/post-commit" +chmod +x "$g/hooks/pre-commit" "$g/hooks/post-commit" +printf '#!/bin/sh\n` + mark("fsmonitor") + `\nexit 1\n' > "$g/fsmon.sh"; chmod +x "$g/fsmon.sh" +git config core.fsmonitor "$g/fsmon.sh" +printf '* filter=evil\n' > .gitattributes +git config filter.evil.clean '` + mark("filter") + `; cat ` + f.outside + `' +git config filter.evil.smudge cat +printf '#!/bin/sh\n` + mark("gpg") + `\necho >&2; echo "[GNUPG:] SIG_CREATED D 1 8 00 0 X" >&2\necho "-----BEGIN PGP SIGNATURE-----"; echo x; echo "-----END PGP SIGNATURE-----"\n' > "$g/gpg.sh" +chmod +x "$g/gpg.sh" +git config commit.gpgsign true +git config gpg.program "$g/gpg.sh" +git config remote.origin.url ` + f.decoy + ` +ln -s ` + f.outside + ` link-to-outside +echo new > edit.txt +echo added > added.txt +rm gone.txt +mkdir sub; echo deep > sub/deep.txt +` +} + +func TestAGitRunNeverRunsGitInTheAgentsTree(t *testing.T) { + f := newGitRunFixture(t) + f.setAgent(t, theAgent(f)) + res, code := f.run(t) + if code != 0 || res["ok"] != true { + t.Errorf("the run failed: code %d, %v", code, res) + } + + // Nothing the agent set in its .git ran. + ents, _ := os.ReadDir(f.marks) + for _, e := range ents { + t.Errorf("the agent's %s ran as this process", e.Name()) + } + // The push went to the repository the run was given, not the one the + // agent named. + if refs := strings.TrimSpace(testGit(t, f.decoy, "for-each-ref")); refs != "" { + t.Errorf("the push went to the agent's remote: %s", refs) + } + out, err := exec.Command("git", "--git-dir", f.origin, "rev-parse", "--verify", "-q", "agent-work").Output() + if err != nil { + t.Fatalf("the branch is not in the origin: %v", err) + } + tip := strings.TrimSpace(string(out)) + if n := strings.TrimSpace(testGit(t, f.origin, "rev-list", "--count", "main.."+tip)); n != "1" { + t.Errorf("the branch carries %s commits on main, want 1", n) + } + // Exactly the agent's file changes: the link is not there, and the outside + // file's content is in no file of the commit. + files := strings.Fields(testGit(t, f.origin, "ls-tree", "-r", "--format=%(objectmode) %(path)", tip)) + got := map[string]string{} + for i := 0; i+1 < len(files); i += 2 { + got[files[i+1]] = files[i] + } + want := map[string]string{".gitattributes": "100644", "added.txt": "100644", "edit.txt": "100644", + "keep.txt": "100644", "sub/deep.txt": "100644", "tool.sh": "100755"} + if !sameMap(got, want) { + t.Errorf("the commit holds %v, want %v", got, want) + } + for path := range got { + body := testGit(t, f.origin, "show", tip+":"+path) + if strings.Contains(body, gitRunOutsideSecret) { + t.Errorf("%s in the commit holds the file outside the tree", path) + } + } + for path, body := range map[string]string{"edit.txt": "new\n", "added.txt": "added\n", "keep.txt": "keep\n"} { + if _, ok := got[path]; ok { + if b := testGit(t, f.origin, "show", tip+":"+path); b != body { + t.Errorf("%s = %q, want %q", path, b, body) + } + } + } + if s, _ := res["worktree_skipped"].([]any); len(s) != 1 || s[0] != "link-to-outside" { + t.Errorf("left out = %v, want [link-to-outside]", res["worktree_skipped"]) + } + for _, k := range []string{"branch", "commit", "commit_out"} { + if _, ok := res[k]; !ok { + t.Errorf("the result has no %s", k) + } + } + if res["commit"] != tip { + t.Errorf("the result names commit %v, the branch is at %s", res["commit"], tip) + } +} + +// A link the repository itself holds stays as it is when the agent leaves it +// alone: a run that cannot copy a link must not delete it. +func TestAGitRunKeepsALinkTheRepositoryHolds(t *testing.T) { + f := newGitRunFixture(t) + seed := filepath.Join(t.TempDir(), "s") + testGit(t, "", "clone", "-q", f.origin, seed) + if err := os.Symlink("keep.txt", filepath.Join(seed, "alias")); err != nil { + t.Fatal(err) + } + testGit(t, seed, "add", "-A") + testGit(t, seed, "commit", "-q", "-m", "a link") + testGit(t, seed, "push", "-q", "origin", "main") + f.setAgent(t, "cd \"$1\" && echo new > edit.txt\n") + res, code := f.run(t) + if code != 0 { + t.Fatalf("the run failed: %v", res) + } + if mode := strings.Fields(testGit(t, f.origin, "ls-tree", "agent-work", "alias")); len(mode) == 0 || mode[0] != "120000" { + t.Errorf("the repository's link is gone from the branch: %v", mode) + } + if diff := strings.Fields(testGit(t, f.origin, "diff", "--name-only", "main", "agent-work")); len(diff) != 1 || diff[0] != "edit.txt" { + t.Errorf("the branch changes %v, want [edit.txt]", diff) + } +} + +// A file far larger than the copy may take (sparse, so it costs nothing to +// make) is refused from its size, before a byte of it is read: the run ends +// at once, pushes nothing, and says why. +func TestAGitRunRefusesATreeLargerThanItCopies(t *testing.T) { + f := newGitRunFixture(t) + f.setAgent(t, "cd \"$1\" && truncate -s 64G huge.bin && echo new > edit.txt\n") + type out struct { + res map[string]any + code int + } + ch := make(chan out, 1) + start := time.Now() + go func() { r, c := f.run(t); ch <- out{r, c} }() + var o out + select { + case o = <-ch: + case <-time.After(30 * time.Second): + t.Fatal("a run with a 64 GiB file was still going after 30 s") + } + t.Logf("returned in %v", time.Since(start)) + if o.code == 0 || o.res["ok"] != false { + t.Fatalf("a tree over the cap was pushed: %v", o.res) + } + if e, _ := o.res["error"].(string); !strings.Contains(e, "huge.bin") || !strings.Contains(e, "too large") { + t.Errorf("the error does not say what was too large: %q", e) + } + if err := exec.Command("git", "--git-dir", f.origin, "rev-parse", "--verify", "-q", "agent-work").Run(); err == nil { + t.Error("the branch was pushed") + } +} + +// The tree a worktree run returns is refused the same way, and no part of +// it is returned. +func TestAWorktreeRunRefusesATreeLargerThanTheLaneCarries(t *testing.T) { + f := newGitRunFixture(t) + f.setAgent(t, "cd \"$1\" && truncate -s 64G huge.bin\n") + ch := make(chan []byte, 1) + start := time.Now() + go func() { + b, _ := runWorktree(harnessSpec{Mode: "worktree", HarnessID: "h1", Image: "harness:test", MaxMinutes: 1}) + ch <- b + }() + var b []byte + select { + case b = <-ch: + case <-time.After(30 * time.Second): + t.Fatal("a run with a 64 GiB file was still going after 30 s") + } + t.Logf("returned in %v", time.Since(start)) + var res map[string]any + if err := json.Unmarshal(b, &res); err != nil { + t.Fatal(err) + } + if res["ok"] != false || res["worktree_b64"] != nil { + t.Fatalf("a tree over the cap was returned: ok=%v, %d bytes of tree", res["ok"], len(b)) + } + if e, _ := res["error"].(string); !strings.Contains(e, "huge.bin") || !strings.Contains(e, "too large") { + t.Errorf("the error does not say what was too large: %q", e) + } +} + +func sameMap(a, b map[string]string) bool { + if len(a) != len(b) { + return false + } + ka := make([]string, 0, len(a)) + for k := range a { + ka = append(ka, k) + } + sort.Strings(ka) + for _, k := range ka { + if b[k] != a[k] { + return false + } + } + return true +} + +// A repository with no commit yet gets the run's commit as its first. +func TestAGitRunIntoAnEmptyRepositoryPushesItsFirstCommit(t *testing.T) { + f := newGitRunFixture(t) + empty := filepath.Join(t.TempDir(), "empty.git") + testGit(t, "", "init", "-q", "--bare", "-b", "main", empty) + f.setAgent(t, "cd \"$1\" && echo first > first.txt\n") + b, code := runGit(harnessSpec{Mode: "git", HarnessID: "h1", Image: "harness:test", + RepoURL: empty, Branch: "agent-work", MaxMinutes: 1}) + if code != 0 { + t.Fatalf("the run failed: %s", b) + } + if got := testGit(t, empty, "show", "agent-work:first.txt"); got != "first\n" { + t.Errorf("first.txt = %q", got) + } +} diff --git a/client-go/harness_gitrun_reallfs_linux_test.go b/client-go/harness_gitrun_reallfs_linux_test.go new file mode 100644 index 0000000..81c7941 --- /dev/null +++ b/client-go/harness_gitrun_reallfs_linux_test.go @@ -0,0 +1,143 @@ +package main + +// A git run against a repository that keeps a file in Git LFS, with the real +// git-lfs. It runs only where git-lfs is installed; every server it talks to +// is in this test, on loopback. + +import ( + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/cgi" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" +) + +func lfsPointer(body string) (oid, pointer string) { + sum := sha256.Sum256([]byte(body)) + oid = hex.EncodeToString(sum[:]) + return oid, fmt.Sprintf("version https://git-lfs.github.com/spec/v1\noid sha256:%s\nsize %d\n", oid, len(body)) +} + +func TestAGitRunWithTheRealGitLFS(t *testing.T) { + if out, err := exec.Command("git-lfs", "version").Output(); err != nil || !strings.HasPrefix(string(out), "git-lfs/") { + t.Skip("git-lfs is not installed") + } + f := newGitRunFixture(t) + realGit, _ := exec.LookPath("git") + want := "Basic " + base64.StdEncoding.EncodeToString([]byte("harness:"+gitRunToken)) + + var mu sync.Mutex + store := map[string][]byte{} + v1oid, v1ptr := lfsPointer("v1\n") + store[v1oid] = []byte("v1\n") + var srvURL string + backend := &cgi.Handler{Path: realGit, Args: []string{"http-backend"}, Root: "/git", + Env: []string{"GIT_PROJECT_ROOT=" + filepath.Dir(f.origin), "GIT_HTTP_EXPORT_ALL=1", + "REMOTE_USER=harness", "GIT_CONFIG_NOSYSTEM=1", "HOME=" + t.TempDir()}} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasPrefix(r.URL.Path, "/store/") { // the object hrefs the batch answer gives + oid := strings.TrimPrefix(r.URL.Path, "/store/") + mu.Lock() + defer mu.Unlock() + if r.Method == http.MethodPut { + b, _ := io.ReadAll(r.Body) + store[oid] = b + return + } + _, _ = w.Write(store[oid]) + return + } + if r.Header.Get("Authorization") != want { + w.Header().Set("WWW-Authenticate", `Basic realm="test"`) + http.Error(w, "auth", http.StatusUnauthorized) + return + } + if !strings.Contains(r.URL.Path, "/info/lfs/") { + backend.ServeHTTP(w, r) + return + } + if !strings.HasSuffix(r.URL.Path, "/objects/batch") { + http.NotFound(w, r) // locks: not supported here + return + } + var req struct { + Operation string `json:"operation"` + Objects []struct { + Oid string `json:"oid"` + Size int64 `json:"size"` + } `json:"objects"` + } + _ = json.NewDecoder(r.Body).Decode(&req) + var objs []map[string]any + mu.Lock() + for _, o := range req.Objects { + obj := map[string]any{"oid": o.Oid, "size": o.Size, "authenticated": true} + _, have := store[o.Oid] + if req.Operation == "download" || !have { + obj["actions"] = map[string]any{req.Operation: map[string]any{"href": srvURL + "/store/" + o.Oid}} + } + objs = append(objs, obj) + } + mu.Unlock() + w.Header().Set("Content-Type", "application/vnd.git-lfs+json") + _ = json.NewEncoder(w).Encode(map[string]any{"transfer": "basic", "objects": objs}) + })) + t.Cleanup(srv.Close) + srvURL = srv.URL + var foreignHits atomic.Int64 + foreign := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + foreignHits.Add(1) + http.Error(w, "no", http.StatusForbidden) + })) + t.Cleanup(foreign.Close) + + seed := filepath.Join(t.TempDir(), "s") + testGit(t, "", "clone", "-q", f.origin, seed) + for name, body := range map[string]string{".gitattributes": "*.bin filter=lfs diff=lfs merge=lfs -text\n", "big.bin": v1ptr} { + if err := os.WriteFile(filepath.Join(seed, name), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + testGit(t, seed, "add", "-A") + testGit(t, seed, "commit", "-q", "-m", "an lfs file") + testGit(t, seed, "push", "-q", "origin", "main") + + // The agent changes the large file and names a foreign LFS server. + f.setAgent(t, `set -e +cd "$1" +grep -qx v1 big.bin +echo v2 > big.bin +printf '[lfs]\n\turl = `+foreign.URL+`/x.git/info/lfs\n\tpushurl = `+foreign.URL+`/x.git/info/lfs\n[remote "origin"]\n\tlfsurl = `+foreign.URL+`/y\n' > .lfsconfig +`) + repoURL := strings.Replace(srv.URL, "http://", "http://harness:"+gitRunToken+"@", 1) + "/git/" + filepath.Base(f.origin) + b, code := runGit(harnessSpec{Mode: "git", HarnessID: "h1", Image: "harness:test", + RepoURL: repoURL, Branch: "agent-work", MaxMinutes: 1}) + t.Logf("result: %s", b) + if code != 0 { + t.Fatalf("the run failed") + } + v2oid, v2ptr := lfsPointer("v2\n") + if got := testGit(t, f.origin, "show", "agent-work:big.bin"); got != v2ptr { + t.Errorf("big.bin on the branch = %q, want the pointer to v2", got) + } + mu.Lock() + got := string(store[v2oid]) + mu.Unlock() + if got != "v2\n" { + t.Errorf("the LFS object for v2 was not sent to the repository's own LFS server (%q)", got) + } + if n := foreignHits.Load(); n != 0 { + t.Errorf("the foreign LFS server named in .lfsconfig got %d requests", n) + } +} diff --git a/client-go/harness_tar_linux.go b/client-go/harness_tar_linux.go new file mode 100644 index 0000000..c811c5b --- /dev/null +++ b/client-go/harness_tar_linux.go @@ -0,0 +1,43 @@ +package main + +import ( + "errors" + "strconv" + + "golang.org/x/sys/unix" +) + +// openTreeEntry opens the entry name in the folder dfd with flags, for +// packFolder. +// +// A folder is opened directly: with O_DIRECTORY the kernel refuses anything +// that is not a folder before it opens it. Anything else is first opened as a +// path only (O_PATH), which runs no driver code, so a device, pipe or terminal +// swapped in since the look is never opened: opening some devices does +// something by itself (a watchdog arms, a tape rewinds on close). Only a plain +// file is then opened for reading, through /proc/self/fd, which reaches the +// file the path fd holds and no other. +func openTreeEntry(dfd int, name string, flags int) (int, error) { + if flags&unix.O_DIRECTORY != 0 { + return unix.Openat(dfd, name, flags, 0) + } + pfd, err := unix.Openat(dfd, name, unix.O_PATH|unix.O_NOFOLLOW|unix.O_CLOEXEC, 0) + if err != nil { + return -1, err + } + defer unix.Close(pfd) + var st unix.Stat_t + if err := unix.Fstat(pfd, &st); err != nil { + return -1, err + } + if st.Mode&unix.S_IFMT != unix.S_IFREG { + return -1, errNotPlainFile + } + fd, err := unix.Open("/proc/self/fd/"+strconv.Itoa(pfd), flags&^unix.O_NOFOLLOW, 0) + if errors.Is(err, unix.ENOENT) { + // No /proc mounted: open by name as other systems do. packOpened + // still checks that what was opened is the file that was looked at. + return unix.Openat(dfd, name, flags, 0) + } + return fd, err +} diff --git a/client-go/harness_tar_linux_test.go b/client-go/harness_tar_linux_test.go new file mode 100644 index 0000000..1a8140a --- /dev/null +++ b/client-go/harness_tar_linux_test.go @@ -0,0 +1,70 @@ +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "golang.org/x/sys/unix" +) + +// A name swapped for a special file between the walk's look and its open must +// never be opened: opening a device runs its driver (a watchdog arms, a tape +// rewinds on close, a serial line raises DTR), whatever is read after. A named +// pipe stands in for any of them, since anyone can make one, and inotify +// reports every open of it except one made only as a path (O_PATH), which runs +// no driver code. Both a name looked at as a plain file and one looked at as +// a folder are swapped. +func TestMakeTarNeverOpensASpecialFileSwappedInBeforeTheOpen(t *testing.T) { + for _, swap := range []string{"a.txt", "sub"} { + t.Run(swap, func(t *testing.T) { + _, _, tree := outsideAndTree(t) + in, err := unix.InotifyInit1(unix.IN_NONBLOCK | unix.IN_CLOEXEC) + if err != nil { + t.Skipf("no inotify: %v", err) + } + defer unix.Close(in) + watched := false + packTreeBeforeOpen = func(rel string) { + if rel != swap { + return + } + p := filepath.Join(tree, swap) + if err := os.RemoveAll(p); err != nil { + t.Errorf("remove %s: %v", p, err) + return + } + if err := unix.Mkfifo(p, 0o644); err != nil { + t.Errorf("mkfifo %s: %v", p, err) + return + } + if _, err := unix.InotifyAddWatch(in, p, unix.IN_OPEN); err != nil { + t.Errorf("watch %s: %v", p, err) + return + } + watched = true + } + defer func() { packTreeBeforeOpen = nil }() + b, skipped, err := packWorktree(tree) + if err != nil { + t.Fatalf("packWorktree: %v", err) + } + if !watched { + t.Fatal("the swap never ran") + } + buf := make([]byte, 4096) + if n, err := unix.Read(in, buf); n > 0 { + t.Fatalf("the special file swapped in for %s was opened (%d bytes of inotify events)", swap, n) + } else if err != unix.EAGAIN { + t.Fatalf("read inotify: %d %v", n, err) + } + if _, ok := tarEntries(t, b)[swap]; ok { + t.Fatalf("%s was a named pipe when it was opened and must be left out", swap) + } + if !strings.Contains(","+strings.Join(skipped, ",")+",", ","+swap+",") { + t.Fatalf("%s must be named as left out, got %v", swap, skipped) + } + }) + } +} diff --git a/client-go/harness_tar_other.go b/client-go/harness_tar_other.go new file mode 100644 index 0000000..671e9e3 --- /dev/null +++ b/client-go/harness_tar_other.go @@ -0,0 +1,13 @@ +//go:build !windows && !linux + +package main + +import "golang.org/x/sys/unix" + +// openTreeEntry opens the entry name in the folder dfd with flags, for +// packFolder. These systems have no open that runs no driver code, so a +// special file swapped in since the look is opened (O_NONBLOCK, O_NOCTTY), +// then closed unread by packOpened. +func openTreeEntry(dfd int, name string, flags int) (int, error) { + return unix.Openat(dfd, name, flags, 0) +} diff --git a/client-go/harness_tar_unix.go b/client-go/harness_tar_unix.go new file mode 100644 index 0000000..8eb9fdb --- /dev/null +++ b/client-go/harness_tar_unix.go @@ -0,0 +1,114 @@ +//go:build !windows + +package main + +import ( + "errors" + "fmt" + "os" + + "golang.org/x/sys/unix" +) + +// errNotPlainFile is openTreeEntry's refusal of a name that is no longer a +// plain file when it is opened. +var errNotPlainFile = errors.New("not a plain file") + +// packTree hands the tree under root to sink, walking it by open folder +// (packWorktree says why). Each name is looked at with fstatat and opened +// relative to the folder above it (openTreeEntry, per system), never +// following a link, and it goes in only if the opened file is still the plain +// file or folder that was looked at (same type, device and inode). A pipe or +// device seen as one is never opened, so the walk cannot wait on one; on +// Linux one swapped in between is not opened either, and elsewhere O_NONBLOCK +// and O_NOCTTY cover it. Each left-out entry goes to sink.skip. +func packTree(sink treeSink, root string) error { + fd, err := unix.Open(root, unix.O_RDONLY|unix.O_DIRECTORY|unix.O_NOFOLLOW|unix.O_CLOEXEC, 0) + if err != nil { + return &os.PathError{Op: "open", Path: root, Err: err} + } + d := os.NewFile(uintptr(fd), root) + defer d.Close() + return packFolder(sink, d, fd, "", 0) +} + +func packFolder(sink treeSink, d *os.File, dfd int, rel string, depth int) error { + names, err := readTreeNames(sink, d, rel) + if err != nil { + return err + } + if err := sink.folder(rel, names); err != nil { + return err + } + for _, name := range names { + p := name + if rel != "" { + p = rel + "/" + name + } + var st unix.Stat_t + if err := unix.Fstatat(dfd, name, &st, unix.AT_SYMLINK_NOFOLLOW); err != nil { + sink.skip(p) // gone since the folder was read + continue + } + kind := st.Mode & unix.S_IFMT + if kind == unix.S_IFDIR && name == ".git" { + continue + } + if (kind != unix.S_IFREG && kind != unix.S_IFDIR) || (kind == unix.S_IFDIR && depth >= maxTreeDepth) { + sink.skip(p) + continue + } + if packTreeBeforeOpen != nil { + packTreeBeforeOpen(p) + } + flags := unix.O_RDONLY | unix.O_NOFOLLOW | unix.O_CLOEXEC | unix.O_NONBLOCK | unix.O_NOCTTY + if kind == unix.S_IFDIR { + flags |= unix.O_DIRECTORY + } + cfd, err := openTreeEntry(dfd, name, flags) + if err != nil { + sink.skip(p) // a link or special file now, or gone + continue + } + err = packOpened(sink, cfd, name, st, p, depth) + if err != nil { + return err + } + } + return nil +} + +// packOpened hands sink the entry opened as fd, and for a folder what is in it, +// if it is still the entry fstatat saw; it closes fd. +func packOpened(sink treeSink, fd int, name string, seen unix.Stat_t, p string, depth int) error { + var got unix.Stat_t + if err := unix.Fstat(fd, &got); err != nil { + unix.Close(fd) + return fmt.Errorf("stat %q: %w", p, err) + } + kind := got.Mode & unix.S_IFMT + if kind != seen.Mode&unix.S_IFMT || got.Dev != seen.Dev || got.Ino != seen.Ino { + unix.Close(fd) + sink.skip(p) // swapped between the look and the open + return nil + } + // A plain file or folder now: back to blocking reads, so os.File reads it + // as it reads any file it opened itself. + if err := unix.SetNonblock(fd, false); err != nil { + unix.Close(fd) + return fmt.Errorf("open %q: %w", p, err) + } + f := os.NewFile(uintptr(fd), name) + defer f.Close() + fi, err := f.Stat() + if err != nil { + return fmt.Errorf("stat %q: %w", p, err) + } + if err := sink.put(f, fi, p); err != nil { + return err + } + if kind == unix.S_IFDIR { + return packFolder(sink, f, fd, p, depth+1) + } + return nil +} diff --git a/client-go/harness_tar_unix_test.go b/client-go/harness_tar_unix_test.go new file mode 100644 index 0000000..7920f49 --- /dev/null +++ b/client-go/harness_tar_unix_test.go @@ -0,0 +1,204 @@ +//go:build !windows + +package main + +import ( + "archive/tar" + "bytes" + "io" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "golang.org/x/sys/unix" +) + +const tarOutsideSecret = "outside-the-tree-3f9a" + +// tarInsideA is as long as the outside file: a writer who knows a file's size +// can make the one it swaps out the same size, so the tar's header fits. +const tarInsideA = "inside-the-tree-a-000" + +// tarEntries reads back what makeTar produced: name -> content. +func tarEntries(t *testing.T, b []byte) map[string]string { + t.Helper() + out := map[string]string{} + tr := tar.NewReader(bytes.NewReader(b)) + for { + hdr, err := tr.Next() + if err == io.EOF { + return out + } + if err != nil { + t.Fatalf("read tar: %v", err) + } + data, err := io.ReadAll(tr) + if err != nil { + t.Fatalf("read %s: %v", hdr.Name, err) + } + out[hdr.Name] = string(data) + } +} + +// outsideAndTree makes a file outside the tree that a run must never return, +// and a tree with one file and one folder. +func outsideAndTree(t *testing.T) (outsideDir, secret, tree string) { + t.Helper() + outsideDir = t.TempDir() + secret = filepath.Join(outsideDir, "secret") + if err := os.WriteFile(secret, []byte(tarOutsideSecret), 0o600); err != nil { + t.Fatal(err) + } + tree = t.TempDir() + if err := os.WriteFile(filepath.Join(tree, "a.txt"), []byte(tarInsideA), 0o644); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(filepath.Join(tree, "sub"), 0o755); err != nil { + t.Fatal(err) + } + // Named as the outside file is: a writer who knows the names in a folder + // outside can fill a folder of the tree with the same names, then swap it. + if err := os.WriteFile(filepath.Join(tree, "sub", "secret"), []byte("inside b"), 0o644); err != nil { + t.Fatal(err) + } + return outsideDir, secret, tree +} + +func assertNoOutside(t *testing.T, ents map[string]string) { + t.Helper() + for name, data := range ents { + if strings.Contains(data, tarOutsideSecret) { + t.Fatalf("the tar holds a file from outside the tree, as %q", name) + } + } +} + +// The tree's writer swaps a file for a link, and a folder for a link, after +// the walk has seen each one as a plain file or folder and before it opens it. +// The hook stands exactly in that gap, so the swap lands there every time. +func TestMakeTarNeverReadsThroughALinkSwappedInBeforeTheOpen(t *testing.T) { + for _, swap := range []string{"a.txt", "sub"} { + t.Run(swap, func(t *testing.T) { + outsideDir, secret, tree := outsideAndTree(t) + packTreeBeforeOpen = func(rel string) { + if rel != swap { + return + } + p := filepath.Join(tree, swap) + _ = os.RemoveAll(p) + if swap == "sub" { + _ = os.Symlink(outsideDir, p) + } else { + _ = os.Symlink(secret, p) + } + } + defer func() { packTreeBeforeOpen = nil }() + b, err := makeTar(tree) + if err != nil { + t.Fatalf("makeTar: %v", err) + } + ents := tarEntries(t, b) + assertNoOutside(t, ents) + if swap == "a.txt" { + if _, ok := ents["a.txt"]; ok { + t.Fatalf("a.txt was a link when it was opened and must be left out, got it in the tar") + } + } + }) + } +} + +// No hook: a writer flips one name between a plain file and a link to a file +// outside the tree as fast as it can while the tree is packed again and again. +func TestMakeTarNeverReadsThroughALinkUnderAConcurrentWriter(t *testing.T) { + _, secret, tree := outsideAndTree(t) + name := filepath.Join(tree, "a.txt") + tmpLink, tmpFile := filepath.Join(tree, ".l"), filepath.Join(tree, ".f") + stop := make(chan struct{}) + done := make(chan struct{}) + go func() { + defer close(done) + for { + select { + case <-stop: + return + default: + } + _ = os.WriteFile(tmpFile, []byte(tarInsideA), 0o644) + _ = os.Rename(tmpFile, name) + _ = os.Symlink(secret, tmpLink) + _ = os.Rename(tmpLink, name) + } + }() + defer func() { close(stop); <-done }() + deadline := time.Now().Add(3 * time.Second) + passes := 0 + defer func() { t.Logf("%d passes", passes) }() + for i := 0; time.Now().Before(deadline); i++ { + passes = i + 1 + b, err := makeTar(tree) + if err != nil { + continue // a name that vanished mid-walk; the next pass tries again + } + for n, data := range tarEntries(t, b) { + if strings.Contains(data, tarOutsideSecret) { + t.Fatalf("pass %d: the tar holds the file outside the tree, as %q", i, n) + } + } + } +} + +// A link, a named pipe and a socket in the tree are left out and named as +// left out; the pipe is never opened, so the walk does not wait on it. +func TestMakeTarLeavesOutAndNamesWhatIsNotAPlainFileOrFolder(t *testing.T) { + _, secret, tree := outsideAndTree(t) + if err := os.Symlink(secret, filepath.Join(tree, "link")); err != nil { + t.Fatal(err) + } + if err := unix.Mkfifo(filepath.Join(tree, "sub", "pipe"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(filepath.Join(tree, ".git"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(tree, ".git", "config"), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + type res struct { + b []byte + skipped []string + err error + } + ch := make(chan res, 1) + go func() { b, s, err := packWorktree(tree); ch <- res{b, s, err} }() + var r res + select { + case r = <-ch: + case <-time.After(10 * time.Second): + t.Fatal("packing the tree waited on the named pipe") + } + if r.err != nil { + t.Fatalf("packWorktree: %v", r.err) + } + ents := tarEntries(t, r.b) + assertNoOutside(t, ents) + for _, want := range []string{"a.txt", "sub", "sub/secret"} { + if _, ok := ents[want]; !ok { + t.Errorf("%s missing from the tar (got %v)", want, ents) + } + } + if ents["a.txt"] != tarInsideA || ents["sub/secret"] != "inside b" { + t.Errorf("file contents changed in the tar: %v", ents) + } + for _, gone := range []string{"link", "sub/pipe", ".git", ".git/config"} { + if _, ok := ents[gone]; ok { + t.Errorf("%s is in the tar and must not be", gone) + } + } + got := strings.Join(r.skipped, ",") + if got != "link,sub/pipe" { + t.Errorf("left out = %q, want link,sub/pipe", got) + } +} diff --git a/client-go/harness_tar_windows.go b/client-go/harness_tar_windows.go new file mode 100644 index 0000000..2321b99 --- /dev/null +++ b/client-go/harness_tar_windows.go @@ -0,0 +1,83 @@ +//go:build windows + +package main + +import ( + "fmt" + "os" + "path/filepath" +) + +// packTree hands the tree under root to sink (packWorktree says why it is +// walked this way). Every name is opened through an os.Root at the tree, so +// no link, junction or name swapped in while it is packed can open a file +// outside the tree; an entry goes in only if the opened file is the plain +// file or folder that was looked at. Each left-out entry goes to sink.skip. +func packTree(sink treeSink, root string) error { + r, err := os.OpenRoot(root) + if err != nil { + return err + } + defer r.Close() + return packRootFolder(sink, r, "", 0) +} + +func packRootFolder(sink treeSink, r *os.Root, rel string, depth int) error { + open := rel + if open == "" { + open = "." + } + d, err := r.Open(open) + if err != nil { + return fmt.Errorf("open folder %q: %w", rel, err) + } + names, err := readTreeNames(sink, d, filepath.ToSlash(rel)) + d.Close() + if err != nil { + return err + } + if err := sink.folder(filepath.ToSlash(rel), names); err != nil { + return err + } + for _, name := range names { + p := filepath.Join(rel, name) + tp := filepath.ToSlash(p) + seen, err := r.Lstat(p) + if err != nil { + sink.skip(tp) + continue + } + if seen.IsDir() && name == ".git" { + continue + } + if (!seen.Mode().IsRegular() && !seen.IsDir()) || (seen.IsDir() && depth >= maxTreeDepth) { + sink.skip(tp) + continue + } + if packTreeBeforeOpen != nil { + packTreeBeforeOpen(tp) + } + f, err := r.Open(p) + if err != nil { + sink.skip(tp) + continue + } + fi, err := f.Stat() + if err != nil || fi.Mode().Type() != seen.Mode().Type() { + f.Close() + sink.skip(tp) + continue + } + err = sink.put(f, fi, tp) + f.Close() + if err != nil { + return err + } + if fi.IsDir() { + if err := packRootFolder(sink, r, p, depth+1); err != nil { + return err + } + } + } + return nil +} diff --git a/client-go/harness_tree_linux_test.go b/client-go/harness_tree_linux_test.go new file mode 100644 index 0000000..44d1ba5 --- /dev/null +++ b/client-go/harness_tree_linux_test.go @@ -0,0 +1,125 @@ +package main + +// A harness run's tree is bind-mounted into the container at /work and opened +// to every user, because the agent runs as a uid of its own. What keeps the +// other users of this machine out of it is the folder it sits in, which only +// this process can enter. Opening the tree follows no link in it. + +import ( + "archive/tar" + "bytes" + "encoding/base64" + "os" + "path/filepath" + "strconv" + "strings" + "testing" +) + +// harnessStandIn writes a stand-in container runtime named name into a new +// folder. On `ps` and `pull` it succeeds; on `run` it records, into the +// returned file, the mode and owner of the folder the /work tree sits in, then +// fails, so the run stops there. +func harnessStandIn(t *testing.T, name string) (bin, record string) { + t.Helper() + bin = t.TempDir() + record = filepath.Join(bin, "record") + script := "#!/bin/sh\ncase \"$1\" in ps|pull) exit 0;; run) ;; *) exit 1;; esac\n" + + "for a; do case \"$a\" in *:/work) d=${a%:/work};; esac; done\n" + + "[ -n \"$d\" ] && stat -c '%a %u' \"$(dirname \"$d\")\" > " + record + "\nexit 1\n" + if err := os.WriteFile(filepath.Join(bin, name), []byte(script), 0o755); err != nil { + t.Fatal(err) + } + return bin, record +} + +func TestAWorktreeRunsTreeSitsInAFolderOnlyTheBastionCanEnter(t *testing.T) { + bin, record := harnessStandIn(t, "docker") + t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH")) + // The temporary folder every user may write in, as /tmp is. + shared := t.TempDir() + if err := os.Chmod(shared, os.ModeSticky|0o777); err != nil { + t.Fatal(err) + } + t.Setenv("TMPDIR", shared) + + var tb bytes.Buffer + tw := tar.NewWriter(&tb) + _ = tw.WriteHeader(&tar.Header{Name: "src/", Typeflag: tar.TypeDir, Mode: 0o755}) + _ = tw.WriteHeader(&tar.Header{Name: "src/a.txt", Typeflag: tar.TypeReg, Mode: 0o644, Size: 2}) + _, _ = tw.Write([]byte("a\n")) + _ = tw.Close() + spec := harnessSpec{Mode: "worktree", HarnessID: "h1", Image: "harness:test", + WorktreeB64: base64.StdEncoding.EncodeToString(tb.Bytes())} + if _, code := runWorktree(spec); code == 0 { + t.Fatal("the stand-in runtime fails the start; the run must report it") + } + got, err := os.ReadFile(record) + if err != nil { + t.Fatalf("the tree was not mounted: %v", err) + } + if want := "700 " + strconv.Itoa(os.Geteuid()); strings.TrimSpace(string(got)) != want { + t.Fatalf("the folder the tree sits in: want %q, got %q", want, strings.TrimSpace(string(got))) + } +} + +func TestOpeningARunsTreeFollowsNoLinkAndKeepsWhatIsExecutable(t *testing.T) { + bin, _ := harnessStandIn(t, "runtime") + rt := filepath.Join(bin, "runtime") + outside := filepath.Join(t.TempDir(), "shadow") + if err := os.WriteFile(outside, []byte("x\n"), 0o600); err != nil { + t.Fatal(err) + } + tree := t.TempDir() + files := map[string]os.FileMode{"run.sh": 0o755, "notes.txt": 0o644} + for name, mode := range files { + if err := os.WriteFile(filepath.Join(tree, name), nil, mode); err != nil { + t.Fatal(err) + } + } + if err := os.Mkdir(filepath.Join(tree, "sub"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, filepath.Join(tree, "sub", "link")); err != nil { + t.Fatal(err) + } + if _, err := driveHarnessAgent(rt, tree, harnessSpec{HarnessID: "h1", Image: "harness:test"}, 1); err == nil { + t.Fatal("the stand-in runtime fails the start; the run must report it") + } + for p, want := range map[string]os.FileMode{ + tree: 0o777, filepath.Join(tree, "sub"): 0o777, + filepath.Join(tree, "run.sh"): 0o777, filepath.Join(tree, "notes.txt"): 0o666, + outside: 0o600, + } { + fi, err := os.Stat(p) + if err != nil { + t.Fatal(err) + } + if fi.Mode().Perm() != want { + t.Errorf("%s: want %04o, got %04o", p, want, fi.Mode().Perm()) + } + } +} + +func TestARunsTreeInAFolderOtherUsersCanEnterIsRefused(t *testing.T) { + bin, record := harnessStandIn(t, "runtime") + rt := filepath.Join(bin, "runtime") + open := t.TempDir() + if err := os.Chmod(open, 0o755); err != nil { + t.Fatal(err) + } + tree := filepath.Join(open, "tree") + if err := os.Mkdir(tree, 0o755); err != nil { + t.Fatal(err) + } + _, err := driveHarnessAgent(rt, tree, harnessSpec{HarnessID: "h1", Image: "harness:test"}, 1) + if err == nil || !strings.Contains(err.Error(), "open to other users") { + t.Fatalf("want the tree refused for its folder, got %v", err) + } + if _, err := os.Stat(record); err == nil { + t.Fatal("a refused tree must start no container") + } + if fi, _ := os.Stat(tree); fi.Mode().Perm() != 0o755 { + t.Fatalf("a refused tree must not be opened: %04o", fi.Mode().Perm()) + } +} diff --git a/client-go/harness_treebounds_unix_test.go b/client-go/harness_treebounds_unix_test.go new file mode 100644 index 0000000..914b7ee --- /dev/null +++ b/client-go/harness_treebounds_unix_test.go @@ -0,0 +1,119 @@ +//go:build !windows + +package main + +import ( + "errors" + "os" + "path/filepath" + "runtime" + "strconv" + "strings" + "testing" +) + +// A copy of a tree of very many empty files stops at its limit, and what it +// holds in memory does not grow with the number of files. +func TestACopyOfVeryManyEmptyFilesIsRefusedWithBoundedMemory(t *testing.T) { + src, dst := t.TempDir(), t.TempDir() + for i := 0; i < 50; i++ { + d := filepath.Join(src, "d"+strconv.Itoa(i)) + if err := os.Mkdir(d, 0o755); err != nil { + t.Fatal(err) + } + for j := 0; j < 400; j++ { + if err := os.WriteFile(filepath.Join(d, "f"+strconv.Itoa(j)), nil, 0o644); err != nil { + t.Fatal(err) + } + } + } + var before, after runtime.MemStats + runtime.GC() + runtime.ReadMemStats(&before) + _, err := mirrorTree(src, dst, 1<<20) + runtime.ReadMemStats(&after) + if !errors.Is(err, errTreeTooLarge) { + t.Fatalf("20000 empty files under a 1 MiB limit: %v", err) + } + got := after.TotalAlloc - before.TotalAlloc + t.Logf("refused: %v; %d KiB allocated", err, got>>10) + if got > 16<<20 { + t.Errorf("the copy allocated %d MiB", got>>20) + } + n := 0 + _ = filepath.WalkDir(dst, func(string, os.DirEntry, error) error { n++; return nil }) + if n > (1<<20)/512+1 { + t.Errorf("%d entries were written under a limit of %d", n, (1<<20)/512) + } +} + +// Left-out entries are counted in full, and only the first +// maxSkippedReported names are kept. +func TestLeftOutEntriesAreCountedAndTheirNamesBounded(t *testing.T) { + src, dst := t.TempDir(), t.TempDir() + for i := 0; i < 3*maxSkippedReported; i++ { + if err := os.Symlink("x", filepath.Join(src, "l"+strconv.Itoa(i))); err != nil { + t.Fatal(err) + } + } + skips, err := mirrorTree(src, dst, maxCopiedTreeBytes) + if err != nil { + t.Fatal(err) + } + if skips.count != 3*maxSkippedReported || len(skips.names) != maxSkippedReported { + t.Errorf("count %d, %d names kept", skips.count, len(skips.names)) + } + res := map[string]interface{}{} + skips.report(res) + if res["worktree_skipped_count"] != 3*maxSkippedReported { + t.Errorf("reported %v", res) + } +} + +func TestRepoGitAuthPutsTheTokenOnlyInTheEnvironment(t *testing.T) { + bare, env, err := repoGitAuth("https://host-user:tok3n0f7hisrun5a1e@git.example.invalid/o/r.git") + if err != nil || bare != "https://git.example.invalid/o/r.git" { + t.Fatalf("bare = %q, %v", bare, err) + } + joined := strings.Join(env, "\n") + if !strings.Contains(joined, "GIT_CONFIG_KEY_0=http.https://git.example.invalid/o/r.git.extraHeader") || + !strings.Contains(joined, "GIT_CONFIG_VALUE_0=Authorization: Basic ") { + t.Errorf("env = %q", env) + } + if _, env, _ := repoGitAuth("/srv/x/origin.git"); env != nil { + t.Errorf("a path got credentials: %v", env) + } +} + +// One folder holding very many names is refused once the names read pass +// the limit, without the whole folder being read first. +func TestAFolderOfVeryManyNamesIsRefusedWithoutReadingItWhole(t *testing.T) { + src := t.TempDir() + long := strings.Repeat("n", 190) + for i := 0; i < 30000; i++ { + if err := os.WriteFile(filepath.Join(src, long+strconv.Itoa(i)), nil, 0o644); err != nil { + t.Fatal(err) + } + } + saved := maxReturnedTreeBytes + maxReturnedTreeBytes = 1 << 20 + t.Cleanup(func() { maxReturnedTreeBytes = saved }) + for name, walk := range map[string]func() error{ + "copy": func() error { _, err := mirrorTree(src, t.TempDir(), 1<<20); return err }, + "tar": func() error { _, _, err := packWorktree(src); return err }, + } { + var before, after runtime.MemStats + runtime.GC() + runtime.ReadMemStats(&before) + err := walk() + runtime.ReadMemStats(&after) + got := after.TotalAlloc - before.TotalAlloc + t.Logf("%s: %v; %d KiB allocated", name, err, got>>10) + if !errors.Is(err, errTreeTooLarge) { + t.Errorf("%s: a folder of 30000 names under a 1 MiB limit: %v", name, err) + } + if got > 4<<20 { + t.Errorf("%s: %d KiB allocated; the 30000 names alone are about 6 MiB", name, got>>10) + } + } +} diff --git a/client-go/harness_treecap_unix_test.go b/client-go/harness_treecap_unix_test.go new file mode 100644 index 0000000..66aa0c4 --- /dev/null +++ b/client-go/harness_treecap_unix_test.go @@ -0,0 +1,71 @@ +//go:build !windows + +package main + +import ( + "os" + "path/filepath" + "runtime" + "strings" + "testing" + "time" +) + +// A file far larger than the tree a run may return (sparse, so it costs +// nothing to make) is refused from its size: packing returns at once, with +// an error and no tar, having read none of it. +func TestPackingRefusesAFileLargerThanTheLaneCarriesWithoutReadingIt(t *testing.T) { + tree := t.TempDir() + if err := os.WriteFile(filepath.Join(tree, "a.txt"), []byte("a\n"), 0o644); err != nil { + t.Fatal(err) + } + f, err := os.Create(filepath.Join(tree, "huge.bin")) + if err != nil { + t.Fatal(err) + } + if err := f.Truncate(64 << 30); err != nil { + t.Skipf("no sparse file here: %v", err) + } + f.Close() + type res struct { + b []byte + err error + } + var before, after runtime.MemStats + runtime.ReadMemStats(&before) + ch := make(chan res, 1) + start := time.Now() + go func() { b, _, err := packWorktree(tree); ch <- res{b, err} }() + var r res + select { + case r = <-ch: + case <-time.After(20 * time.Second): + t.Fatal("packing a tree with a 64 GiB file was still going after 20 s") + } + runtime.ReadMemStats(&after) + t.Logf("returned in %v, %d MiB allocated", time.Since(start), (after.TotalAlloc-before.TotalAlloc)>>20) + if r.err == nil || !strings.Contains(r.err.Error(), "huge.bin") || !strings.Contains(r.err.Error(), "too large") { + t.Fatalf("want the tree refused for huge.bin, got %v", r.err) + } + if r.b != nil { + t.Fatal("a tar came back with the refusal") + } + if got := after.TotalAlloc - before.TotalAlloc; got > 64<<20 { + t.Fatalf("packing allocated %d MiB", got>>20) + } +} + +// A result larger than one message on the lane is replaced by a refusal that +// says so, never sent to be dropped by the host; one that fits goes as it is. +func TestAResultTheLaneCannotCarryIsRefusedNotSent(t *testing.T) { + small := []byte(`{"ok":true,"mode":"worktree"}`) + if b, code := fitLane(small, 0); string(b) != string(small) || code != 0 { + t.Fatalf("a result that fits was changed: %s %d", b, code) + } + big := append([]byte(`{"ok":true,"mode":"worktree","worktree_b64":"`), make([]byte, uvicornWSMaxSize)...) + b, code := fitLane(big, 0) + if code == 0 || len(b) > 1024 || !strings.Contains(string(b), `"result_too_large":true`) || + !strings.Contains(string(b), `"ok":false`) { + t.Fatalf("an oversized result was not refused: %d bytes, code %d: %.200s", len(b), code, b) + } +} diff --git a/client-go/hostops_test.go b/client-go/hostops_test.go index e63dc9b..7f67bbe 100644 --- a/client-go/hostops_test.go +++ b/client-go/hostops_test.go @@ -484,7 +484,7 @@ func TestRepeatedRefusedWritesQuarantineThePairing(t *testing.T) { for i := 0; i < quarantineStrikes; i++ { driveJob(t, tAnchor, putJob(""), ctx, "") } - if u := quarantinedUntil(tAnchor, time.Now()); u.IsZero() || u.Sub(time.Now()) > quarantineBase+time.Minute { + if u := quarantinedUntil(tAnchor, time.Now()); u.IsZero() || time.Until(u) > quarantineBase+time.Minute { t.Fatalf("a cleared hold is forgiven — the next one is the base length again: %v", u) } // Strikes double when the hold lapses on its own. @@ -494,7 +494,7 @@ func TestRepeatedRefusedWritesQuarantineThePairing(t *testing.T) { for i := 0; i < quarantineStrikes; i++ { driveJob(t, tAnchor, putJob(""), ctx, "") } - if u := quarantinedUntil(tAnchor, time.Now()); u.IsZero() || u.Sub(time.Now()) < 2*quarantineBase-time.Minute { + if u := quarantinedUntil(tAnchor, time.Now()); u.IsZero() || time.Until(u) < 2*quarantineBase-time.Minute { t.Fatalf("a repeat hold doubles: %v", u) } } @@ -1010,12 +1010,15 @@ func TestAnchoredEgressRefusesLoopbackAndPrivateDestinations(t *testing.T) { if addr, why := egressVettedAddr(ctx, "localhost:80"); why == "" || addr != "" { t.Fatalf("a refused name yields no address: %q %q", addr, why) } + // A local box refuses this machine too, and keeps its LAN + // (egress_local_test.go has the rest). t.Setenv("XDG_CONFIG_HOME", t.TempDir()) - if why := egressInternalRefusal(ctx, "127.0.0.1:11434"); why != "" { - t.Fatalf("a local box keeps its lane: %s", why) + stubOwnAddrs(t, loopbackOnly, localRange()) + if why := egressInternalRefusal(ctx, "127.0.0.1:11434"); why == "" { + t.Fatal("a local box must refuse its own ollama on the egress lane") } - if addr, _ := egressVettedAddr(ctx, "example.com:443"); addr != "example.com:443" { - t.Fatalf("a local box dials the name it was given: %q", addr) + if addr, why := egressVettedAddr(ctx, "192.168.1.1:80"); why != "" || addr != "192.168.1.1:80" { + t.Fatalf("a local box keeps its LAN: %q %q", addr, why) } } diff --git a/client-go/killtree.go b/client-go/killtree.go index 333b5ca..97b8ae4 100644 --- a/client-go/killtree.go +++ b/client-go/killtree.go @@ -90,7 +90,7 @@ var procIdentity = osProcIdentity // walked. The tree kill used to test only the root, so stopping a finding // took everything under it — a replaced sshd took every ssh session with it, // accounting entries included, which is exactly what the guards on the root -// exist to refuse (review of #100). Killing a parent does not kill its +// exist to refuse. Killing a parent does not kill its // children, so a spared subtree simply survives, re-parented. // // A PID IS NOT A PROCESS. Each pid's identity is read before it is stopped and @@ -111,7 +111,7 @@ func killTree(root int, spare func(int) bool) int { return killTreeExpect(root, // killTreeExpect is killTree for a root whose identity was read when it was // chosen (the finding walk): if the pid no longer carries that identity, the // number has changed hands since and nothing is signalled. That closes the gap -// between choosing the root and the first read inside the kill (review of #100). +// between choosing the root and the first read inside the kill. func killTreeExpect(root int, spare func(int) bool, rootID string) int { if spare(root) { return 0 diff --git a/client-go/killtree_test.go b/client-go/killtree_test.go index ca8b718..ec16501 100644 --- a/client-go/killtree_test.go +++ b/client-go/killtree_test.go @@ -208,7 +208,7 @@ func identityRig(t *testing.T, script map[int][]string, stopped map[int]bool) { } } -// A PID IS NOT A PROCESS (review of #100). 202's number changes hands between +// A PID IS NOT A PROCESS. 202's number changes hands between // the read and the stop: the stranger is resumed and never killed. 203 dies // while frozen and its number is reused before the kill: not killed, and not // resumed either (the frozen process is gone; the new one was never stopped). @@ -239,7 +239,7 @@ func TestKillTreeNeverSignalsAStrangerUnderARecycledPid(t *testing.T) { } // A PROCESS SOMEONE ELSE HAD STOPPED is killed with the tree but never resumed -// by us when the kill fails (review of #100, N4): its stopped state was not +// by us when the kill fails: its stopped state was not // ours to undo. // // MUTATION: resume on kill failure regardless of wasStopped. RESULT: red. @@ -311,7 +311,7 @@ func TestRecoverFrozenTreesThawsOnlyWhatItFroze(t *testing.T) { } } -// THE FROZEN SET IS JOURNALLED AS IT IS FROZEN (review of #100): a bastion that +// THE FROZEN SET IS JOURNALLED AS IT IS FROZEN: a bastion that // dies during the freeze rounds still leaves a record of what it stopped. // // MUTATION: write the journal only after the freeze rounds. RESULT: red — the diff --git a/client-go/lab.go b/client-go/lab.go deleted file mode 100644 index 61c3ef6..0000000 --- a/client-go/lab.go +++ /dev/null @@ -1,849 +0,0 @@ -package main - -// lab.go — mode=lab: the Lab Harness. -// -// A lab run builds a sealed lab of N hardened hosts on THIS box's own k3s cluster (the -// containment lab), puts the application under test into every host, places ONE coding -// harness pod inside the lab and drives the test runbook through it, collects the -// transcript + per-step verdicts + the runbook worktree, and tears everything down. -// -// THREE STRUCTURAL WALLS (the product claim "spun up in the lab, nowhere else"): -// 1. This mode only answers on a LAB bastion: the lab marker, the inert egress wall file -// and the lab operator on loopback must all be present (labAvailable). A user's home -// bastion returns exit 126 in the same shape as an ungranted op. -// 2. The host-side grant carries a `mode:lab` token, so a grant for a home -// box never covers a lab run and a lab grant never covers a home box. -// 3. The lab cluster has no path out (cloud firewall + labguard per run + tripwire). -// -// What runs where: the lab HOSTS do not think — they hold and run the app under test. The -// coding harness pod thinks, with the per-run credential the host delivered, and its -// tools reach the lab hosts in-cluster. Nothing here talks to the platform. -// -// Gate: only gate="auto" is supported today (typed deliberately, exactly like the old bench's -// --auto). gate="ask" is the App lane (per-call y/n from outside the lab) and is not built -// yet; until then it is refused, never silently downgraded to auto. - -import ( - "bytes" - "crypto/rand" - "encoding/base64" - "encoding/hex" - "encoding/json" - "fmt" - "io" - "net/http" - "os" - "path/filepath" - "sort" - "strconv" - "strings" - "syscall" - "time" -) - -const ( - labMarkerPath = "/etc/privatae-lab/marker" - labGuardPath = "/etc/nftables-labguard.nft" - labOperatorURL = "http://127.0.0.1:8011/healthz" - labStorageRoot = "/var/lib/rancher/k3s/storage" - labWorkspace = "/var/lib/skema/projects" // the harness image's workspace root (same image, same root) - labHarnessPort = 8844 - labHostIDFloor = 9001 // lab host entity ids start here; 900 is the bench's own test entity - labHostIDCeil = 9899 - labTemplateEntity = 900 // the lab's bench entity; its registry row is the template for lab hosts - labDefaultHosts = 1 - labMaxHostsCap = 8 // per-box cap — protects the LAB BOX, never a user's workflow; marker max_hosts= raises it -) - -// labMarker holds the key=value lines of /etc/privatae-lab/marker. Written by the lab -// rebuild, never by this binary. -type labMarker struct { - present bool - hostImage string // host_image= — the host image lab hosts run (must already be on the node) - maxHosts int // max_hosts= — optional -} - -func readLabMarker() labMarker { - m := labMarker{} - b, err := os.ReadFile(labMarkerPath) - if err != nil { - return m - } - m.present = true - for _, line := range strings.Split(string(b), "\n") { - k, v, ok := strings.Cut(strings.TrimSpace(line), "=") - if !ok { - continue - } - switch strings.TrimSpace(k) { - case "host_image": - m.hostImage = strings.TrimSpace(v) - case "max_hosts": - m.maxHosts, _ = strconv.Atoi(strings.TrimSpace(v)) - } - } - return m -} - -// labAvailable is wall #1. Every check is a thing only the lab box has. -func labAvailable() (labMarker, error) { - m := readLabMarker() - if !m.present { - return m, fmt.Errorf("no lab marker") - } - if _, err := os.Stat(labGuardPath); err != nil { - return m, fmt.Errorf("no egress wall file") - } - c := &http.Client{Timeout: 3 * time.Second} - resp, err := c.Get(labOperatorURL) - if err != nil { - return m, fmt.Errorf("lab operator not on loopback") - } - resp.Body.Close() - if resp.StatusCode != 200 { - return m, fmt.Errorf("lab operator unhealthy (%d)", resp.StatusCode) - } - if _, err := runCmd(10*time.Second, "k3s", "kubectl", "version", "--client=true"); err != nil { - return m, fmt.Errorf("no cluster tooling") - } - if m.hostImage == "" { - return m, fmt.Errorf("marker has no host_image=") - } - return m, nil -} - -// kc runs `k3s kubectl …` with a timeout. -func kc(timeout time.Duration, args ...string) (string, error) { - return runCmd(timeout, "k3s", append([]string{"kubectl"}, args...)...) -} - -// kcApply applies a manifest from a temp file (runCmd has no stdin). -func kcApply(manifest string) (string, error) { - f, err := os.CreateTemp("", "lab-manifest-*.yaml") - if err != nil { - return "", err - } - defer os.Remove(f.Name()) - if _, err := f.WriteString(manifest); err != nil { - f.Close() - return "", err - } - f.Close() - return kc(60*time.Second, "apply", "-f", f.Name()) -} - -func labFail(res map[string]interface{}, code int, msg string) int { - res["ok"], res["error"] = false, msg - return code -} - -// runLab is the mode=lab entry point (harness.go runHarness). The result is marshalled -// AFTER the inner run's deferred teardown has recorded its log, so the caller always sees -// what happened including the cleanup. -func runLab(spec harnessSpec) ([]byte, int) { - res := map[string]interface{}{"mode": "lab", "ran_on": orUnknown(hostnameSafe())} - code := runLabInner(spec, res) - b, _ := json.Marshal(res) - return b, code -} - -func runLabInner(spec harnessSpec, res map[string]interface{}) int { - marker, err := labAvailable() - if err != nil { - // Same shape + exit as an ungranted op on the host side (exit 126). - return labFail(res, 126, "mode 'lab' not available on this server — "+err.Error()) - } - // ── validate the spec ───────────────────────────────────────────────────── - if spec.Gate != "auto" { - return labFail(res, 1, "gate='ask' (per-call y/n from the App) is not available yet — pass gate='auto', typed deliberately") - } - if spec.Image == "" { - return labFail(res, 1, "image required for mode=lab (the coding harness that drives the tests)") - } - if len(spec.Tests) == 0 { - return labFail(res, 1, "tests required for mode=lab (the runbook: ordered steps)") - } - hosts := spec.Hosts - if hosts <= 0 { - hosts = labDefaultHosts - } - maxHosts := labMaxHostsCap - if marker.maxHosts > 0 { - maxHosts = marker.maxHosts - } - if hosts > maxHosts { - return labFail(res, 1, fmt.Sprintf("hosts=%d exceeds this lab box's max_hosts=%d", hosts, maxHosts)) - } - app := spec.App - if app == "" { - app = "app" - } - if !validWSName(app) { - return labFail(res, 1, "app must be a plain directory name") - } - engine := spec.Engine - if engine == "" { - engine = spec.HarnessID - } - maxMin := spec.MaxMinutes - if maxMin <= 0 { - maxMin = 10 - } - var rb [4]byte - _, _ = rand.Read(rb[:]) - runID := hex.EncodeToString(rb[:]) - res["run_id"] = runID - start := time.Now() - var log []string - note := func(f string, a ...interface{}) { - log = append(log, time.Now().UTC().Format("15:04:05 ")+fmt.Sprintf(f, a...)) - } - - // ── 0. one run at a time, and reap whatever a previous run left ────────── - // The teardown below is a defer, so it covers every ordinary exit — but NOT a - // SIGKILL or a power loss (measured: a killed self-test left its two - // entity_registry rows behind). "The lab keeps nothing" has to hold after a crash - // too, so a run starts by removing anything in the lab's own id range / name - // prefixes. The lock is what makes that safe: without it a second concurrent run - // would reap the first one's hosts. - unlock, err := labLock() - if err != nil { - return labFail(res, 1, err.Error()) - } - defer unlock() - if reaped := labReapOrphans(); reaped != "" { - note("reaped a previous run's leftovers: %s", reaped) - res["reaped"] = reaped - } - - // ── 1. lab hosts: N hardened Entity CRs on the lab cluster ──────────────── - ids, err := labAllocateIDs(hosts) - if err != nil { - return labFail(res, 1, "allocate host ids: "+err.Error()) - } - res["hosts"] = ids - // ALWAYS tear down, whatever happens below — the lab keeps nothing between runs. - defer func() { - note("teardown: %d hosts + namespace lab-%s", len(ids), runID) - labTeardown(runID, ids) - res["log"] = strings.Join(log, "\n") - }() - for i, id := range ids { - name := fmt.Sprintf("lab-%s-%d", runID, i+1) - if err := labRegistryRow(id, name); err != nil { - return labFail(res, 1, fmt.Sprintf("host %d registry row: %v", id, err)) - } - manifest := fmt.Sprintf(`apiVersion: privatae.ai/v1 -kind: Entity -metadata: - name: entity-%d - namespace: default - labels: - privatae.ai/lab-run: "%s" -spec: - entityId: %d - lifecycleTier: always_on - image: %s - hardened: true -`, id, runID, id, marker.hostImage) - if out, err := kcApply(manifest); err != nil { - return labFail(res, 1, fmt.Sprintf("host %d apply: %s", id, truncate(out, 200))) - } - note("host %d (%s) applied", id, name) - } - for _, id := range ids { - // The operator builds the namespace + StatefulSet asynchronously after the CR lands; - // `kubectl wait` on a pod that does not exist yet fails at once, so poll for the pod - // first, then wait for Ready. - ns, pod := fmt.Sprintf("entity-%d", id), fmt.Sprintf("pod/entity-%d-0", id) - appeared := false - for i := 0; i < 60; i++ { - if _, err := kc(15*time.Second, "-n", ns, "get", pod); err == nil { - appeared = true - break - } - time.Sleep(3 * time.Second) - } - if !appeared { - ev, _ := kc(20*time.Second, "-n", ns, "get", "events", "--sort-by=.lastTimestamp") - return labFail(res, 1, fmt.Sprintf("host %d: the operator never created its pod (3 min): %s", id, truncate(ev, 300))) - } - if out, err := kc(300*time.Second, "-n", ns, "wait", "--for=condition=Ready", pod, "--timeout=270s"); err != nil { - ev, _ := kc(20*time.Second, "-n", ns, "get", "events", "--sort-by=.lastTimestamp") - return labFail(res, 1, fmt.Sprintf("host %d never became Ready: %s | %s", id, truncate(out, 160), truncate(ev, 300))) - } - } - hostIPs := map[int]string{} - for _, id := range ids { - ip, _ := kc(20*time.Second, "-n", fmt.Sprintf("entity-%d", id), "get", "pod", - fmt.Sprintf("entity-%d-0", id), "-o", "jsonpath={.status.podIP}") - hostIPs[id] = strings.TrimSpace(ip) - } - res["host_ips"] = hostIPs - note("%d hosts Ready", len(ids)) - - // ── 2. the app under test + its settings into every host ───────────────── - if spec.WorktreeB64 != "" { - tarIn, err := base64.StdEncoding.DecodeString(spec.WorktreeB64) - if err != nil { - return labFail(res, 1, "bad worktree_b64: "+err.Error()) - } - for _, id := range ids { - dir, err := labHostProjectDir(id, app) - if err != nil { - return labFail(res, 1, fmt.Sprintf("host %d projects dir: %v", id, err)) - } - if err := extractTar(tarIn, dir); err != nil { - return labFail(res, 1, fmt.Sprintf("host %d app extract: %v", id, err)) - } - if err := labWriteEnv(dir, spec.Envs); err != nil { - return labFail(res, 1, fmt.Sprintf("host %d envs: %v", id, err)) - } - _ = filepath.Walk(dir, func(p string, _ os.FileInfo, werr error) error { - if werr == nil { - _ = os.Chown(p, 1000, 1000) // the entity uid — the host owns its own project - } - return nil - }) - } - note("app %q + %d settings placed in every host", app, len(spec.Envs)) - } - - // ── 3. the ONE harness pod inside the lab (same image + env as the pool) ── - ns := "lab-" + runID - if out, err := kc(30*time.Second, "create", "namespace", ns); err != nil { - return labFail(res, 1, "namespace: "+truncate(out, 200)) - } - if spec.AgentCreds != "" { - f, err := os.CreateTemp("", "lab-creds-*") - if err != nil { - return labFail(res, 1, "creds temp: "+err.Error()) - } - _ = os.WriteFile(f.Name(), []byte(spec.AgentCreds), 0o600) - f.Close() - out, err := kc(30*time.Second, "-n", ns, "create", "secret", "generic", "harness-creds", - "--from-file=.credentials.json="+f.Name()) - _ = os.Remove(f.Name()) // per-run: nothing durable on the box - if err != nil { - return labFail(res, 1, "creds secret: "+truncate(out, 200)) - } - } - if out, err := kcApply(labHarnessPodManifest(ns, spec.Image, engine, spec.AgentCreds != "")); err != nil { - return labFail(res, 1, "harness pod: "+truncate(out, 200)) - } - if out, err := kc(240*time.Second, "-n", ns, "wait", "--for=condition=Ready", "pod/harness", "--timeout=210s"); err != nil { - ev, _ := kc(20*time.Second, "-n", ns, "get", "events", "--sort-by=.lastTimestamp") - return labFail(res, 1, "harness pod never Ready: "+truncate(out, 160)+" | "+truncate(ev, 400)) - } - hip, _ := kc(20*time.Second, "-n", ns, "get", "pod", "harness", "-o", "jsonpath={.status.podIP}") - hip = strings.TrimSpace(hip) - if hip == "" { - return labFail(res, 1, "harness pod has no IP") - } - base := fmt.Sprintf("http://[%s]:%d", hip, labHarnessPort) - if !strings.Contains(hip, ":") { - base = fmt.Sprintf("http://%s:%d", hip, labHarnessPort) - } - if ok, _ := waitReachable(base+"/", 60*time.Second); !ok { - return labFail(res, 1, "harness sidecar not reachable on :"+strconv.Itoa(labHarnessPort)) - } - note("harness pod %s Ready at %s", engine, hip) - - // ── 4. the runbook: one governed /run per step ──────────────────────────── - // The workspace the harness works in carries the runbook + a hosts file, so the - // agent knows what it is testing and where each lab host answers. - wsName := "lab-run" - wsTar, err := labRunbookTar(app, spec.Tests, ids, hostIPs, spec.Envs) - if err != nil { - return labFail(res, 1, "runbook tar: "+err.Error()) - } - if code, body := labPost(base+"/workspace/load?dir="+wsName, "application/x-tar", wsTar, 60*time.Second); code != 200 { - return labFail(res, 1, fmt.Sprintf("workspace/load %d: %s", code, truncate(body, 200))) - } - projectDir := labWorkspace + "/" + wsName - client := &http.Client{Timeout: time.Duration(maxMin+2) * time.Minute} - steps := make([]map[string]interface{}, 0, len(spec.Tests)) - allOK := true - for i, step := range spec.Tests { - prompt := labStepPrompt(i+1, len(spec.Tests), step, app, ids, hostIPs) - reqBody, _ := json.Marshal(map[string]interface{}{ - "prompt": prompt, "project_dir": projectDir, "max_minutes": maxMin, - }) - t0 := time.Now() - resp, err := client.Post(base+"/run", "application/json", bytes.NewReader(reqBody)) - st := map[string]interface{}{"step": i + 1, "test": step} - if err != nil { - st["ok"], st["error"] = false, "harness /run: "+err.Error() - steps = append(steps, st) - allOK = false - note("step %d: /run failed: %v", i+1, err) - break - } - rbody, _ := io.ReadAll(resp.Body) - resp.Body.Close() - var r map[string]interface{} - _ = json.Unmarshal(rbody, &r) - okv, _ := r["ok"].(bool) - st["ok"] = okv - st["exit_code"] = r["exit_code"] - st["elapsed_s"] = round2(time.Since(t0).Seconds()) - if tr, _ := r["transcript"].(string); tr != "" { - st["transcript"] = tr - } - if e, _ := r["error"].(string); e != "" { - st["error"] = e - } - steps = append(steps, st) - if !okv { - allOK = false - } - note("step %d: ok=%v (%.0fs)", i+1, okv, time.Since(t0).Seconds()) - } - res["steps"] = steps - - // ── 5. collect: the runbook worktree after (the agent's notes/results live there) ── - if code, body := labGet(base+"/workspace/export?dir="+wsName, 60*time.Second); code == 200 { - res["worktree_b64"] = base64.StdEncoding.EncodeToString([]byte(body)) - } - res["meter_s"] = round2(time.Since(start).Seconds()) - res["ok"] = allOK - if allOK { - return 0 - } - return 1 -} - -// ── helpers ────────────────────────────────────────────────────────────────── - -func validWSName(s string) bool { - if s == "" || len(s) > 64 || strings.ContainsAny(s, "/\\ ") || strings.HasPrefix(s, ".") { - return false - } - return true -} - -// labDeleteRows removes the lab's OWN entity_registry rows and reports how many. -// Three independent scopes, all required, so this can never reach a real entity: -// the `lab-` display-name prefix, the lab id range, and (for a run's own teardown) -// that run's id list. runID=="" + ids==nil is the orphan reap: every lab row in the range. -// Returns "" when it removed nothing. -func labDeleteRows(runID string, ids []int) string { - where := fmt.Sprintf("display_name LIKE 'lab-%s%%' AND id BETWEEN %d AND %d", - runID+map[bool]string{true: "-", false: ""}[runID != ""], labHostIDFloor, labHostIDCeil) - if len(ids) > 0 { - var in []string - for _, id := range ids { - in = append(in, strconv.Itoa(id)) // ints: nothing quotable reaches the SQL - } - where += " AND id IN (" + strings.Join(in, ",") + ")" - } else if runID != "" { - return "" // a run's teardown with no ids has nothing of its own to remove - } - sql := "DELETE FROM entity_registry WHERE " + where + ";" - f, err := os.CreateTemp("", "lab-del-*.sql") - if err != nil { - return "" - } - defer os.Remove(f.Name()) - _ = os.Chmod(f.Name(), 0o644) // the postgres user reads it - _, _ = f.WriteString(sql) - f.Close() - out, err := runCmd(20*time.Second, "sudo", "-u", "postgres", "psql", "-d", "parallax", "-q", - "-v", "ON_ERROR_STOP=1", "-c", "\\set QUIET on", "-f", f.Name()) - if err != nil { - return "" - } - out = strings.TrimSpace(out) - if out == "" || strings.HasSuffix(out, "DELETE 0") { - return "" - } - return out -} - -// labLock serialises lab runs on this box. Returns the release func. An ordinary exit -// releases it; a killed run leaves a stale lock, so a lock whose pid is gone is taken over -// (and the takeover is what makes the reap below safe rather than racy). -func labLock() (func(), error) { - const path = "/var/tmp/lab-run.lock" - take := func() error { - f, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) - if err != nil { - return err - } - fmt.Fprintf(f, "%d\n", os.Getpid()) - return f.Close() - } - if err := take(); err != nil { - if b, rerr := os.ReadFile(path); rerr == nil { - if pid, perr := strconv.Atoi(strings.TrimSpace(string(b))); perr == nil { - if p, ferr := os.FindProcess(pid); ferr != nil || p.Signal(syscall.Signal(0)) != nil { - _ = os.Remove(path) // stale: the holder is gone - if err2 := take(); err2 == nil { - return func() { _ = os.Remove(path) }, nil - } - } else { - return nil, fmt.Errorf("another lab run is in flight on this box (pid %d) — labs are one at a time", pid) - } - } - } - return nil, fmt.Errorf("could not take the lab run lock: %v", err) - } - return func() { _ = os.Remove(path) }, nil -} - -// labReapOrphans removes anything a PREVIOUS lab run left behind. Scoped three ways so it -// can only ever touch lab leavings: the `lab-` namespace prefix, the lab entity id range -// (the bench's own entity 900 is deliberately below it), and the `lab-` display-name prefix. -// Returns a short description of what it removed, or "". -func labReapOrphans() string { - var did []string - if out, err := kc(30*time.Second, "get", "ns", "-o", "name"); err == nil { - for _, l := range strings.Split(out, "\n") { - ns := strings.TrimPrefix(strings.TrimSpace(l), "namespace/") - if strings.HasPrefix(ns, "lab-") { - _, _ = kc(60*time.Second, "delete", "namespace", ns, "--ignore-not-found", "--wait=false") - did = append(did, "ns "+ns) - } - } - } - var ids []int - if out, err := kc(30*time.Second, "get", "entity", "-A", - "-o", "jsonpath={range .items[*]}{.spec.entityId}{\"\\n\"}{end}"); err == nil { - for _, l := range strings.Split(out, "\n") { - if v, e := strconv.Atoi(strings.TrimSpace(l)); e == nil && v >= labHostIDFloor && v <= labHostIDCeil { - ids = append(ids, v) - } - } - } - if len(ids) > 0 { - did = append(did, fmt.Sprintf("%d host(s) %v", len(ids), ids)) - } - // Volumes + registry rows are cleaned for the id range even when no CR remains - // (a crash can leave either half). - labTeardown("", ids) - orphans, _ := filepath.Glob(fmt.Sprintf("%s/pvc-*_entity-9[0-9][0-9][0-9]_*", labStorageRoot)) - for _, p := range orphans { - _ = os.RemoveAll(p) - did = append(did, "volume "+filepath.Base(p)) - } - if n := labDeleteRows("", nil); n != "" { - did = append(did, "registry rows "+n) - } - if len(did) == 0 { - return "" - } - return strings.Join(did, ", ") -} - -// labAllocateIDs picks N free entity ids in the lab range by reading the Entity CRs that exist. -func labAllocateIDs(n int) ([]int, error) { - out, err := kc(30*time.Second, "get", "entity", "-A", "-o", "jsonpath={range .items[*]}{.spec.entityId}{\"\\n\"}{end}") - if err != nil { - return nil, fmt.Errorf("list entities: %s", truncate(out, 160)) - } - used := map[int]bool{} - for _, l := range strings.Split(out, "\n") { - if v, e := strconv.Atoi(strings.TrimSpace(l)); e == nil { - used[v] = true - } - } - ids := []int{} - for id := labHostIDFloor; id <= labHostIDCeil && len(ids) < n; id++ { - if !used[id] { - ids = append(ids, id) - } - } - if len(ids) < n { - return nil, fmt.Errorf("only %d free lab host ids", len(ids)) - } - return ids, nil -} - -// labRegistryRow gives the lab operator the registry row it resolves names from -// (the operator reads display_name off entity_registry). Lab-local postgres. entity_type is -// CHECK-constrained to parallax_native|operator, so lab rows are marked by their display_name -// prefix "lab--" (and confined to the lab id range) — teardown deletes exactly those. -func labRegistryRow(id int, name string) error { - // Clone the lab's TEMPLATE entity row (entity 900, the bench's own test entity, restored - // with the lab DB) rather than invent values: the table carries CHECKs and FKs - // (mastercode_version → mastercode_versions) that a hand-written row keeps tripping. - sql := fmt.Sprintf(`INSERT INTO entity_registry (id, entity_type, display_name, master_directive, license_tier, - seed_weights, current_weights, envelope_bounds, mastercode_version, status) -SELECT %d, entity_type, '%s', 'lab host under test', 'standard', - seed_weights, current_weights, envelope_bounds, mastercode_version, 'active' -FROM entity_registry WHERE id = %d -ON CONFLICT (id) DO NOTHING;`, id, name, labTemplateEntity) - f, err := os.CreateTemp("", "lab-row-*.sql") - if err != nil { - return err - } - defer os.Remove(f.Name()) - _ = os.Chmod(f.Name(), 0o644) - if _, err := f.WriteString(sql); err != nil { - f.Close() - return err - } - f.Close() - if out, err := runCmd(20*time.Second, "sudo", "-u", "postgres", "psql", "-d", "parallax", "-v", "ON_ERROR_STOP=1", "-q", "-f", f.Name()); err != nil { - return fmt.Errorf("%s", truncate(out, 200)) - } - return nil -} - -// labHostProjectDir resolves the host's entity-apps PVC on the node's local-path storage and -// returns /projects/, created. The node is root over its own storage — the lab box's -// privilege, and exactly why this mode is lab-only. -func labHostProjectDir(id int, app string) (string, error) { - matches, _ := filepath.Glob(fmt.Sprintf("%s/pvc-*_entity-%d_entity-apps-entity-%d-0", labStorageRoot, id, id)) - if len(matches) != 1 { - return "", fmt.Errorf("expected one entity-apps volume, found %d", len(matches)) - } - dir := filepath.Join(matches[0], "projects", app) - if err := os.MkdirAll(dir, 0o755); err != nil { - return "", err - } - return dir, nil -} - -// labWriteEnv writes the app settings as /.env (KEY=value, sorted, values single-quoted). -func labWriteEnv(dir string, envs map[string]string) error { - if len(envs) == 0 { - return nil - } - keys := make([]string, 0, len(envs)) - for k := range envs { - keys = append(keys, k) - } - sort.Strings(keys) - var b strings.Builder - for _, k := range keys { - if !validEnvKey(k) { - return fmt.Errorf("bad env key %q", k) - } - b.WriteString(k + "='" + strings.ReplaceAll(envs[k], "'", "'\\''") + "'\n") - } - return os.WriteFile(filepath.Join(dir, ".env"), []byte(b.String()), 0o640) -} - -func validEnvKey(k string) bool { - if k == "" || len(k) > 128 { - return false - } - for i, r := range k { - ok := r == '_' || (r >= 'A' && r <= 'Z') || (r >= 'a' && r <= 'z') || (i > 0 && r >= '0' && r <= '9') - if !ok { - return false - } - } - return true -} - -// labHarnessPodManifest mirrors the pool's harness pod (same image, same env, same floor: -// non-root uid, drop ALL, no privilege escalation, RO rootfs, the personal seccomp profile) -// with emptyDirs where the harness must write. -func labHarnessPodManifest(ns, image, engine string, withCreds bool) string { - credsVol, credsMount, credsInit := "", "", "" - if withCreds { - credsVol = ` - name: creds-src - secret: - secretName: harness-creds -` - credsMount = "" - credsInit = ` initContainers: - - name: creds - image: busybox:1.36 - command: ["sh", "-c", "cp /src/.credentials.json /dst/.credentials.json && chown 10005:10005 /dst/.credentials.json && chmod 0644 /dst/.credentials.json"] - securityContext: - runAsUser: 0 - allowPrivilegeEscalation: false - capabilities: {drop: ["ALL"], add: ["CHOWN", "FOWNER", "DAC_OVERRIDE"]} - volumeMounts: - - {name: creds-src, mountPath: /src, readOnly: true} - - {name: home-claude, mountPath: /dst} -` - } - return fmt.Sprintf(`apiVersion: v1 -kind: Pod -metadata: - name: harness - namespace: %s - labels: {app: lab-harness, privatae.ai/lab-run: "%s"} -spec: - restartPolicy: Never - automountServiceAccountToken: false - securityContext: - seccompProfile: {type: Localhost, localhostProfile: seccomp-personal.json} -%s containers: - - name: harness - image: %s - imagePullPolicy: IfNotPresent - env: - - {name: PORT, value: "%d"} - - {name: PRIVATAE_SIDECAR_BIND, value: "::"} - - {name: PRIVATAE_APP_ID, value: "%s"} - - {name: PRIVATAE_WORKSPACE, value: "%s"} - # Both spellings, on purpose: the harness image is published on its own - # schedule, so a lab run must work against one that reads either name. - # The older three come out once no published image reads them. - - {name: SKEMA_SIDECAR_BIND, value: "::"} - - {name: SKEMA_APP_ID, value: "%s"} - - {name: SKEMA_WORKSPACE, value: "%s"} - - {name: HOME, value: "/home/privatae"} - securityContext: - runAsUser: 10005 - runAsGroup: 10005 - runAsNonRoot: true - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - capabilities: {drop: ["ALL"]} - resources: - requests: {cpu: 250m, memory: 512Mi} - limits: {memory: 2Gi} - volumeMounts: - - {name: workspace, mountPath: %s} - - {name: home-claude, mountPath: /home/privatae/.claude} - - {name: tmp, mountPath: /tmp} -%s volumes: - - {name: workspace, emptyDir: {}} - - {name: home-claude, emptyDir: {}} - - {name: tmp, emptyDir: {}} -%s`, ns, strings.TrimPrefix(ns, "lab-"), credsInit, image, labHarnessPort, - engine, labWorkspace, engine, labWorkspace, - labWorkspace, credsMount, credsVol) -} - -// labRunbookTar packs the runbook the harness works from: a README naming the app, the hosts -// and their addresses, the settings, and the ordered tests; plus a results/ dir for the agent. -func labRunbookTar(app string, tests []string, ids []int, ips map[int]string, envs map[string]string) ([]byte, error) { - dir, err := os.MkdirTemp("", "lab-runbook-") - if err != nil { - return nil, err - } - defer os.RemoveAll(dir) - var b strings.Builder - b.WriteString("# Lab run\n\nYou are the test harness inside a sealed lab. Nothing here can reach the internet or the platform.\n\n") - b.WriteString("## Application under test\n\n`" + app + "` — placed in every lab host under " + labWorkspace + "/" + app + "\n\n") - b.WriteString("## Lab hosts\n\n| host | pod address | kernel API |\n|---|---|---|\n") - for i, id := range ids { - b.WriteString(fmt.Sprintf("| lab host %d (entity-%d) | %s | http://[%s]:8800 |\n", i+1, id, ips[id], ips[id])) - } - if len(envs) > 0 { - b.WriteString("\n## Settings (also written to every host's " + app + "/.env)\n\n") - keys := make([]string, 0, len(envs)) - for k := range envs { - keys = append(keys, k) - } - sort.Strings(keys) - for _, k := range keys { - b.WriteString("- " + k + "\n") - } - } - b.WriteString("\n## Tests (run in order; write each verdict to results/.md)\n\n") - for i, t := range tests { - b.WriteString(fmt.Sprintf("%d. %s\n", i+1, t)) - } - if err := os.WriteFile(filepath.Join(dir, "README.md"), []byte(b.String()), 0o644); err != nil { - return nil, err - } - if err := os.MkdirAll(filepath.Join(dir, "results"), 0o755); err != nil { - return nil, err - } - _ = os.WriteFile(filepath.Join(dir, "results", ".keep"), nil, 0o644) - return makeTar(dir) -} - -func labStepPrompt(n, total int, step, app string, ids []int, ips map[int]string) string { - var hosts []string - for i, id := range ids { - hosts = append(hosts, fmt.Sprintf("lab host %d = entity-%d at %s", i+1, id, ips[id])) - } - return fmt.Sprintf("Lab test step %d of %d for the application `%s`.\n\nTEST: %s\n\nLab hosts: %s.\n\n"+ - "Read README.md first. Run the test against the lab hosts, then write a short verdict to results/%d.md "+ - "(first line PASS or FAIL, then the evidence). Do not modify the application; only test it.", - n, total, app, step, strings.Join(hosts, "; "), n) -} - -func labPost(url, ctype string, body []byte, timeout time.Duration) (int, string) { - c := &http.Client{Timeout: timeout} - resp, err := c.Post(url, ctype, bytes.NewReader(body)) - if err != nil { - return 0, err.Error() - } - defer resp.Body.Close() - rb, _ := io.ReadAll(resp.Body) - return resp.StatusCode, string(rb) -} - -func labGet(url string, timeout time.Duration) (int, string) { - c := &http.Client{Timeout: timeout} - resp, err := c.Get(url) - if err != nil { - return 0, err.Error() - } - defer resp.Body.Close() - rb, _ := io.ReadAll(resp.Body) - return resp.StatusCode, string(rb) -} - -// labTeardown removes everything a run created: the harness namespace, the lab hosts (Entity -// CRs → the operator deletes their namespaces), their volumes on local-path storage, and the -// lab-only registry rows. Best-effort, always runs, never raises. -func labTeardown(runID string, ids []int) { - if runID != "" { // "" = the orphan reap, which deletes lab- namespaces itself - _, _ = kc(120*time.Second, "delete", "namespace", "lab-"+runID, "--ignore-not-found", "--wait=false") - } - for _, id := range ids { - _, _ = kc(60*time.Second, "-n", "default", "delete", "entity", fmt.Sprintf("entity-%d", id), "--ignore-not-found", "--wait=false") - } - // wait for the CR to go (finalizers) and the operator to drop the namespace before wiping storage - for _, id := range ids { - for i := 0; i < 40; i++ { - if _, err := kc(15*time.Second, "-n", "default", "get", "entity", fmt.Sprintf("entity-%d", id)); err != nil { - break - } - time.Sleep(3 * time.Second) - } - for i := 0; i < 40; i++ { - if _, err := kc(15*time.Second, "get", "namespace", fmt.Sprintf("entity-%d", id)); err != nil { - break - } - time.Sleep(3 * time.Second) - } - for _, m := range []string{"entity-apps", "entity-data"} { - matches, _ := filepath.Glob(fmt.Sprintf("%s/pvc-*_entity-%d_%s-entity-%d-0", labStorageRoot, id, m, id)) - for _, p := range matches { - _ = os.RemoveAll(p) - } - } - } - _ = labDeleteRows(runID, ids) -} - -// cmdLabSelftest — `privatae-bastion lab-selftest `: run mode=lab from a spec file on -// THIS box without the host relay, print the result. Operator tooling for the lab box; on any -// other box it prints the same refusal the relay would (wall #1). -func cmdLabSelftest(path string) error { - raw, err := os.ReadFile(path) - if err != nil { - return err - } - var spec harnessSpec - if err := json.Unmarshal(raw, &spec); err != nil { - return fmt.Errorf("spec: %w", err) - } - spec.Mode = "lab" - if spec.HarnessID == "" { - spec.HarnessID = "lab" - } - out, code := runLab(spec) - var pretty bytes.Buffer - if json.Indent(&pretty, out, "", " ") == nil { - fmt.Println(pretty.String()) - } else { - fmt.Println(string(out)) - } - if code != 0 { - return fmt.Errorf("lab run exit %d", code) - } - return nil -} diff --git a/client-go/lab_off.go b/client-go/lab_off.go new file mode 100644 index 0000000..e331ecd --- /dev/null +++ b/client-go/lab_off.go @@ -0,0 +1,31 @@ +//go:build !lab + +package main + +// lab_off.go: mode=lab and `lab-selftest` in every release build. The lab +// harness is test tooling for one test lab and is built only with -tags lab, +// so a release bastion answers a lab run the way any bastion without a lab +// answers it: exit 126, in the same shape as an ungranted op. + +import ( + "encoding/json" + "errors" + "fmt" + "os" +) + +const labUnavailable = "mode 'lab' not available on this server — this build has no lab harness" + +func runLab(_ harnessSpec) ([]byte, int) { + b, _ := json.Marshal(map[string]any{"mode": "lab", "ran_on": orUnknown(hostnameSafe()), "ok": false, "error": labUnavailable}) + return b, 126 +} + +// cmdLabSelftest prints the same refusal the relay would give. +func cmdLabSelftest(path string) error { + if _, err := os.Stat(path); err != nil { + return err + } + fmt.Fprintln(os.Stderr, labUnavailable) + return errors.New("lab run exit 126") +} diff --git a/client-go/lab_off_test.go b/client-go/lab_off_test.go new file mode 100644 index 0000000..8cf6bc4 --- /dev/null +++ b/client-go/lab_off_test.go @@ -0,0 +1,21 @@ +//go:build !lab + +package main + +import ( + "encoding/json" + "testing" +) + +// A release build has no lab harness: a lab run is refused with exit 126, the +// shape of an ungranted op, and touches nothing. +func TestAReleaseBuildRefusesALabRun(t *testing.T) { + out, code := runHarness(harnessSpec{Mode: "lab", HarnessID: "lab-1", Gate: "auto", Image: "x", Tests: []string{"a"}}) + if code != 126 { + t.Fatalf("exit %d, want 126", code) + } + var m map[string]any + if err := json.Unmarshal(out, &m); err != nil || m["ok"] != false || m["error"] != labUnavailable { + t.Fatalf("reply %s", out) + } +} diff --git a/client-go/main.go b/client-go/main.go index 546f24c..04c234c 100644 --- a/client-go/main.go +++ b/client-go/main.go @@ -566,6 +566,13 @@ func main() { os.Exit(125) } os.Exit(opSandboxChildMain(os.Args[2])) + case opSandboxScopeSubcommand: + // A root op's scope (ops_taskcap_linux.go): never typed by a person; + // systemd-run starts it for the daemon. + if len(os.Args) < 3 { + os.Exit(125) + } + os.Exit(opSandboxScopeMain(os.Args[2])) case opActionSubcommand, opActionChildSubcommand, opActionProbeSubcommand: // An action op's unit and its sandbox's pid 1 (ops_action_linux.go): // never typed by a person; the daemon starts them. diff --git a/client-go/main_test.go b/client-go/main_test.go index 0ff4c37..ce74e89 100644 --- a/client-go/main_test.go +++ b/client-go/main_test.go @@ -45,6 +45,9 @@ func TestMain(m *testing.M) { if len(os.Args) >= 3 && os.Args[1] == opSandboxSubcommand { os.Exit(opSandboxChildMain(os.Args[2])) } + if len(os.Args) >= 3 && os.Args[1] == opSandboxScopeSubcommand { + os.Exit(opSandboxScopeMain(os.Args[2])) + } if len(os.Args) >= 3 && os.Args[1] == opActionSubcommand { os.Exit(opActionOuterMain(os.Args[2])) } @@ -115,8 +118,13 @@ func TestMain(m *testing.M) { // /proc/101 on the machine running the suite and the decision would depend // on whatever process holds that number. The real-signal test opts back in. procIdentity = func(int) (string, bool, bool) { return "", false, false } + // THE SWEEP IS ONE TOO. A test runs the ops' startup sweep for real, and it + // stops every op unit its names match: the suite's names are its own, so + // it never stops an op the machine's own bastion is running. + opIsolateOpsUnits() code := m.Run() + opIsolatedOpsUnitsDone() _ = os.RemoveAll(dir) os.Exit(code) } diff --git a/client-go/ops.go b/client-go/ops.go index 7e4550b..715bd90 100644 --- a/client-go/ops.go +++ b/client-go/ops.go @@ -446,7 +446,7 @@ func effectOK(f string) error { return errors.New("must be absolute") } if strings.Contains(f, "{") || strings.Contains(f, "..") { - return errors.New("no placeholders, no ..") + return errors.New("no placeholders, no \"..\"") } if strings.Contains(f, "\\") { // A backslash is filepath.Match's escape: `\n` is a pattern the diff --git a/client-go/ops_action_linux.go b/client-go/ops_action_linux.go index 3735875..94a338d 100644 --- a/client-go/ops_action_linux.go +++ b/client-go/ops_action_linux.go @@ -72,7 +72,6 @@ const ( opUIDFile = "op_uid_next" opActionRunTmpfsBytes = 16 << 20 opBuilderInSandbox = "/run/privatae-builder/buildkitd.sock" - opActionUnitPrefix = "privatae-op-" landlockRuleNetPort = 2 // LANDLOCK_RULE_NET_PORT // opActionRetryAfter: a machine that could not run action ops is asked // again after this long, not only at the next restart. @@ -817,13 +816,22 @@ type landlockNetPortAttr struct { Port uint64 } +// landlockRulesetAttr is the kernel's struct landlock_ruleset_attr up to +// "scoped", declared here like landlockNetPortAttr so the call does not move +// with x/sys's naming of it. +type landlockRulesetAttr struct { + HandledAccessFS uint64 + HandledAccessNet uint64 + Scoped uint64 +} + // opLandlock: TCP connects to the declared ports only, no TCP bind, and no // abstract unix socket or signal that leaves the sandbox. The addresses are // the unit's allowlist; the two together are the destinations. func opLandlock(ports []uint16) error { - attr := unix.LandlockRulesetAttr{ - Access_net: unix.LANDLOCK_ACCESS_NET_BIND_TCP | unix.LANDLOCK_ACCESS_NET_CONNECT_TCP, - Scoped: unix.LANDLOCK_SCOPE_ABSTRACT_UNIX_SOCKET | unix.LANDLOCK_SCOPE_SIGNAL, + attr := landlockRulesetAttr{ + HandledAccessNet: unix.LANDLOCK_ACCESS_NET_BIND_TCP | unix.LANDLOCK_ACCESS_NET_CONNECT_TCP, + Scoped: unix.LANDLOCK_SCOPE_ABSTRACT_UNIX_SOCKET | unix.LANDLOCK_SCOPE_SIGNAL, } fd, _, e := unix.Syscall(unix.SYS_LANDLOCK_CREATE_RULESET, uintptr(unsafe.Pointer(&attr)), unsafe.Sizeof(attr), 0) if e != 0 { @@ -861,6 +869,7 @@ func opStartupSweep() { opKillUnit(u) _ = exec.Command("systemctl", "stop", u).Run() } + opSweepScopes() entries, _ := os.ReadDir(root) for _, e := range entries { if !strings.HasPrefix(e.Name(), "run-") { diff --git a/client-go/ops_action_linux_test.go b/client-go/ops_action_linux_test.go index dd85471..ec0f302 100644 --- a/client-go/ops_action_linux_test.go +++ b/client-go/ops_action_linux_test.go @@ -7,9 +7,10 @@ package main // // Run them in a private PID namespace but NOT a private mount namespace: the // unit is started by the machine's systemd, in the machine's mounts, and must -// see the run's tmpfs the test mounted: +// see the run's tmpfs the test mounted. GOTMPDIR and TMPDIR name a folder +// on a disk, not under /tmp (a run gets its own /tmp): // -// GOTMPDIR=/srv/work/tmp-gotest TMPDIR=/srv/work/tmp-gotest unshare --pid --fork go test -run Action . +// GOTMPDIR= TMPDIR= unshare --pid --fork go test -run Action . import ( "bytes" @@ -725,7 +726,7 @@ func TestMemoryFloorSitsAboveTheKubeletsEviction(t *testing.T) { // A kill that does not land (no such unit, no cgroup yet) is never reported // as the op stopped. func TestTheMemoryWatchReportsOnlyAKillThatLanded(t *testing.T) { - stop := opMemoryWatch("privatae-op-nonexistent-optest", 1<<62) + stop := opMemoryWatch(opActionUnitPrefix+"nonexistent", 1<<62) time.Sleep(4 * opMemoryWatchPeriod) v := stop() if v.Stopped { diff --git a/client-go/ops_action_test.go b/client-go/ops_action_test.go index 31781fe..12ef0cf 100644 --- a/client-go/ops_action_test.go +++ b/client-go/ops_action_test.go @@ -131,7 +131,7 @@ func TestActionManifestRefusesWhatIsNotDeclaredOrNotBounded(t *testing.T) { } // A files op carrying an action op's fields is refused, not read as one. files := strings.Replace(opTestManifest, "modes:", "verbs:\n - {verb: unit.restart, unit: a.service}\nmodes:", 1) - if _, err := parseOpManifest([]byte(strings.Replace(files, "%s/", "/srv/work/optest/x/", 1))); err == nil || !strings.Contains(err.Error(), "belong to an action op") { + if _, err := parseOpManifest([]byte(strings.Replace(files, "%s/", "/srv/privatae-optest/x/", 1))); err == nil || !strings.Contains(err.Error(), "belong to an action op") { t.Fatalf("a files op with verbs: %v", err) } } diff --git a/client-go/ops_exec_linux.go b/client-go/ops_exec_linux.go index 6df8433..b007aff 100644 --- a/client-go/ops_exec_linux.go +++ b/client-go/ops_exec_linux.go @@ -224,6 +224,11 @@ type opReport struct { // per change, in path order. An op_run apply is bound to it (ops.go opTerms.Changes). ChangesSHA256 string `json:"changes_sha256"` Committed []string `json:"committed,omitempty"` + // TasksCapped: a cap held the run's tasks, root's included, at + // opSandboxTasksMax. A root op where none could be held runs uncapped, + // and TasksNote says so and why. + TasksCapped bool `json:"tasks_capped,omitempty"` + TasksNote string `json:"tasks_note,omitempty"` // An action op's apply (ops_verbs_linux.go): each verb as it ran, each // check after it, and the receipts for what its program pushed. Verbs []opVerbResult `json:"verbs,omitempty"` @@ -421,18 +426,49 @@ func opRunSandboxed(opDir string, m *opManifest, mode, params string) (*opReport limit := time.Duration(m.TimeLimit) * time.Second ctx, cancel := context.WithTimeout(context.Background(), limit) defer cancel() - cmd := exec.CommandContext(ctx, self, opSandboxSubcommand, specPath) - cmd.SysProcAttr = &syscall.SysProcAttr{ - Cloneflags: syscall.CLONE_NEWNS | syscall.CLONE_NEWPID | syscall.CLONE_NEWNET | syscall.CLONE_NEWIPC | syscall.CLONE_NEWUTS, - Setpgid: true, + // A user op's tasks are held by RLIMIT_NPROC. Root is exempt from it, so + // a root op's are held by a scope of its own (ops_taskcap_linux.go), or + // by nothing where none can be made, and the report says which. + unit := "" + if spec.AsUser { + rep.TasksCapped = true + } else if why := opTasksScopeWhyNot(); why != "" { + rep.TasksNote = opTasksNotCapped(why) + } else { + unit = opScopeUnit(run) + } + var out *cappedBuffer + launch := func(unit string) error { + cmd := opSandboxCmd(ctx, self, specPath, unit) + cmd.Stdin = strings.NewReader(params) + out = &cappedBuffer{max: opOutputMaxBytes} + cmd.Stdout, cmd.Stderr = out, out + // systemd-run hands this environment on to the scope's process. + cmd.Env = []string{"PATH=/usr/sbin:/usr/bin:/sbin:/bin", opSandboxHelperEnv} + cmd.WaitDelay = 2 * time.Second // the pipes close when the namespace is gone; never wait past that + return opRunSandboxCmd(cmd) } - cmd.Stdin = strings.NewReader(params) - out := &cappedBuffer{max: opOutputMaxBytes} - cmd.Stdout, cmd.Stderr = out, out - cmd.Env = []string{"PATH=/usr/sbin:/usr/bin:/sbin:/bin"} - cmd.WaitDelay = 2 * time.Second // the pipes close when the namespace is gone; never wait past that start := time.Now() - runErr := cmd.Run() + runErr := launch(unit) + if unit != "" { + if ctx.Err() == context.DeadlineExceeded { + opKillScope(unit) + } + switch got, ran := opScopeTasks(run); { + case ran && got == strconv.Itoa(opSandboxScopeTasks): + rep.TasksCapped = true + case ran: + rep.TasksNote = opTasksNotCapped(fmt.Sprintf("its scope's pids.max read %q", truncate(got, 200))) + case ctx.Err() != nil: + rep.TasksNote = opTasksNotCapped("its scope did not start inside the time limit") + default: + // systemd-run could not make the scope, or the scope's process + // died before it recorded the cap: either way nothing ran, and the + // op runs as it does where there is no systemd. + rep.TasksNote = opTasksNotCapped("its scope never started the op: " + truncate(opLastLine(out.String()), 200)) + runErr = launch("") + } + } rep.Seconds = time.Since(start).Seconds() rep.Output = out.String() if ctx.Err() == context.DeadlineExceeded { @@ -506,9 +542,16 @@ func opMatchDir(dir, like string) error { return nil } +// opUnprivilegedIdentityHook, when set, is the identity a user-privilege op +// runs as. Only tests set it. +var opUnprivilegedIdentityHook func() (uint32, uint32, error) + // opUnprivilegedIdentity is the session lane's user (session_unix.go): the // dedicated account if it exists, else nobody. func opUnprivilegedIdentity() (uint32, uint32, error) { + if opUnprivilegedIdentityHook != nil { + return opUnprivilegedIdentityHook() + } var u *user.User var err error for _, name := range sessionUnprivUsers { @@ -869,8 +912,10 @@ func opSandboxChildMain(arg string) int { return 125 } // Rlimits are inherited by the grandchild: no fork bombs, no giant files, - // a CPU cap beside the wall-clock one the parent enforces. - _ = unix.Setrlimit(unix.RLIMIT_NPROC, &unix.Rlimit{Cur: 64, Max: 64}) + // a CPU cap beside the wall-clock one the parent enforces. The kernel + // does not hold root to RLIMIT_NPROC; a root op's tasks are capped by + // the scope it runs in (ops_taskcap_linux.go). + _ = unix.Setrlimit(unix.RLIMIT_NPROC, &unix.Rlimit{Cur: opSandboxTasksMax, Max: opSandboxTasksMax}) _ = unix.Setrlimit(unix.RLIMIT_FSIZE, &unix.Rlimit{Cur: opUpperMaxBytes, Max: opUpperMaxBytes}) _ = unix.Setrlimit(unix.RLIMIT_NOFILE, &unix.Rlimit{Cur: 256, Max: 256}) _ = unix.Setrlimit(unix.RLIMIT_CPU, &unix.Rlimit{Cur: uint64(spec.TimeLimit), Max: uint64(spec.TimeLimit)}) @@ -884,23 +929,46 @@ func opSandboxChildMain(arg string) int { fmt.Fprintln(os.Stderr, "sandbox: no_new_privs:", err) return 125 } - cmd := exec.Command(spec.Interpreter, opSandboxMount+"/run", "--mode", spec.Mode) - cmd.Dir = "/" - cmd.Env = []string{ - "PATH=/usr/sbin:/usr/bin:/sbin:/bin", - "HOME=/tmp", - "TMPDIR=/tmp", - "LANG=C.UTF-8", - "PRIVATAE_OP_MODE=" + spec.Mode, - } - cmd.Stdin, cmd.Stdout, cmd.Stderr = os.Stdin, os.Stdout, os.Stderr - if spec.AsUser { + newCmd := func(ownUserNS bool) *exec.Cmd { + cmd := exec.Command(spec.Interpreter, opSandboxMount+"/run", "--mode", spec.Mode) + cmd.Dir = "/" + cmd.Env = []string{ + "PATH=/usr/sbin:/usr/bin:/sbin:/bin", + "HOME=/tmp", + "TMPDIR=/tmp", + "LANG=C.UTF-8", + "PRIVATAE_OP_MODE=" + spec.Mode, + } + cmd.Stdin, cmd.Stdout, cmd.Stderr = os.Stdin, os.Stdout, os.Stderr + if !spec.AsUser { + cmd.Env = append(cmd.Env, "USER=root") + return cmd + } cmd.Env = append(cmd.Env, "USER=privatae-op") - cmd.SysProcAttr = &syscall.SysProcAttr{Credential: &syscall.Credential{Uid: spec.UID, Gid: spec.GID, NoSetGroups: true}} - } else { - cmd.Env = append(cmd.Env, "USER=root") + attr := &syscall.SysProcAttr{Credential: &syscall.Credential{Uid: spec.UID, Gid: spec.GID, NoSetGroups: true}} + if ownUserNS { + // RLIMIT_NPROC counts every task of a user in its user namespace, + // so in the machine's own one the cap above would count whatever + // else runs as this user (nobody is shared with system daemons) + // and refuse the op on a busy machine. In a user namespace of its + // own the count is the op's tasks alone. The map is the identity: + // the op is the same user and sees every file's real owner. + attr.Cloneflags = syscall.CLONE_NEWUSER + attr.UidMappings = []syscall.SysProcIDMap{{ContainerID: 0, HostID: 0, Size: 1<<31 - 1}} + attr.GidMappings = []syscall.SysProcIDMap{{ContainerID: 0, HostID: 0, Size: 1<<31 - 1}} + } + cmd.SysProcAttr = attr + return cmd } - if err := cmd.Run(); err != nil { + cmd := newCmd(true) + err = cmd.Run() + if spec.AsUser && err != nil && cmd.Process == nil { + // No user namespace here (not built in, or turned off): run the op + // as before, with the cap counting the user's other tasks too. + cmd = newCmd(false) + err = cmd.Run() + } + if err != nil { var ee *exec.ExitError if errors.As(err, &ee) { if ws, ok := ee.Sys().(syscall.WaitStatus); ok && ws.Signaled() { diff --git a/client-go/ops_exec_other.go b/client-go/ops_exec_other.go index b7d535b..937a6e7 100644 --- a/client-go/ops_exec_other.go +++ b/client-go/ops_exec_other.go @@ -20,15 +20,14 @@ type opChange struct { Digest string `json:"sha256,omitempty"` Diff string `json:"diff,omitempty"` Note string `json:"note,omitempty"` - - src string - from string } type opReport struct { Mode string `json:"mode"` Exit int `json:"exit"` TimedOut bool `json:"timed_out,omitempty"` + TasksCapped bool `json:"tasks_capped,omitempty"` + TasksNote string `json:"tasks_note,omitempty"` Seconds float64 `json:"seconds"` Output string `json:"output"` Changes []opChange `json:"changes"` @@ -39,7 +38,6 @@ type opReport struct { Checks []opVerbResult `json:"checks,omitempty"` Receipts []opSignedReceipt `json:"receipts,omitempty"` - run string cleanup func() } @@ -60,6 +58,10 @@ func opCommitUpper(_ *opReport) ([]string, error) { func opSandboxChildMain(_ string) int { return 125 } +const opSandboxScopeSubcommand = "__op-sandbox-scope" + +func opSandboxScopeMain(_ string) int { return 125 } + func opMountsUnder(_ string) []string { return nil } const ( diff --git a/client-go/ops_memory_linux.go b/client-go/ops_memory_linux.go index 0616a47..3544920 100644 --- a/client-go/ops_memory_linux.go +++ b/client-go/ops_memory_linux.go @@ -56,8 +56,21 @@ import ( "time" ) +// The names of the ops' slice and of the units in it. The startup sweep stops +// every unit these name, so they are variables only for the tests, which give +// themselves names no running bastion uses (ops_names_linux_test.go): a suite +// run on a machine whose own bastion is running an op never touches it. +// Nothing else sets them. +var ( + opActionSlice = "privatae_ops.slice" + // opActionUnitPrefix names an action op's unit. + opActionUnitPrefix = "privatae-op-" + // opScopeUnitPrefix names a files op's scope. It is not the action ops' + // prefix, so their sweep never takes one for an action unit. + opScopeUnitPrefix = "privatae-opfiles-" +) + const ( - opActionSlice = "privatae_ops.slice" opMemoryFloorMin = 1 << 30 opMemoryKubeletPad = 512 << 20 opMemoryWatchPeriod = 250 * time.Millisecond @@ -85,8 +98,13 @@ func opKillUnit(unit string) bool { return exec.Command("systemctl", "kill", "--signal=SIGKILL", unit).Run() == nil } -// opOthersStalls is each top-level cgroup's memory stall (PSI some avg10, -// percent), the ops' own slice left out. +// opOthersStalls is the memory stall (PSI some avg10, percent) of each +// top-level cgroup, and of each op's unit or scope in the ops' slice. The +// slice's own figure is left out, since it holds the op that is measuring; +// that op's own unit is taken out by opStallsBut. Two ops can run at once +// (the management lane has more than one worker, and a ship plan runs its +// own), and a root op runs in a scope in this slice: an action op beside it +// must still yield when the root op's memory makes the machine wait. func opOthersStalls() map[string]float64 { out := map[string]float64{} entries, err := os.ReadDir(opCgroupRoot) @@ -94,29 +112,50 @@ func opOthersStalls() map[string]float64 { return out } for _, e := range entries { - if !e.IsDir() || e.Name() == opActionSlice { + if !e.IsDir() { continue } - raw, err := os.ReadFile(filepath.Join(opCgroupRoot, e.Name(), "memory.pressure")) - if err != nil { + if e.Name() != opActionSlice { + opReadStall(out, e.Name()) continue } - for _, line := range strings.Split(string(raw), "\n") { - if !strings.HasPrefix(line, "some ") { - continue - } - for _, f := range strings.Fields(line) { - if v, ok := strings.CutPrefix(f, "avg10="); ok { - if x, err := strconv.ParseFloat(v, 64); err == nil { - out[e.Name()] = x - } - } + ops, _ := os.ReadDir(filepath.Join(opCgroupRoot, opActionSlice)) + for _, o := range ops { + if o.IsDir() { + opReadStall(out, opActionSlice+"/"+o.Name()) } } } return out } +// opReadStall puts cgroup cg's PSI some avg10 into out, when it can be read. +func opReadStall(out map[string]float64, cg string) { + raw, err := os.ReadFile(filepath.Join(opCgroupRoot, cg, "memory.pressure")) + if err != nil { + return + } + for _, line := range strings.Split(string(raw), "\n") { + if !strings.HasPrefix(line, "some ") { + continue + } + for _, f := range strings.Fields(line) { + if v, ok := strings.CutPrefix(f, "avg10="); ok { + if x, err := strconv.ParseFloat(v, 64); err == nil { + out[cg] = x + } + } + } + } +} + +// opStallsBut is stalls without the action unit's own cgroup: an op's own +// stalls never stop it. +func opStallsBut(stalls map[string]float64, unit string) map[string]float64 { + delete(stalls, opActionSlice+"/"+unit+".service") + return stalls +} + // opStallRise is the cgroup whose stall rose most above its lowest reading // so far in the run, and by how much. A group stalling throughout (a pod // reclaiming against its own limit, a service at its MemoryMax) is that @@ -355,7 +394,7 @@ func opMemoryWatch(unit string, floor uint64) (stop func() opMemoryVerdict) { // Each group's lowest reading during the run: a group that spiked as // the run began must not hide a rise the run causes later, and one // held at its own limit throughout still never counts. - low := opOthersStalls() + low := opStallsBut(opOthersStalls(), unit) for { if raw, err := os.ReadFile(cur); err == nil { if n, err := strconv.ParseUint(strings.TrimSpace(string(raw)), 10, 64); err == nil { @@ -368,7 +407,7 @@ func opMemoryWatch(unit string, floor uint64) (stop func() opMemoryVerdict) { } _, avail, ok := opMemInfo() short := ok && avail <= floor - now := opOthersStalls() + now := opStallsBut(opOthersStalls(), unit) rise, who := opStallRise(low, now) for cg, x := range now { if l, ok := low[cg]; !ok || x < l { diff --git a/client-go/ops_memory_stalls_linux_test.go b/client-go/ops_memory_stalls_linux_test.go new file mode 100644 index 0000000..0bcc2b8 --- /dev/null +++ b/client-go/ops_memory_stalls_linux_test.go @@ -0,0 +1,71 @@ +package main + +import ( + "fmt" + "os" + "path/filepath" + "testing" +) + +// opFakeStalls points the stall readings at a folder of stand-in cgroups and +// returns how to set one's PSI some avg10. +func opFakeStalls(t *testing.T) func(cg string, some float64) { + t.Helper() + fake := t.TempDir() + prev := opCgroupRoot + opCgroupRoot = fake + t.Cleanup(func() { opCgroupRoot = prev }) + return func(cg string, some float64) { + _ = os.MkdirAll(filepath.Join(fake, cg), 0o755) + _ = os.WriteFile(filepath.Join(fake, cg, "memory.pressure"), + []byte(fmt.Sprintf("some avg10=%.2f avg60=0.00 avg300=0.00 total=1\nfull avg10=0.00 avg60=0.00 avg300=0.00 total=1\n", some)), 0o644) + } +} + +// Two ops can run at once: the management lane has more than one worker +// (mgmtWorkerCount) and a ship plan runs its own, and a root op runs in a +// scope in the ops' slice. Another op's scope or unit there is another +// program whose memory can make the machine wait, and its stalls are read +// beside every top-level group's. +func TestAnotherOpsStallsAreReadBesideTheMachinesOwn(t *testing.T) { + write := opFakeStalls(t) + rootOp := opActionSlice + "/" + opScopeUnitPrefix + "4242.scope" + otherAction := opActionSlice + "/" + opActionUnitPrefix + "run2-apply.service" + write("system.slice", 0.10) + write(opActionSlice, 0.10) + write(rootOp, 3.5) + write(otherAction, 1.25) + got := opOthersStalls() + if got["system.slice"] != 0.10 { + t.Errorf("a top-level group's stall: %v", got) + } + if got[rootOp] != 3.5 || got[otherAction] != 1.25 { + t.Errorf("a root op's scope and another action op's unit are read as other programs: %v", got) + } + if _, ok := got[opActionSlice]; ok { + t.Errorf("the ops' slice's own figure holds the op that is measuring, and is never read: %v", got) + } +} + +// What an action op's watch compares: a root op's scope rising beside it +// stops it, while its own unit's stalls and the slice's figure never do. +func TestAnActionOpYieldsToARootOpBesideItNeverToItself(t *testing.T) { + write := opFakeStalls(t) + own := opActionUnitPrefix + "run1-apply" + rootOp := opActionSlice + "/" + opScopeUnitPrefix + "4242.scope" + write("system.slice", 0.10) + write(opActionSlice, 0.10) + write(opActionSlice+"/"+own+".service", 0.10) + write(rootOp, 0.10) + base := opStallsBut(opOthersStalls(), own) + + write(opActionSlice, 60) + write(opActionSlice+"/"+own+".service", 60) + if rise, who := opStallRise(base, opStallsBut(opOthersStalls(), own)); rise != 0 { + t.Errorf("the op's own stalls rose %.2f in %s and counted", rise, who) + } + write(rootOp, 12.5) + if rise, who := opStallRise(base, opStallsBut(opOthersStalls(), own)); who != rootOp || rise < 12.3 || rise > 12.5 { + t.Errorf("a root op's scope beside it began waiting 12.4%% more: got %.2f in %q", rise, who) + } +} diff --git a/client-go/ops_names_linux_test.go b/client-go/ops_names_linux_test.go new file mode 100644 index 0000000..5b87bca --- /dev/null +++ b/client-go/ops_names_linux_test.go @@ -0,0 +1,79 @@ +package main + +// THE SUITE NEVER NAMES THE UNITS A RUNNING BASTION OWNS. +// +// The startup sweep (ops_action_linux.go opStartupSweep) stops every unit in +// the ops' slice whose name has an op's prefix, and a test runs it for real. +// Under the production names, a suite run on a machine whose own bastion is +// running an op would stop that op; under names shared between runs, two +// suites at once would stop each other's. So before any test runs (TestMain), +// this run takes a slice and prefixes of its own, with its pid and a random +// number in each. + +import ( + "crypto/rand" + "encoding/binary" + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" +) + +// The names every bastion so far gives its ops' slice and units. Written out +// rather than read from the code: they are what a bastion already installed +// on the machine uses, whatever this tree says. +const ( + opBastionSlice = "privatae_ops.slice" + opBastionActionUnits = "privatae-op-" + opBastionScopeUnits = "privatae-opfiles-" +) + +// opIsolateOpsUnits gives this run its own ops' slice and unit prefixes. A +// slice name has no "-", which systemd would read as a slice inside another. +func opIsolateOpsUnits() { + var b [4]byte + _, _ = rand.Read(b[:]) + run := fmt.Sprintf("%dx%08x", os.Getpid(), binary.BigEndian.Uint32(b[:])) + opActionSlice = "privatae_optest" + run + ".slice" + opActionUnitPrefix = "privatae-optest" + run + "-" + opScopeUnitPrefix = "privatae-optest" + run + "files-" +} + +// opIsolatedOpsUnitsDone stops this run's own slice, which systemd keeps +// after its last unit ends. Never a slice a bastion uses. +func opIsolatedOpsUnitsDone() { + if opActionSlice == opBastionSlice || !strings.HasPrefix(opActionSlice, "privatae_optest") { + return + } + if _, err := os.Stat(filepath.Join("/sys/fs/cgroup", opActionSlice)); err == nil { + _ = exec.Command("systemctl", "stop", opActionSlice).Run() + } +} + +// No name this run gives the ops' slice or units is a bastion's, sits in a +// bastion's slice, or shares a prefix with a bastion's units either way (so +// neither sweep takes the other's), and each holds this run's pid. +func TestTheSuiteNeverNamesTheUnitsABastionOwns(t *testing.T) { + if opActionSlice == opBastionSlice || strings.HasPrefix(opActionSlice, strings.TrimSuffix(opBastionSlice, ".slice")+"-") { + t.Errorf("the suite's ops' slice is %q: a bastion's own, or inside it", opActionSlice) + } + for _, mine := range []string{opActionUnitPrefix, opScopeUnitPrefix} { + for _, theirs := range []string{opBastionActionUnits, opBastionScopeUnits} { + if strings.HasPrefix(mine, theirs) || strings.HasPrefix(theirs, mine) { + t.Errorf("the suite's unit prefix %q and a bastion's %q: a sweep under one takes the other's units", mine, theirs) + } + } + } + if strings.HasPrefix(opScopeUnitPrefix, opActionUnitPrefix) { + t.Errorf("the suite's scope prefix %q starts with its action prefix %q: the action sweep would take a scope for its own", opScopeUnitPrefix, opActionUnitPrefix) + } + pid := strconv.Itoa(os.Getpid()) + for _, n := range []string{opActionSlice, opActionUnitPrefix, opScopeUnitPrefix} { + if !strings.Contains(n, pid) { + t.Errorf("%q does not hold this run's pid %s", n, pid) + } + } +} diff --git a/client-go/ops_names_other_test.go b/client-go/ops_names_other_test.go new file mode 100644 index 0000000..f528787 --- /dev/null +++ b/client-go/ops_names_other_test.go @@ -0,0 +1,8 @@ +//go:build !linux + +package main + +// Ops run only on Linux (ops_names_linux_test.go): elsewhere there are no +// units to keep apart. +func opIsolateOpsUnits() {} +func opIsolatedOpsUnitsDone() {} diff --git a/client-go/ops_nproc_linux_test.go b/client-go/ops_nproc_linux_test.go new file mode 100644 index 0000000..e0e6520 --- /dev/null +++ b/client-go/ops_nproc_linux_test.go @@ -0,0 +1,98 @@ +package main + +import ( + "encoding/base64" + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "syscall" + "testing" + "time" +) + +// opNprocTestUID is a uid no account on a test machine uses. +const opNprocTestUID = 61999 + +// A user-privilege op starts however many tasks other programs already run +// as its user (a busy machine's daemons running as nobody), and the cap on +// its tasks still stops the op's own fork bomb. +func TestAUserOpStartsWhateverElseRunsAsItsUser(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("needs root") + } + if ok, why := opSandboxAvailable(); !ok { + t.Skip(why) + } + opUnprivilegedIdentityHook = func() (uint32, uint32, error) { return opNprocTestUID, opNprocTestUID, nil } + t.Cleanup(func() { opUnprivilegedIdentityHook = nil }) + var others []*exec.Cmd + t.Cleanup(func() { + for _, c := range others { + _ = c.Process.Kill() + _ = c.Wait() + } + }) + for i := 0; i < 64; i++ { // the sandbox's cap + c := exec.Command("sleep", "60") + c.SysProcAttr = &syscall.SysProcAttr{Credential: &syscall.Credential{Uid: opNprocTestUID, Gid: opNprocTestUID, NoSetGroups: true}} + if err := c.Start(); err != nil { + t.Fatal(err) + } + others = append(others, c) + } + + runOp := func(extra string) (*opReport, error) { + n := time.Now().UnixNano() + root := filepath.Join(opTestRoot, fmt.Sprintf("np%d", n)) + opDir := filepath.Join(opTestRoot, fmt.Sprintf("npop%d", n)) + for _, p := range []string{root, opDir} { + p := p + t.Cleanup(func() { _ = os.RemoveAll(p) }) + } + _ = os.MkdirAll(root, 0o777) + _ = os.Chmod(root, 0o777) + _ = os.MkdirAll(opDir, 0o755) + for _, f := range opFixture(root, extra) { + raw, _ := base64.StdEncoding.DecodeString(f.B64) + _ = os.WriteFile(filepath.Join(opDir, f.Path), raw, os.FileMode(f.Mode)) + } + man := strings.Replace(fmt.Sprintf(opTestManifest, root), "privilege: root", "privilege: user", 1) + m, err := parseOpManifest([]byte(man)) + if err != nil { + t.Fatal(err) + } + rep, err := opRunSandboxed(opDir, m, opModeDryRun, `{"name":"x","count":1,"kind":"a"}`) + if rep != nil { + t.Cleanup(rep.cleanup) + } + return rep, err + } + + rep, err := runOp(`echo "RAN-AS=$(id -u)"; true`) + if err != nil { + t.Fatalf("a user op with 64 other tasks of its user on the machine: %v", err) + } + if !strings.Contains(rep.Output, fmt.Sprintf("RAN-AS=%d", opNprocTestUID)) { + t.Errorf("the op did not run as its user: %q", rep.Output) + } + + // The op's own tasks are still capped: a loop starting 100 is stopped, + // and only after about as many as the cap, its own, not the others'. + rep, err = runOp(`i=0; while [ $i -lt 100 ]; do echo "STARTING=$i"; sleep 5 & i=$((i+1)); done; echo DONE; true`) + if err == nil || !strings.Contains(strings.ToLower(err.Error()), "fork") || strings.Contains(rep.Output, "DONE") { + t.Fatalf("100 background tasks were not stopped by the cap: %v, %q", err, rep.Output) + } + last := "" + for _, line := range strings.Split(rep.Output, "\n") { + if strings.HasPrefix(line, "STARTING=") { + last = strings.TrimPrefix(line, "STARTING=") + } + } + if n, _ := strconv.Atoi(last); n < 50 { + t.Errorf("the cap stopped the op after %s of its own tasks: it counts other tasks too", last) + } + t.Logf("the op's fork loop was stopped at task %s", last) +} diff --git a/client-go/ops_taskcap_linux.go b/client-go/ops_taskcap_linux.go new file mode 100644 index 0000000..3d27e07 --- /dev/null +++ b/client-go/ops_taskcap_linux.go @@ -0,0 +1,200 @@ +//go:build linux + +package main + +// ops_taskcap_linux.go — a root op's tasks are capped by a cgroup. +// +// A files op's tasks are capped at opSandboxTasksMax by RLIMIT_NPROC, and the +// kernel never applies that limit to root: a task whose real user is root, +// or that holds CAP_SYS_RESOURCE or CAP_SYS_ADMIN, forks past it. A user op +// runs as an unprivileged user and the limit holds. A root op's runaway fork +// was held by nothing but its time limit. +// +// A cgroup's pids.max counts every task in the cgroup, root's included. So a +// root op runs in a scope of its own, in the ops' slice, with TasksMax set, +// started by systemd-run as an action op's unit is. The scope's process is +// this binary as `__op-sandbox-scope `: it reads its own cgroup's +// pids.max, writes what it read beside the spec, and only then starts the +// sandbox's pid 1 exactly as the bastion does without a scope. The report +// says the tasks were capped only when that value is the cap. +// +// Where no scope can be made (no systemd, no pids controller, systemd-run +// failing), the op runs as before, and its report says its tasks were not +// capped and why. A scope that could not start ran nothing: the value is +// written before the sandbox starts, so with no value the op never ran. + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strconv" + "strings" + "syscall" +) + +const ( + // opSandboxTasksMax is the most tasks a files op's sandbox holds at once, + // threads included: RLIMIT_NPROC for a user op, pids.max for a root op. + opSandboxTasksMax = 64 + // opSandboxScopeTasks is the scope's TasksMax: the op's own cap plus room + // for the two bastion processes that start it inside the scope (this + // binary, twice, each with its runtime's threads; 8 tasks between them + // where measured, held there by opSandboxHelperEnv), so a root op may + // start at least as many tasks as a user op. + opSandboxScopeTasks = opSandboxTasksMax + 32 + // opSandboxHelperEnv is in the environment of both bastion processes that + // start the sandbox. The Go runtime starts threads by the CPUs it may use, + // and in a scope each one counts against the cap: measured, the two held + // 16 tasks with 12 CPUs, 44 with the runtime set for 64, and set for 256 + // the sandbox's pid 1 could not start a thread and the op never ran. With + // one, they hold 8 on a machine of any size. Neither process does more + // than start its child and wait for it. The op never sees this: pid 1 + // sets the op's environment itself. + opSandboxHelperEnv = "GOMAXPROCS=1" + opSandboxScopeSubcommand = "__op-sandbox-scope" + // opScopeTasksFile, beside the spec on the run's tmpfs, holds the + // pids.max the scope's process read in its own cgroup. + opScopeTasksFile = "scope_pids_max" +) + +// opTasksScopeWhyNot says why this machine cannot hold an op's tasks in a +// scope of their own, "" when it can. +func opTasksScopeWhyNot() string { + if _, err := exec.LookPath("systemd-run"); err != nil { + return "systemd-run is not installed" + } + if st, err := os.Stat("/run/systemd/system"); err != nil || !st.IsDir() { + return "systemd is not this machine's init" + } + raw, err := os.ReadFile("/sys/fs/cgroup/cgroup.controllers") + if err != nil { + return "this machine does not run the unified cgroup hierarchy" + } + for _, c := range strings.Fields(string(raw)) { + if c == "pids" { + return "" + } + } + return "the kernel offers no pids cgroup controller" +} + +// opTasksScopeArgs is the systemd-run command line for a root op's run. +func opTasksScopeArgs(unit, self, specPath string) []string { + return []string{"--scope", "--quiet", "--collect", "--unit=" + unit, "--slice=" + opActionSlice, + "-p", "TasksMax=" + strconv.Itoa(opSandboxScopeTasks), + "--", self, opSandboxScopeSubcommand, specPath} +} + +// opScopeUnit names the scope of the run whose tmpfs is run. +func opScopeUnit(run string) string { + return opScopeUnitPrefix + strings.TrimPrefix(filepath.Base(run), "run-") +} + +// opKillScope kills every task in a files op's scope through the kernel. +func opKillScope(unit string) { + _ = os.WriteFile(filepath.Join("/sys/fs/cgroup", opActionSlice, unit+".scope", "cgroup.kill"), []byte("1"), 0o600) +} + +// opSweepScopes kills what a files op's scope still holds when the bastion +// died under it. +func opSweepScopes() { + entries, _ := os.ReadDir(filepath.Join("/sys/fs/cgroup", opActionSlice)) + for _, e := range entries { + if n := e.Name(); e.IsDir() && strings.HasPrefix(n, opScopeUnitPrefix) && strings.HasSuffix(n, ".scope") { + opKillScope(strings.TrimSuffix(n, ".scope")) + } + } +} + +// opSandboxCmd is the command that starts the sandbox: the sandbox's pid 1 +// directly, or systemd-run making the scope it runs in. +func opSandboxCmd(ctx context.Context, self, specPath, unit string) *exec.Cmd { + if unit == "" { + cmd := exec.CommandContext(ctx, self, opSandboxSubcommand, specPath) + cmd.SysProcAttr = &syscall.SysProcAttr{ + Cloneflags: syscall.CLONE_NEWNS | syscall.CLONE_NEWPID | syscall.CLONE_NEWNET | syscall.CLONE_NEWIPC | syscall.CLONE_NEWUTS, + Setpgid: true, + } + return cmd + } + cmd := exec.CommandContext(ctx, "systemd-run", opTasksScopeArgs(unit, self, specPath)...) + // The scope is outside the bastion's own cgroup, so a bastion stopped or + // restarted does not take it down: the scope's process dies with the + // bastion instead (systemd-run execs it, which keeps the signal), and the + // sandbox's pid 1 dies with that process. + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true, Pdeathsig: syscall.SIGKILL} + return cmd +} + +// opRunSandboxCmd runs cmd on a thread held for the whole run: a parent- +// death signal is sent when the thread that started the child exits. +func opRunSandboxCmd(cmd *exec.Cmd) error { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + return cmd.Run() +} + +// opScopeTasks reads what the scope's process found its cgroup's pids.max to +// be: ok is false when the scope's process never ran. +func opScopeTasks(run string) (string, bool) { + raw, err := os.ReadFile(filepath.Join(run, opScopeTasksFile)) + if err != nil { + return "", false + } + return strings.TrimSpace(string(raw)), true +} + +// opOwnPidsMax reads pids.max of the cgroup this process is in. +func opOwnPidsMax() string { + raw, err := os.ReadFile("/proc/self/cgroup") + if err != nil { + return "unreadable: " + err.Error() + } + for _, line := range strings.Split(string(raw), "\n") { + if p, ok := strings.CutPrefix(line, "0::"); ok { + v, err := os.ReadFile(filepath.Join("/sys/fs/cgroup", filepath.Clean("/"+p), "pids.max")) + if err != nil { + return "unreadable: " + err.Error() + } + return strings.TrimSpace(string(v)) + } + } + return "unreadable: no unified cgroup" +} + +// opSandboxScopeMain is the scope's process: it records its cgroup's +// pids.max, then starts the sandbox's pid 1 and returns its exit. +func opSandboxScopeMain(specPath string) int { + if err := os.WriteFile(filepath.Join(filepath.Dir(specPath), opScopeTasksFile), []byte(opOwnPidsMax()), 0o600); err != nil { + fmt.Fprintln(os.Stderr, "sandbox: cannot record the scope's task cap:", err) + return 125 + } + self, err := os.Executable() + if err != nil { + fmt.Fprintln(os.Stderr, "sandbox:", err) + return 125 + } + cmd := exec.Command(self, opSandboxSubcommand, specPath) + cmd.SysProcAttr = &syscall.SysProcAttr{ + Cloneflags: syscall.CLONE_NEWNS | syscall.CLONE_NEWPID | syscall.CLONE_NEWNET | syscall.CLONE_NEWIPC | syscall.CLONE_NEWUTS, + Pdeathsig: syscall.SIGKILL, + } + cmd.Stdin, cmd.Stdout, cmd.Stderr = os.Stdin, os.Stdout, os.Stderr + cmd.Env = []string{"PATH=/usr/sbin:/usr/bin:/sbin:/bin", opSandboxHelperEnv} + return opExitOf(cmd.Run()) +} + +// opTasksNotCapped is the report's note for a root op whose tasks no cgroup +// held. +func opTasksNotCapped(why string) string { + return "this root op's tasks were not capped: " + why +} + +// opLastLine is the last line s holds, "" for none. +func opLastLine(s string) string { + lines := strings.Split(strings.TrimSpace(s), "\n") + return lines[len(lines)-1] +} diff --git a/client-go/ops_taskcap_linux_test.go b/client-go/ops_taskcap_linux_test.go new file mode 100644 index 0000000..4f9cd16 --- /dev/null +++ b/client-go/ops_taskcap_linux_test.go @@ -0,0 +1,327 @@ +package main + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "os" + "path/filepath" + "runtime" + "strconv" + "strings" + "testing" + "time" + + "golang.org/x/sys/unix" +) + +// opTaskCapRun runs one dry run of the op fixture with the given run body +// (@ROOT@ in it is the op's effects folder), at the given privilege, and +// returns its report and its report's JSON (what the host receives). A +// timeLimit above 0 replaces the fixture's. +func opTaskCapRun(t *testing.T, privilege, body string, timeLimit int) (*opReport, string, error) { + t.Helper() + n := time.Now().UnixNano() + root := filepath.Join(opTestRoot, fmt.Sprintf("tc%d", n)) + opDir := filepath.Join(opTestRoot, fmt.Sprintf("tcop%d", n)) + for _, p := range []string{root, opDir} { + p := p + t.Cleanup(func() { _ = os.RemoveAll(p) }) + } + _ = os.MkdirAll(root, 0o777) + _ = os.Chmod(root, 0o777) + _ = os.MkdirAll(opDir, 0o755) + for _, f := range opFixture(root, strings.ReplaceAll(body, "@ROOT@", root)) { + raw, _ := base64.StdEncoding.DecodeString(f.B64) + _ = os.WriteFile(filepath.Join(opDir, f.Path), raw, os.FileMode(f.Mode)) + } + man := strings.Replace(fmt.Sprintf(opTestManifest, root), "privilege: root", "privilege: "+privilege, 1) + if timeLimit > 0 { + man = strings.Replace(man, "time_limit: 30", fmt.Sprintf("time_limit: %d", timeLimit), 1) + } + m, err := parseOpManifest([]byte(man)) + if err != nil { + t.Fatal(err) + } + rep, err := opRunSandboxed(opDir, m, opModeDryRun, `{"name":"x","count":1,"kind":"a"}`) + if rep == nil { + return nil, "", err + } + t.Cleanup(rep.cleanup) + raw, _ := json.Marshal(rep) + return rep, string(raw), err +} + +// opFakeSystemdRun puts a stand-in for systemd-run first on PATH. With a +// cgroup it does what `systemd-run --scope -p TasksMax=N` does for the +// process it runs: a cgroup of its own whose pids.max is N, then the command. +// The cgroup is a fresh one at the top of the hierarchy, removed after the +// test; no unit is started. Without one it fails the way systemd-run does +// with no bus to reach. It records the arguments it was given. +func opFakeSystemdRun(t *testing.T, cgroup string) (argsFile string) { + t.Helper() + dir := t.TempDir() + argsFile = filepath.Join(dir, "args") + body := "echo 'Failed to connect to bus: No such file or directory' >&2; exit 1\n" + if cgroup != "" { + body = `max= +while [ $# -gt 0 ]; do + case "$1" in + --) shift; break ;; + -p) shift; case "$1" in TasksMax=*) max=${1#TasksMax=} ;; esac ;; + esac + shift +done +mkdir -p ` + cgroup + ` +[ -z "$max" ] || echo "$max" > ` + cgroup + `/pids.max +echo $$ > ` + cgroup + `/cgroup.procs +exec "$@" +` + t.Cleanup(func() { + // The cgroup empties once the sandbox's pid 1 is gone, which takes + // every task the op left behind with it. + for i := 0; i < 100; i++ { + if err := os.Remove(cgroup); err == nil || os.IsNotExist(err) { + return + } + time.Sleep(100 * time.Millisecond) + } + t.Errorf("the test cgroup %s did not empty", cgroup) + }) + } + script := "#!/bin/sh\nprintf '%s\\n' \"$@\" > " + argsFile + "\n" + body + if err := os.WriteFile(filepath.Join(dir, "systemd-run"), []byte(script), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", dir+":"+os.Getenv("PATH")) + return argsFile +} + +// A root op's tasks are capped as a user op's are. RLIMIT_NPROC does not +// hold root (the kernel exempts it), so a root op runs in a scope of its own +// whose pids.max counts every task in it, root's included: a loop starting +// 100 background tasks is stopped at the cap, and the report says a cap held. +func TestARootOpIsStoppedAtItsTaskCap(t *testing.T) { + requireSandbox(t) + if ctl, err := os.ReadFile("/sys/fs/cgroup/cgroup.subtree_control"); err != nil || !strings.Contains(" "+string(ctl), " pids") { + t.Skip("no pids controller at the top of the unified cgroup hierarchy here") + } + if _, err := os.Stat("/run/systemd/system"); err != nil { + t.Skip("the bastion asks for a scope only where systemd is init") + } + cgroup := fmt.Sprintf("/sys/fs/cgroup/privatae-optest-%d", os.Getpid()) + argsFile := opFakeSystemdRun(t, cgroup) + + rep, js, err := opTaskCapRun(t, "root", `i=0; while [ $i -lt 100 ]; do echo "STARTING=$i"; sleep 5 & i=$((i+1)); done; echo DONE; true`, 0) + if rep == nil { + t.Fatalf("the op did not run: %v", err) + } + if err == nil || !strings.Contains(strings.ToLower(err.Error()), "fork") || strings.Contains(rep.Output, "DONE") { + t.Fatalf("a root op started 100 background tasks with nothing stopping it: %v, %q", err, rep.Output) + } + last := "" + for _, line := range strings.Split(rep.Output, "\n") { + if strings.HasPrefix(line, "STARTING=") { + last = strings.TrimPrefix(line, "STARTING=") + } + } + if n, _ := strconv.Atoi(last); n < 50 { + t.Errorf("the cap stopped the op after %s of its own tasks: the sandbox's own processes take too much of it", last) + } + t.Logf("the root op's fork loop was stopped at task %s", last) + if !strings.Contains(js, `"tasks_capped":true`) || strings.Contains(js, `"tasks_note"`) { + t.Errorf("the report does not say the cap held: %s", js) + } + + raw, _ := os.ReadFile(argsFile) + args := strings.Split(strings.TrimSpace(string(raw)), "\n") + want := []string{"--scope", "--slice=" + opActionSlice, "-p\nTasksMax="} + for _, w := range want { + if !strings.Contains(string(raw), w) { + t.Errorf("systemd-run was not given %q: %q", w, args) + } + } + unit := "" + for _, a := range args { + if strings.HasPrefix(a, "--unit=") { + unit = strings.TrimPrefix(a, "--unit=") + } + } + if !strings.HasPrefix(unit, opScopeUnitPrefix) || strings.HasPrefix(unit, opActionUnitPrefix) { + t.Errorf("the scope's unit is %q: it must be a files op's, never one the action ops' sweep takes for its own", unit) + } + + // A root op that starts a few tasks finishes as before. + rep, js, err = opTaskCapRun(t, "root", `for i in 1 2 3 4 5; do sleep 0.1 & done; wait; echo RAN-AS=$(id -u); true`, 0) + if err != nil { + t.Fatalf("a root op starting five tasks: %v", err) + } + if !strings.Contains(rep.Output, "RAN-AS=0") || !strings.Contains(js, `"tasks_capped":true`) { + t.Errorf("a capped root op: %q %s", rep.Output, js) + } + + // The time limit still ends a run in its scope, with everything it + // started (the cgroup must empty for the cleanup to remove it). + rep, _, err = opTaskCapRun(t, "root", `sleep 30 & sleep 30; true`, 1) + if err == nil || rep == nil || !rep.TimedOut { + t.Fatalf("a root op past its time limit in its scope: %v", err) + } +} + +// opTaskCapRoom runs a root op's fork loop in its scope and returns how many +// tasks the op held when its next fork was refused. +func opTaskCapRoom(t *testing.T) int { + t.Helper() + rep, _, err := opTaskCapRun(t, "root", `i=0; while [ $i -lt 150 ]; do echo "STARTING=$i"; sleep 5 & i=$((i+1)); done; echo DONE; true`, 0) + if rep == nil || err == nil || strings.Contains(rep.Output, "DONE") { + t.Fatalf("the fork loop was not stopped by the scope's cap: %v", err) + } + last := -1 + for _, line := range strings.Split(rep.Output, "\n") { + if v, ok := strings.CutPrefix(line, "STARTING="); ok { + last, _ = strconv.Atoi(v) + } + } + // STARTING=n is printed before the fork of task n: the shell and n tasks + // were running when that fork was refused. + return last + 1 +} + +// A root op has the same room under its scope's cap on a machine of any +// size. The two bastion processes in the scope are Go programs, and the Go +// runtime starts threads by the CPUs it may use: left to that, with the +// runtime set for 64 CPUs they held 44 of the scope's 96 tasks, leaving the +// op fewer than a user op's 64, and set for 256 the sandbox's pid 1 could not +// start at all. Here the bastion's CPUs are narrowed to one, then given back: +// the op must have as much room either way. +func TestARootOpsTaskRoomDoesNotShrinkWithTheMachinesCPUs(t *testing.T) { + requireSandbox(t) + if ctl, err := os.ReadFile("/sys/fs/cgroup/cgroup.subtree_control"); err != nil || !strings.Contains(" "+string(ctl), " pids") { + t.Skip("no pids controller at the top of the unified cgroup hierarchy here") + } + if _, err := os.Stat("/run/systemd/system"); err != nil { + t.Skip("the bastion asks for a scope only where systemd is init") + } + var all unix.CPUSet + if err := unix.SchedGetaffinity(0, &all); err != nil || all.Count() < 4 { + t.Skip("needs at least 4 CPUs to show a difference") + } + opFakeSystemdRun(t, fmt.Sprintf("/sys/fs/cgroup/privatae-optest-cpus-%d", os.Getpid())) + + // The sandbox is started from this goroutine's thread (opRunSandboxCmd + // holds it), and every process under it takes that thread's CPUs. + runtime.LockOSThread() + defer runtime.UnlockOSThread() + defer func() { _ = unix.SchedSetaffinity(0, &all) }() + var one unix.CPUSet + for c := 0; ; c++ { + if all.IsSet(c) { + one.Set(c) + break + } + } + if err := unix.SchedSetaffinity(0, &one); err != nil { + t.Fatal(err) + } + narrow := opTaskCapRoom(t) + if err := unix.SchedSetaffinity(0, &all); err != nil { + t.Fatal(err) + } + wide := opTaskCapRoom(t) + t.Logf("a root op held %d tasks with 1 CPU and %d with %d CPUs", narrow, wide, all.Count()) + if narrow < opSandboxTasksMax || wide < opSandboxTasksMax { + t.Errorf("a root op held fewer tasks than a user op's %d: %d with 1 CPU, %d with %d", opSandboxTasksMax, narrow, wide, all.Count()) + } + if wide < narrow-3 { + t.Errorf("a root op's room shrank from %d tasks to %d with %d CPUs: the bastion's own processes in its scope took it", narrow, wide, all.Count()) + } +} + +// Where no scope can be made, a root op runs as before and its report says +// its tasks were not capped, and why; it never claims a cap. A user op's +// tasks are held by RLIMIT_NPROC with no scope at all. +func TestARootOpWithNoScopeSaysItsTasksWereNotCapped(t *testing.T) { + requireSandbox(t) + body := `echo RAN-AS=$(id -u); true` + + // systemd-run is there and fails: the op still runs, once. + opFakeSystemdRun(t, "") + rep, js, err := opTaskCapRun(t, "root", `echo RUN >> @ROOT@/runs; echo "RUNS=$(wc -l < @ROOT@/runs)"; `+body, 0) + if err != nil { + t.Fatalf("a root op where systemd-run fails: %v", err) + } + if !strings.Contains(rep.Output, "RAN-AS=0") || !strings.Contains(rep.Output, "RUNS=1") { + t.Errorf("the op did not run as root, once: %q", rep.Output) + } + if strings.Contains(js, `"tasks_capped":true`) || !strings.Contains(js, `"tasks_note":"`) || !strings.Contains(js, "not capped") || !strings.Contains(js, "Failed to connect to bus") { + t.Errorf("the report must say the root op's tasks were not capped and why: %s", js) + } + + // No systemd-run at all. + empty := t.TempDir() + t.Setenv("PATH", empty) + _, js, err = opTaskCapRun(t, "root", body, 0) + if err != nil { + t.Fatalf("a root op with no systemd-run: %v", err) + } + if strings.Contains(js, `"tasks_capped":true`) || !strings.Contains(js, "not capped") || !strings.Contains(js, "systemd-run is not installed") { + t.Errorf("the report must say the root op's tasks were not capped and why: %s", js) + } + + // A user op needs no scope: its cap is the rlimit, and the report says so. + if _, _, err := opUnprivilegedIdentity(); err == nil { + _, js, err = opTaskCapRun(t, "user", body, 0) + if err != nil { + t.Fatalf("a user op: %v", err) + } + if !strings.Contains(js, `"tasks_capped":true`) || strings.Contains(js, `"tasks_note"`) { + t.Errorf("a user op's report: %s", js) + } + } +} + +// An action op's program needs no scope of its own: it runs as a uid that no +// other run, account or subordinate range has (opNextUID), so RLIMIT_NPROC +// counts its own tasks only, and its unit's TasksMax holds every task in the +// unit, root's included. systemd-run is a stand-in that records what it was +// given and the spec it was handed; no unit starts. +func TestAnActionOpsTasksAreItsOwnAndItsUnitIsCapped(t *testing.T) { + // The run mounts its own tmpfs before it reaches systemd-run, which takes + // what the sandbox takes: root that may mount and make namespaces. + requireSandbox(t) + dir := t.TempDir() + rec := filepath.Join(dir, "rec") + script := "#!/bin/sh\nprintf '%s\\n' \"$@\" >> " + rec + "\nfor a; do last=$a; done\ncat \"$last\" >> " + rec + "\necho >> " + rec + "\n" + if err := os.WriteFile(filepath.Join(dir, "systemd-run"), []byte(script), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", dir+":"+os.Getenv("PATH")) + m, err := parseOpManifest([]byte(opActionManifest)) + if err != nil { + t.Fatal(err) + } + opDir := t.TempDir() + uids := map[uint32]bool{} + for i := 0; i < 2; i++ { + _ = os.Remove(rec) + if _, _, err := opRunActionProgram(opDir, m, opModeDryRun, `{}`, nil, fmt.Sprintf("tc%d", i)); err != nil { + t.Fatalf("the stand-in unit: %v", err) + } + raw, err := os.ReadFile(rec) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(raw), "-p\nTasksMax=512\n") || !strings.Contains(string(raw), "--slice="+opActionSlice+"\n") { + t.Errorf("the action op's unit is not held to a task cap: %q", raw) + } + var spec opActionSpec + js := string(raw)[strings.Index(string(raw), "{"):] + if err := json.Unmarshal([]byte(strings.TrimSpace(js)), &spec); err != nil { + t.Fatalf("the spec the unit was handed: %v %q", err, js) + } + if spec.UID < opUIDBase || spec.GID != spec.UID || uids[spec.UID] { + t.Errorf("run %d's program runs as uid %d: not a fresh one of the ops' range", i, spec.UID) + } + uids[spec.UID] = true + } +} diff --git a/client-go/ops_test.go b/client-go/ops_test.go index f0d1297..069e3e3 100644 --- a/client-go/ops_test.go +++ b/client-go/ops_test.go @@ -519,7 +519,7 @@ func requireSandbox(t *testing.T) { } } -const opTestRoot = "/srv/work/optest" +const opTestRoot = "/srv/privatae-optest" // opProbeRun is a python3 op that tries what the sandbox must deny: a dial // to the host's sshd and a write under /proc. It prints a marker for each diff --git a/client-go/ops_verbs_linux_test.go b/client-go/ops_verbs_linux_test.go index 0028ddd..c1833f8 100644 --- a/client-go/ops_verbs_linux_test.go +++ b/client-go/ops_verbs_linux_test.go @@ -303,6 +303,9 @@ func TestTreeCheckoutMovesOnlyForward(t *testing.T) { if _, err := exec.LookPath("git"); err != nil { t.Skip("no git") } + if os.Geteuid() != 0 { + t.Skip("needs root: the op moves only a tree root owns, and the fixtures are made by this user") + } setupBox(t) useFileRepos(t) base := t.TempDir() diff --git a/client-go/ops_view_test.go b/client-go/ops_view_test.go index e5af31f..3bb713d 100644 --- a/client-go/ops_view_test.go +++ b/client-go/ops_view_test.go @@ -110,7 +110,7 @@ func TestOpManifestReadsAreJudgedLikeEffects(t *testing.T) { `"/proc/1/environ"`: false, `"/"`: false, } { - _, err := parseOpManifest([]byte(strings.Replace(strings.Replace(base, "%s/", "/srv/work/optest/x/", 1), "%s", read, 1))) + _, err := parseOpManifest([]byte(strings.Replace(strings.Replace(base, "%s/", "/srv/privatae-optest/x/", 1), "%s", read, 1))) if (err == nil) != ok { t.Errorf("reads %s: ok=%v err=%v", read, ok, err) } diff --git a/client-go/ownaddr.go b/client-go/ownaddr.go index 48e84d0..1cf0ec9 100644 --- a/client-go/ownaddr.go +++ b/client-go/ownaddr.go @@ -2,8 +2,8 @@ package main // ownaddr.go — is a destination THIS machine? // -// The outbound lanes that dial for someone else (the egress lane on an -// anchored box, egress.go; a coder run's proxy, coder_egress.go; the backup +// The outbound lanes that dial for someone else (the egress lane, +// egress.go; a coder run's proxy, coder_egress.go; the backup // lanes' fetch and push, backupfetch.go) must never be turned back on the // machine itself: a service listening on 0.0.0.0 here // answers on every address the kernel delivers locally, not only on loopback @@ -19,6 +19,15 @@ package main // `local` route (`ip route add local 192.0.2.0/24 dev lo`, AnyIP) makes a // whole range this machine. Only a route of type unicast leaves the // machine; local, broadcast, anycast and multicast routes are refused. +// A destination on a link that stays on this machine is refused too: a +// container's veth, or a bridge whose every port is a veth, a tap or a +// dummy (the bridge networks Docker, Podman, LXD and libvirt make here). +// A container or VM on one of them is a service on this machine. A route +// through a gateway on such a link leaves: the gateway sends the packet +// on, as the machine a container runs on does for the container. A link +// a main-table default route goes out of is never inside: it is this +// machine's way out, so a bastion in a container keeps the network on +// its own veth, its LAN included. // // The IPv4 address inside an IPv4-mapped, NAT64 (64:ff9b::/96) or 6to4 // (2002::/16) address is checked as well, since that is what such an address diff --git a/client-go/ownaddr_inside_linux_test.go b/client-go/ownaddr_inside_linux_test.go new file mode 100644 index 0000000..02c5541 --- /dev/null +++ b/client-go/ownaddr_inside_linux_test.go @@ -0,0 +1,325 @@ +//go:build linux + +package main + +import ( + "context" + "encoding/binary" + "errors" + "net" + "os" + "os/exec" + "strings" + "syscall" + "testing" + + "golang.org/x/sys/unix" +) + +// The rule alone, over link tables written out here: which links keep a +// packet on this machine. +func TestInsideMachineReadsTheLinkAndABridgesPorts(t *testing.T) { + links := map[uint32]linkInfo{ + 1: {kind: ""}, // lo + 2: {kind: ""}, // a network card + 3: {kind: "bridge"}, // docker0 + 4: {kind: "veth", master: 3}, // a container's link on docker0 + 5: {kind: "veth"}, // a pod's link, routed to directly (Calico) + 6: {kind: "bridge"}, // virbr0 + 7: {kind: "dummy", master: 6}, // libvirt's placeholder port + 8: {kind: "tun", master: 6}, // a VM's tap + 9: {kind: "bridge"}, // br0 on a server: the LAN + 10: {kind: "", master: 9}, // its network card + 11: {kind: "veth", master: 9}, // and a container bridged onto the LAN + 12: {kind: "bridge"}, // a Docker network with no container yet + 13: {kind: "wireguard"}, // a VPN + 14: {kind: "vxlan"}, // a tunnel to other machines (flannel.1) + 15: {kind: "bridge"}, // a bridge over a VLAN of the card + 16: {kind: "vlan", master: 15}, // + 17: {kind: "bridge"}, // a bridge with a bridge as its port + 18: {kind: "bridge", master: 17}, // + 19: {kind: "tun"}, // a VPN's tun + 20: {kind: "macvlan"}, // a macvlan over the card + } + for oif, want := range map[uint32]bool{ + 1: false, 2: false, 3: true, 4: true, 5: true, 6: true, 9: false, 12: true, + 13: false, 14: false, 15: false, 17: false, 19: false, 20: false, + 99: false, // not in the table: linkInsideMachine refuses it before asking + } { + if got := insideMachine(links, oif); got != want { + t.Errorf("link %d (%q): inside this machine = %v, want %v", oif, links[oif].kind, got, want) + } + } +} + +// A link dump's answers, written out here: the kind and master of each link, +// NLMSG_DONE ends it, and a dump that ended on an error (the status NLMSG_DONE +// carries) or was interrupted is not taken as the whole table. +// +// MUTATION: return done at NLMSG_DONE without reading its status. +// RESULT: red — a dump that stopped on EMSGSIZE, missing its later links, is +// taken as the table. +func TestReadLinkDumpReadsKindsMastersAndTheDumpsStatus(t *testing.T) { + ne := binary.NativeEndian + attr := func(typ uint16, val []byte) []byte { + b := make([]byte, (unix.SizeofRtAttr+len(val)+3)&^3) + ne.PutUint16(b[0:], uint16(unix.SizeofRtAttr+len(val))) + ne.PutUint16(b[2:], typ) + copy(b[unix.SizeofRtAttr:], val) + return b + } + u32 := func(v uint32) []byte { b := make([]byte, 4); ne.PutUint32(b, v); return b } + link := func(seq, idx, master uint32, kind string, flags uint16) syscall.NetlinkMessage { + d := make([]byte, unix.SizeofIfInfomsg) + ne.PutUint32(d[4:], idx) + if kind != "" { + d = append(d, attr(unix.IFLA_LINKINFO, attr(unix.IFLA_INFO_KIND, append([]byte(kind), 0)))...) + } + if master != 0 { + d = append(d, attr(unix.IFLA_MASTER, u32(master))...) + } + return syscall.NetlinkMessage{Header: syscall.NlMsghdr{Type: unix.RTM_NEWLINK, Flags: flags, Seq: seq}, Data: d} + } + doneWith := func(seq uint32, errno syscall.Errno) syscall.NetlinkMessage { + return syscall.NetlinkMessage{Header: syscall.NlMsghdr{Type: unix.NLMSG_DONE, Seq: seq}, Data: u32(uint32(-int32(errno)))} + } + + links := map[uint32]linkInfo{} + done, intr, err := readLinkDump([]syscall.NetlinkMessage{ + link(7, 3, 0, "bridge", 0), link(7, 4, 3, "veth", 0), link(7, 2, 0, "", 0), + link(8, 9, 0, "bridge", 0), // another request's answer + doneWith(7, 0), + }, 7, links) + if err != nil || !done || intr { + t.Fatalf("a whole dump: done %v interrupted %v err %v", done, intr, err) + } + want := map[uint32]linkInfo{3: {kind: "bridge"}, 4: {kind: "veth", master: 3}, 2: {}} + if len(links) != len(want) { + t.Fatalf("links %v, want %v", links, want) + } + for i, l := range want { + if links[i] != l { + t.Errorf("link %d: %+v, want %+v", i, links[i], l) + } + } + + if done, _, err := readLinkDump([]syscall.NetlinkMessage{link(7, 3, 0, "bridge", 0), doneWith(7, unix.EMSGSIZE)}, + 7, map[uint32]linkInfo{}); err == nil || done { + t.Errorf("a dump that ended on EMSGSIZE is an error, got done %v err %v", done, err) + } + if _, intr, err := readLinkDump([]syscall.NetlinkMessage{link(7, 3, 0, "bridge", unix.NLM_F_MULTI|unix.NLM_F_DUMP_INTR)}, + 7, map[uint32]linkInfo{}); err != nil || !intr { + t.Errorf("an answer marked interrupted marks the dump, got %v %v", intr, err) + } + if done, intr, err := readLinkDump([]syscall.NetlinkMessage{link(7, 3, 0, "bridge", 0)}, 7, map[uint32]linkInfo{}); done || intr || err != nil { + t.Errorf("a batch without NLMSG_DONE is not the end: %v %v %v", done, intr, err) + } + errMsg := doneWith(7, unix.EPERM) + errMsg.Header.Type = unix.NLMSG_ERROR + if _, _, err := readLinkDump([]syscall.NetlinkMessage{errMsg}, 7, map[uint32]linkInfo{}); !errors.Is(err, unix.EPERM) { + t.Errorf("an NLMSG_ERROR in the dump is its error, got %v", err) + } +} + +const insideNetnsEnv = "PRIVATAE_TEST_INSIDE_NETNS" + +// THE KERNEL'S OWN ANSWER FOR NETWORKS INSIDE THIS MACHINE. The local egress +// lane keeps the person's LAN, but a container or VM on a network made on +// this machine is a service on this machine, reached around its switches the +// same way loopback is. In a private user and network namespace this builds +// the networks Docker, libvirt and Calico make (a bridge whose only port is a +// veth, a bridge whose ports are a dummy and nothing else, a bridge with no +// port yet, a veth routed to directly, and the same with an IPv6 range and a +// public range), and links that leave the machine: a bridge over a VLAN, +// which stands for a bridge over the network card; a route straight over a +// dummy, which stands for the card itself; veths that carry this machine's +// main-table default routes, as a container's own veth does; and routes +// through a gateway. It asks the real rtnetlink lookup, the local egress lane +// on an unanchored box, a coder run's proxy and the backup lanes' wall. +// +// MUTATION: return routeLeaves for every unicast route (drop the link check). +// RESULT: red — every inside address passes the local lane, the public one +// passes the coder lane. +// MUTATION: count "dummy" as an outward port. +// RESULT: red — the libvirt-style bridge is the LAN. +// MUTATION: drop the main-table default-route links from the judgement. +// RESULT: red — from a machine that is itself a container, the network on its +// own veths (IPv4, a multipath default's second veth, IPv6) is this machine. +// MUTATION: skip a default route's RTA_MULTIPATH nexthops. +// RESULT: red — both veths of the multipath default are inside. +// MUTATION: count a default route in any table, not only main. +// RESULT: red — the veth whose default is in table 100 leaves. +// MUTATION: judge the out link of a route through a gateway too. +// RESULT: red — from a machine that is itself a container, the internet +// (IPv4 and IPv6) and a LAN behind a router VM are this machine, and every +// lane refuses them. +func TestNetworksInsideThisMachineAreThisMachine(t *testing.T) { + if os.Getenv(insideNetnsEnv) != "1" { + if _, err := exec.LookPath("ip"); err != nil { + if _, err := os.Stat("/usr/sbin/ip"); err != nil { + t.Skip("needs ip(8)") + } + } + cmd := exec.Command(os.Args[0], "-test.run=^TestNetworksInsideThisMachineAreThisMachine$", "-test.v", "-test.count=1") + cmd.Env = append(os.Environ(), insideNetnsEnv+"=1") + cmd.SysProcAttr = &syscall.SysProcAttr{ + Cloneflags: syscall.CLONE_NEWUSER | syscall.CLONE_NEWNET, + UidMappings: []syscall.SysProcIDMap{{ContainerID: 0, HostID: os.Getuid(), Size: 1}}, + GidMappings: []syscall.SysProcIDMap{{ContainerID: 0, HostID: os.Getgid(), Size: 1}}, + } + out, err := cmd.CombinedOutput() + if cmd.ProcessState == nil { + t.Skipf("no private user and network namespace here: %v", err) + } + t.Logf("in a private network namespace:\n%s", out) + if strings.Contains(string(out), "--- SKIP") { + t.Skip("the namespaced run could not build its networks") + } + if err != nil { + t.Fatalf("the namespaced run failed: %v", err) + } + if !strings.Contains(string(out), "--- PASS: TestNetworksInsideThisMachineAreThisMachine") { + t.Fatal("the namespaced run did not run") + } + return + } + + ipBin, err := exec.LookPath("ip") + if err != nil { + ipBin = "/usr/sbin/ip" + } + for _, args := range [][]string{ + {"link", "set", "lo", "up"}, + // Docker: a bridge whose only port is a container's veth, with an IPv6 + // range too, and a second network on a public range. + {"link", "add", "br-ctr", "type", "bridge"}, + {"link", "add", "vh0", "type", "veth", "peer", "name", "vc0"}, + {"link", "set", "vh0", "master", "br-ctr"}, + {"addr", "add", "172.31.5.1/24", "dev", "br-ctr"}, + {"-6", "addr", "add", "2001:db8:5::1/64", "dev", "br-ctr", "nodad"}, + {"link", "add", "br-pub", "type", "bridge"}, + {"link", "add", "vh1", "type", "veth", "peer", "name", "vc1"}, + {"link", "set", "vh1", "master", "br-pub"}, + {"addr", "add", "203.0.113.1/24", "dev", "br-pub"}, + // libvirt: a bridge whose port is a dummy. + {"link", "add", "br-vm", "type", "bridge"}, + {"link", "add", "dvm0", "type", "dummy"}, + {"link", "set", "dvm0", "master", "br-vm"}, + {"addr", "add", "192.168.122.1/24", "dev", "br-vm"}, + // A Docker network no container has joined yet. + {"link", "add", "br-empty", "type", "bridge"}, + {"addr", "add", "172.29.0.1/24", "dev", "br-empty"}, + // Calico: a pod's veth, routed to directly (the route is added once + // the link is up, below). + {"link", "add", "cali0", "type", "veth", "peer", "name", "cali0p"}, + // The LAN: a bridge over a VLAN of a card, and the card itself. + {"link", "add", "dlan0", "type", "dummy"}, + {"link", "add", "link", "dlan0", "name", "dlan0.7", "type", "vlan", "id", "7"}, + {"link", "add", "br-lan", "type", "bridge"}, + {"link", "set", "dlan0.7", "master", "br-lan"}, + {"addr", "add", "10.77.0.1/24", "dev", "br-lan"}, + // The way out through a veth: this machine is itself a container, and + // its default routes go through its veths to a gateway, the machine it + // runs on: IPv4 over two (a multipath default), IPv6 over a third. A + // veth that carries a main-table default leads out of this machine, so + // the network on it is the LAN, not inside. + {"link", "add", "ctr0", "type", "veth", "peer", "name", "ctr0p"}, + {"addr", "add", "10.88.0.2/24", "dev", "ctr0"}, + {"link", "add", "ctr1", "type", "veth", "peer", "name", "ctr1p"}, + {"addr", "add", "10.89.0.2/24", "dev", "ctr1"}, + {"link", "add", "ctr6", "type", "veth", "peer", "name", "ctr6p"}, + {"-6", "addr", "add", "2001:db8:88::2/64", "dev", "ctr6", "nodad"}, + // A veth with a default route in another table only: still inside. + {"link", "add", "pbr0", "type", "veth", "peer", "name", "pbr0p"}, + {"addr", "add", "10.91.0.2/24", "dev", "pbr0"}, + // A range routed through a veth with this machine's own address on it + // as the gateway: the kernel answers it as on the link, so it is + // inside, whatever the gateway. + {"link", "add", "cilh", "type", "veth", "peer", "name", "ciln"}, + {"addr", "add", "10.66.0.191/32", "dev", "cilh"}, + } { + if out, err := exec.Command(ipBin, args...).CombinedOutput(); err != nil { + t.Skipf("ip %s: %v %s", strings.Join(args, " "), err, out) + } + } + for _, l := range []string{"br-ctr", "vh0", "vc0", "br-pub", "vh1", "vc1", "br-vm", "dvm0", "br-empty", + "cali0", "cali0p", "dlan0", "dlan0.7", "br-lan", "ctr0", "ctr0p", "ctr1", "ctr1p", "ctr6", "ctr6p", + "pbr0", "pbr0p", "cilh", "ciln"} { + if out, err := exec.Command(ipBin, "link", "set", l, "up").CombinedOutput(); err != nil { + t.Skipf("ip link set %s up: %v %s", l, err, out) + } + } + // The card: a link with no kind cannot be made here, so a route over the + // VLAN's lower link stands for it. A dummy as the route's own link leaves: + // only a bridge's ports are read for dummies. + for _, args := range [][]string{ + {"route", "add", "172.30.9.0/24", "dev", "cali0"}, + {"route", "add", "192.168.50.0/24", "dev", "dlan0"}, + // Through a gateway on an inside link, the packet goes where that + // gateway sends it: the internet from a container, a LAN behind a + // router VM on a bridge of taps. + {"route", "add", "default", "nexthop", "via", "10.88.0.1", "dev", "ctr0", "nexthop", "via", "10.89.0.1", "dev", "ctr1"}, + {"-6", "route", "add", "default", "via", "2001:db8:88::1", "dev", "ctr6"}, + {"route", "add", "default", "via", "10.91.0.1", "dev", "pbr0", "table", "100"}, + {"route", "add", "192.168.60.0/24", "via", "192.168.122.254", "dev", "br-vm"}, + {"route", "add", "10.66.0.0/24", "via", "10.66.0.191", "dev", "cilh"}, + } { + if out, err := exec.Command(ipBin, args...).CombinedOutput(); err != nil { + t.Skipf("ip %s: %v %s", strings.Join(args, " "), err, out) + } + } + ownAddrReset() + + inside := []string{"172.31.5.9", "2001:db8:5::9", "203.0.113.9", "192.168.122.9", "172.29.0.9", "172.30.9.9", + "10.66.0.9", "10.91.0.9"} + leaves := []string{"10.77.0.9", "192.168.50.9", "198.51.100.9", "2001:db8:ff::9", "192.168.60.9", + "10.88.0.9", "10.89.0.9", "2001:db8:88::9"} + for _, a := range inside { + if got, err := ownRouteLookup(net.ParseIP(a)); err != nil || got != routeThisMachine { + t.Errorf("route lookup %s: got kind %d err %v, want this machine", a, got, err) + } + } + for _, a := range leaves { + if got, err := ownRouteLookup(net.ParseIP(a)); err != nil || got != routeLeaves { + t.Errorf("route lookup %s: got kind %d err %v, want leaves", a, got, err) + } + } + + // The local egress lane on an unanchored box: an inside address, by address + // and by name, is refused as this machine; the LAN passes. + t.Setenv("XDG_CONFIG_HOME", t.TempDir()) + if currentAnchor() != nil || boxMode() != modeLocal { + t.Fatal("the namespaced run must be a local, unanchored box") + } + names := map[string][]string{"db.example.test": {"172.31.5.9"}, "vm.example.test": {"192.168.122.9"}, + "nas.example.test": {"10.77.0.9"}} + stubEgressResolve(t, names) + ctx := context.Background() + for _, hp := range []string{"172.31.5.9:5432", "[2001:db8:5::9]:5432", "203.0.113.9:443", "192.168.122.9:22", + "172.29.0.9:80", "172.30.9.9:8080", "10.66.0.9:80", "10.91.0.9:80", "db.example.test:5432", "vm.example.test:22"} { + if got, why := egressVettedAddrs(ctx, hp); !strings.Contains(why, "that is this machine") { + t.Errorf("local egress: %s is on a network inside this machine and must be refused as this machine, got %v %q", hp, got, why) + } + } + for hp, want := range map[string]string{"10.77.0.9:445": "10.77.0.9:445", "192.168.50.9:631": "192.168.50.9:631", + "nas.example.test:445": "10.77.0.9:445", "198.51.100.9:443": "198.51.100.9:443", + "[2001:db8:ff::9]:443": "[2001:db8:ff::9]:443", "192.168.60.9:445": "192.168.60.9:445", + "10.88.0.9:80": "10.88.0.9:80", "10.89.0.9:80": "10.89.0.9:80", "[2001:db8:88::9]:80": "[2001:db8:88::9]:80"} { + if got, why := egressVettedAddrs(ctx, hp); why != "" || len(got) != 1 || got[0] != want { + t.Errorf("local egress: %s is the LAN and passes as %s: %v %q", hp, want, got, why) + } + } + // A coder run's proxy refuses the public range on an inside network too + // (its private ranges it refuses by class already). + if got, why := coderVettedAddr(ctx, "203.0.113.9:443", nil); !strings.Contains(why, "never this machine") { + t.Errorf("coder run: 203.0.113.9 is on a network inside this machine, got dial %q, %q", got, why) + } + // From a machine that is itself a container, the internet is through its + // veth: a coder run's proxy and the backup lanes still reach it. + if got, why := coderVettedAddr(ctx, "198.51.100.9:443", nil); why != "" || got != "198.51.100.9:443" { + t.Errorf("coder run: 198.51.100.9 is through the gateway and passes, got dial %q, %q", got, why) + } + if !backupAddrAllowed(net.ParseIP("2001:db8:ff::9")) || !backupAddrAllowed(net.ParseIP("198.51.100.9")) { + t.Error("backup: an address through the gateway leaves this machine and is allowed") + } +} diff --git a/client-go/ownaddr_linux.go b/client-go/ownaddr_linux.go index 0bae0b3..29d2c8e 100644 --- a/client-go/ownaddr_linux.go +++ b/client-go/ownaddr_linux.go @@ -7,8 +7,11 @@ import ( "errors" "fmt" "net" + "strings" + "sync" "sync/atomic" "syscall" + "time" "golang.org/x/sys/unix" ) @@ -23,6 +26,12 @@ var netlinkRouteSeq atomic.Uint32 // connection from this process (same uid, no mark, no bound device, which is // how the egress dialers dial). // +// A unicast route straight onto a link that stays inside this machine, with +// no gateway, is this machine too (insideMachine): a container's or VM's +// address on a bridge network Docker, Podman, LXD or libvirt made here +// reaches a service on this machine. A route through a gateway leaves, and +// so does one out of a link a main-table default route goes out of. +// // No route at all (ENETUNREACH, EHOSTUNREACH, or EACCES from a prohibit // route) is routeNowhere: the kernel delivers such a packet nowhere, locally // included, and the dial fails on its own. So does an IPv6 lookup on a kernel @@ -36,18 +45,11 @@ func netlinkRouteKind(ip net.IP) (routeKind, error) { if addr == nil { return 0, errors.New("not an IP address") } - fd, err := unix.Socket(unix.AF_NETLINK, unix.SOCK_RAW|unix.SOCK_CLOEXEC, unix.NETLINK_ROUTE) + fd, err := netlinkSocket() if err != nil { return 0, fmt.Errorf("route lookup: %w", err) } defer unix.Close(fd) - tv := unix.Timeval{Sec: 2} - if err := unix.SetsockoptTimeval(fd, unix.SOL_SOCKET, unix.SO_RCVTIMEO, &tv); err != nil { - return 0, fmt.Errorf("route lookup: %w", err) - } - if err := unix.Bind(fd, &unix.SockaddrNetlink{Family: unix.AF_NETLINK}); err != nil { - return 0, fmt.Errorf("route lookup: %w", err) - } // nlmsghdr | rtmsg | rtattr(RTA_DST) + address, each 4-byte aligned. attrLen := unix.SizeofRtAttr + len(addr) @@ -110,12 +112,405 @@ func netlinkRouteKind(ip net.IP) (routeKind, error) { if len(m.Data) < unix.SizeofRtMsg { return 0, errors.New("route lookup: short route message") } - return routeKindOfType(m.Data[7]), nil // rtm_type + kind := routeKindOfType(m.Data[7]) // rtm_type + if kind != routeLeaves { + return kind, nil + } + oif, gw, err := routeOutLink(&m) + if err != nil { + return 0, fmt.Errorf("route lookup: %w", err) + } + if oif == 0 { + return routeLeaves, nil // no single link named (a multipath route) + } + if gw { + // Handed to a gateway, the packet goes where that machine + // sends it on: the internet, from the machine a container + // runs on; the LAN behind a router VM. The destination is + // not on the out link, so the link says nothing about it. + return routeLeaves, nil + } + inside, err := linkInsideMachine(oif) + if err != nil { + return 0, fmt.Errorf("route lookup: %w", err) + } + if inside { + return routeThisMachine, nil + } + return routeLeaves, nil } } } } +// netlinkSocket opens an rtnetlink socket that gives up on a reply after 2 s. +func netlinkSocket() (int, error) { + fd, err := unix.Socket(unix.AF_NETLINK, unix.SOCK_RAW|unix.SOCK_CLOEXEC, unix.NETLINK_ROUTE) + if err != nil { + return -1, err + } + tv := unix.Timeval{Sec: 2} + if err := unix.SetsockoptTimeval(fd, unix.SOL_SOCKET, unix.SO_RCVTIMEO, &tv); err != nil { + unix.Close(fd) + return -1, err + } + if err := unix.Bind(fd, &unix.SockaddrNetlink{Family: unix.AF_NETLINK}); err != nil { + unix.Close(fd) + return -1, err + } + return fd, nil +} + +// routeOutLink reads the link a route answer sends the packet out of +// (RTA_OIF), 0 when the answer names none, and whether the answer hands the +// packet to a gateway (RTA_GATEWAY, or RTA_VIA for a gateway of the other +// family) rather than to the destination on the link itself. The kernel +// answers a route whose gateway is one of this machine's own addresses as on +// the link, with no gateway, so the link decides there. +func routeOutLink(m *syscall.NetlinkMessage) (oif uint32, gw bool, err error) { + attrs, err := syscall.ParseNetlinkRouteAttr(m) + if err != nil { + return 0, false, err + } + for _, a := range attrs { + switch a.Attr.Type { + case unix.RTA_OIF: + if len(a.Value) < 4 { + return 0, false, errors.New("short output link attribute") + } + oif = binary.NativeEndian.Uint32(a.Value) + case unix.RTA_GATEWAY, unix.RTA_VIA: + gw = true + } + } + return oif, gw, nil +} + +// linkInfo is what insideMachine reads of a link. +type linkInfo struct { + kind string // IFLA_INFO_KIND ("veth", "bridge", "tun", "vlan", ...); "" for a network card or loopback + master uint32 // IFLA_MASTER: the bridge or bond it is a port of; 0 for none +} + +// innerPortKinds are the bridge ports that keep a bridge inside this machine: +// a veth (a container's link), a tun (the kernel kind of a tap, a VM's network +// card) and a dummy (libvirt's placeholder port). +var innerPortKinds = map[string]bool{"veth": true, "tun": true, "dummy": true} + +// insideMachine reports whether a packet sent out of link oif stays on this +// machine: oif is a veth, whose other end is a container's network namespace +// here, or a bridge whose every port is an inner port, which is the network +// Docker, Podman, LXD and libvirt each make for their containers and VMs (a +// bridge with no port yet included). A bridge with any other port (a network +// card, a bond, a VLAN) is the person's LAN, and every other link leaves this +// machine: a network card, a VPN (wireguard, tun), a tunnel to other machines +// (vxlan, ipip). +func insideMachine(links map[uint32]linkInfo, oif uint32) bool { + switch links[oif].kind { + case "veth": + return true + case "bridge": + for idx, l := range links { + if idx != oif && l.master == oif && !innerPortKinds[l.kind] { + return false + } + } + return true + } + return false +} + +// The link table and the links a default route goes out of, re-read +// together at most every ownAddrTTL, as the interface addresses are. A link +// missing from a cached table (a network made since) reads both again; one +// missing from a fresh table cannot be judged. A failed read of either is an +// error, and the caller refuses. +var linkCache struct { + mu sync.Mutex + at time.Time + links map[uint32]linkInfo + wayOut map[uint32]bool +} + +// linkInsideMachine: insideMachine for oif, unless a main-table default route +// goes out of oif. Such a link is this machine's way out to the world, not a +// network inside it: a bastion that is itself in a container reaches its LAN +// (or the machine it runs on and its other containers) over its own veth, +// and a host whose default route goes through a router VM reaches that VM +// over the bridge of taps it shares with it. +func linkInsideMachine(oif uint32) (bool, error) { + linkCache.mu.Lock() + defer linkCache.mu.Unlock() + fresh := linkCache.at.IsZero() || time.Since(linkCache.at) >= ownAddrTTL + if !fresh { + if _, ok := linkCache.links[oif]; !ok { + fresh = true + } + } + if fresh { + links, err := dumpLinks() + if err != nil { + return false, err + } + wayOut, err := dumpDefaultRouteLinks() + if err != nil { + return false, err + } + linkCache.links, linkCache.wayOut, linkCache.at = links, wayOut, time.Now() + } + if _, ok := linkCache.links[oif]; !ok { + return false, fmt.Errorf("link %d is not in the link table", oif) + } + if linkCache.wayOut[oif] { + return false, nil + } + return insideMachine(linkCache.links, oif), nil +} + +// dumpLinks lists every link with its kind and master (RTM_GETLINK, the dump +// `ip -d link` reads). +func dumpLinks() (map[uint32]linkInfo, error) { + var links map[uint32]linkInfo + err := dumpRetried("link dump", unix.RTM_GETLINK, unix.SizeofIfInfomsg, func() func(*syscall.NetlinkMessage) error { + links = map[uint32]linkInfo{} + return func(m *syscall.NetlinkMessage) error { return addLink(links, m) } + }) + if err != nil { + return nil, err + } + return links, nil +} + +// dumpDefaultRouteLinks lists the links an IPv4 (0.0.0.0/0) or IPv6 (::/0) +// unicast default route in the main table goes out of, every nexthop's link +// of a multipath default included. One RTM_GETROUTE dump of family AF_UNSPEC +// answers both families (rtnetlink dumps each family's routes for it), one +// round trip where two per-family dumps would be two. No single-address +// lookup can stand in: RTM_GETROUTE without a dump answers for a destination, +// never for a route, and the destination 0.0.0.0 is answered as this machine. +func dumpDefaultRouteLinks() (map[uint32]bool, error) { + var out map[uint32]bool + err := dumpRetried("route dump", unix.RTM_GETROUTE, unix.SizeofRtMsg, func() func(*syscall.NetlinkMessage) error { + out = map[uint32]bool{} + return func(m *syscall.NetlinkMessage) error { return addDefaultRouteLinks(out, m) } + }) + if err != nil { + return nil, err + } + return out, nil +} + +// dumpRetried takes a dump (netlinkDump), and takes it again from the start +// when the kernel marks it interrupted (the table changed while it ran), up +// to three times; start gives each attempt an empty table to fill. +func dumpRetried(what string, msgType uint16, hdrLen int, start func() func(*syscall.NetlinkMessage) error) error { + for try := 0; ; try++ { + interrupted, err := netlinkDump(msgType, hdrLen, start()) + if err != nil { + return fmt.Errorf("%s: %w", what, err) + } + if !interrupted || try == 2 { + return nil + } + } +} + +// netlinkDump sends one dump request of msgType, its family header hdrLen +// bytes of zero (every family, everything), and hands each answer to add. +func netlinkDump(msgType uint16, hdrLen int, add func(*syscall.NetlinkMessage) error) (bool, error) { + fd, err := netlinkSocket() + if err != nil { + return false, err + } + defer unix.Close(fd) + total := unix.SizeofNlMsghdr + hdrLen + req := make([]byte, total) + ne := binary.NativeEndian + seq := netlinkRouteSeq.Add(1) + ne.PutUint32(req[0:], uint32(total)) + ne.PutUint16(req[4:], msgType) + ne.PutUint16(req[6:], unix.NLM_F_REQUEST|unix.NLM_F_DUMP) + ne.PutUint32(req[8:], seq) + if err := unix.Sendto(fd, req, 0, &unix.SockaddrNetlink{Family: unix.AF_NETLINK}); err != nil { + return false, err + } + interrupted := false + buf := make([]byte, 65536) + for { + n, from, err := unix.Recvfrom(fd, buf, 0) + if errors.Is(err, unix.EINTR) { + continue + } + if err != nil { + return false, err + } + if sa, ok := from.(*unix.SockaddrNetlink); !ok || sa.Pid != 0 { + continue + } + msgs, err := syscall.ParseNetlinkMessage(buf[:n]) + if err != nil { + return false, err + } + done, intr, err := readDump(msgs, seq, add) + if err != nil { + return false, err + } + interrupted = interrupted || intr + if done { + return interrupted, nil + } + } +} + +// readDump hands one batch of a dump's answers to add: done at NLMSG_DONE, +// interrupted when the kernel marks an answer NLM_F_DUMP_INTR. NLMSG_DONE +// carries the dump's own status, and a dump that stopped on an error (one the +// kernel could not fit, say) ends with that error in it and the answers after +// it missing, so it is an error here, as an NLMSG_ERROR is. +func readDump(msgs []syscall.NetlinkMessage, seq uint32, add func(*syscall.NetlinkMessage) error) (done, interrupted bool, err error) { + ne := binary.NativeEndian + for i := range msgs { + m := &msgs[i] + if m.Header.Seq != seq { + continue + } + if m.Header.Flags&unix.NLM_F_DUMP_INTR != 0 { + interrupted = true + } + switch m.Header.Type { + case unix.NLMSG_DONE: + if len(m.Data) >= 4 { + if errno := unix.Errno(-int32(ne.Uint32(m.Data[0:4]))); errno != 0 { + return false, false, errno + } + } + return true, interrupted, nil + case unix.NLMSG_ERROR: + if len(m.Data) >= 4 { + if errno := unix.Errno(-int32(ne.Uint32(m.Data[0:4]))); errno != 0 { + return false, false, errno + } + } + return false, false, errors.New("ended with no answer") + default: + if err := add(m); err != nil { + return false, false, err + } + } + } + return false, interrupted, nil +} + +// readLinkDump is readDump for a link dump, into links. +func readLinkDump(msgs []syscall.NetlinkMessage, seq uint32, links map[uint32]linkInfo) (done, interrupted bool, err error) { + return readDump(msgs, seq, func(m *syscall.NetlinkMessage) error { return addLink(links, m) }) +} + +// addLink reads a link's index, kind and master into links. +func addLink(links map[uint32]linkInfo, m *syscall.NetlinkMessage) error { + if m.Header.Type != unix.RTM_NEWLINK { + return nil + } + if len(m.Data) < unix.SizeofIfInfomsg { + return errors.New("short link message") + } + ne := binary.NativeEndian + idx := ne.Uint32(m.Data[4:8]) // ifi_index + attrs, err := syscall.ParseNetlinkRouteAttr(m) + if err != nil { + return err + } + var li linkInfo + for _, a := range attrs { + switch a.Attr.Type { + case unix.IFLA_MASTER: + if len(a.Value) >= 4 { + li.master = ne.Uint32(a.Value) + } + case unix.IFLA_LINKINFO: + li.kind = linkInfoKind(a.Value) + } + } + links[idx] = li + return nil +} + +// addDefaultRouteLinks adds to out the links an IPv4 or IPv6 unicast default +// route of the main table goes out of: its RTA_OIF, and each nexthop's link +// in its RTA_MULTIPATH. Every other route is passed over. +func addDefaultRouteLinks(out map[uint32]bool, m *syscall.NetlinkMessage) error { + if m.Header.Type != unix.RTM_NEWROUTE { + return nil + } + if len(m.Data) < unix.SizeofRtMsg { + return errors.New("short route message") + } + family, dstLen, table, rtype := m.Data[0], m.Data[1], uint32(m.Data[4]), m.Data[7] + if (family != unix.AF_INET && family != unix.AF_INET6) || dstLen != 0 || rtype != unix.RTN_UNICAST { + return nil + } + attrs, err := syscall.ParseNetlinkRouteAttr(m) + if err != nil { + return err + } + ne := binary.NativeEndian + var links []uint32 + for _, a := range attrs { + switch a.Attr.Type { + case unix.RTA_TABLE: // the table's id when it does not fit rtm_table + if len(a.Value) >= 4 { + table = ne.Uint32(a.Value) + } + case unix.RTA_OIF: + if len(a.Value) >= 4 { + links = append(links, ne.Uint32(a.Value)) + } + case unix.RTA_MULTIPATH: // struct rtnexthop, each followed by its own attributes + for b := a.Value; len(b) >= unix.SizeofRtNexthop; { + l := int(ne.Uint16(b[0:2])) + if l < unix.SizeofRtNexthop || l > len(b) { + return errors.New("bad multipath nexthop") + } + links = append(links, ne.Uint32(b[4:8])) // rtnh_ifindex + next := (l + 3) &^ 3 + if next >= len(b) { + break + } + b = b[next:] + } + } + } + if table != unix.RT_TABLE_MAIN { + return nil + } + for _, l := range links { + if l != 0 { + out[l] = true + } + } + return nil +} + +// linkInfoKind reads IFLA_INFO_KIND out of a nested IFLA_LINKINFO. +func linkInfoKind(b []byte) string { + ne := binary.NativeEndian + for len(b) >= unix.SizeofRtAttr { + l := int(ne.Uint16(b[0:2])) + if l < unix.SizeofRtAttr || l > len(b) { + return "" + } + if ne.Uint16(b[2:4]) == unix.IFLA_INFO_KIND { + return strings.TrimRight(string(b[unix.SizeofRtAttr:l]), "\x00") + } + next := (l + 3) &^ 3 + if next > len(b) { + return "" + } + b = b[next:] + } + return "" +} + // routeKindOfType reads a route's rtm_type: only a unicast route leaves this // machine. Local, broadcast, anycast, multicast and every other type are // delivered here or cannot be told apart from it, so they are this machine. diff --git a/client-go/ownaddr_other.go b/client-go/ownaddr_other.go index b2eb6a9..408b31f 100644 --- a/client-go/ownaddr_other.go +++ b/client-go/ownaddr_other.go @@ -4,4 +4,5 @@ package main // No route lookup is built for this platform: ownRouteLookup stays nil and // ownDestination (ownaddr.go) compares against the interface addresses only. -// An address delivered locally through a route alone (AnyIP) is not seen here. +// An address delivered locally through a route alone (AnyIP) is not seen here, +// nor a container's or VM's address on a network made on this machine. diff --git a/client-go/ownaddr_test.go b/client-go/ownaddr_test.go index 9d75d56..3a3d086 100644 --- a/client-go/ownaddr_test.go +++ b/client-go/ownaddr_test.go @@ -136,7 +136,7 @@ func TestCoderEgressRefusesAnAddressTheKernelDeliversHere(t *testing.T) { // The anchored egress lane refuses this machine too: an address on its NIC // and an address a `local` route delivers here, in every spelling, and a -// failed lookup. A neighbour passes, and a local box keeps the lane it had. +// failed lookup. A neighbour passes, and a local box refuses this machine too. // // MUTATION: drop the ownDestination check from egressVettedAddr. // RESULT: red — the machine's own addresses are vetted and dialled. @@ -161,8 +161,11 @@ func TestAnchoredEgressRefusesThisMachinesOwnAddresses(t *testing.T) { t.Fatal("a failed route lookup refuses on an anchored box") } t.Setenv("XDG_CONFIG_HOME", t.TempDir()) - if addr, why := egressVettedAddr(ctx, "198.51.100.7:8080"); why != "" || addr != "198.51.100.7:8080" { - t.Fatalf("a local box keeps its lane: %q %q", addr, why) + stubOwnAddrs(t, []string{"127.0.0.1/8", "198.51.100.7/24"}, localRange("203.0.113.0/24")) + for _, hp := range []string{"198.51.100.7:8080", "203.0.113.5:80"} { + if addr, why := egressVettedAddr(ctx, hp); why == "" || addr != "" { + t.Errorf("%s is this machine and must be refused on a local box too, got %q", hp, addr) + } } } diff --git a/client-go/pair.go b/client-go/pair.go index cc4cea6..7c928b0 100644 --- a/client-go/pair.go +++ b/client-go/pair.go @@ -193,7 +193,7 @@ func pairPrecheck(cfgs []*Config, handle string, replace, forceActive bool) (exi if boxMode() == modeLocal && len(cfgs) > 0 { return nil, "", fmt.Errorf("this bastion is in local mode and already serves %s — a local bastion serves one host. "+ "To rotate that host's credentials: `privatae-bastion pair --replace `. "+ - "To serve several hosts, reinstall as a networked bastion (install.sh --networked, root) and anchor it.", + "To serve several hosts, reinstall as a networked bastion (install.sh --networked, root) and anchor it", cfgs[0].HostName) } if forceActive { @@ -231,40 +231,76 @@ func cmdPair(token, redeemURL, hostCAFile string, replace bool) error { cfg, all = c, fresh } } - fmt.Printf("\nThis machine is networked and anchored to %s: every terminal and every write\n", cfg.HostName) - fmt.Println("waits for your approval, and any later host waits for this one to approve it.") - fmt.Printf("Enrol your second factor now, so you can approve here:\n %s\n\n", stepupEnrollHint()) + } + pairReport(os.Stdout, cfg, all, anchored) + return nil +} + +// pairReport is what `pair` prints once the pairing stands. Whatever the form +// (anchored just now, over TCP, over a unix socket, pending, or over the +// tunnel), it ends by saying to enrol a second factor when this pairing has +// none and no approval device is pinned: without one, a local machine installs +// a folder grant on the host's signed grant alone, and an anchored one has no +// way to approve anything the host asks for. +func pairReport(w io.Writer, cfg *Config, all []*Config, anchored bool) { + enrol := "" + if !secondFactorAvailable(cfg.dcnDir()) { + enrol = stepupEnrollHint(cfg.HostName, len(all)) + } + defer func() { pairEnrolNote(w, cfg.HostName, enrol) }() + if anchored { + fmt.Fprintf(w, "\nThis machine is networked and anchored to %s: every terminal and every write\n", cfg.HostName) + fmt.Fprintln(w, "waits for your approval, and any later host waits for this one to approve it.") + if enrol != "" { + fmt.Fprintf(w, "Enrol your second factor now, so you can approve here:\n %s\n\n", enrol) + enrol = "" + } } if cfg.tcpTransport() && cfg.active() { - fmt.Printf("Paired with %s over TCP.\n\n", cfg.HostName) - fmt.Printf(" lanes: %s (mutual TLS, the Host's CA pinned, name %s)\n coder runs: none over TCP\n", cfg.LaneAddr, tcpServerName) - fmt.Printf(" local: http://%s\n\nStart (or restart) the bastion:\n privatae-bastion run\n", cfg.ListenAddr) - return nil + fmt.Fprintf(w, "Paired with %s over TCP.\n\n", cfg.HostName) + fmt.Fprintf(w, " lanes: %s (mutual TLS, the Host's CA pinned, name %s)\n coder runs: none over TCP\n", cfg.LaneAddr, tcpServerName) + fmt.Fprintf(w, " local: http://%s\n\nStart (or restart) the bastion:\n privatae-bastion run\n", cfg.ListenAddr) + return } if cfg.unixTransport() && cfg.active() { - fmt.Printf("Paired with %s, a Host on this machine.\n\n", cfg.HostName) - fmt.Printf(" lanes: %s (mutual TLS, the Host's CA pinned)\n coder runs: a socket per run in %s\n", cfg.LaneSocket, cfg.RunsDir) - fmt.Printf(" local: http://%s\n\nStart (or restart) the bastion:\n privatae-bastion run\n", cfg.ListenAddr) - return nil + fmt.Fprintf(w, "Paired with %s, a Host on this machine.\n\n", cfg.HostName) + fmt.Fprintf(w, " lanes: %s (mutual TLS, the Host's CA pinned)\n coder runs: a socket per run in %s\n", cfg.LaneSocket, cfg.RunsDir) + fmt.Fprintf(w, " local: http://%s\n\nStart (or restart) the bastion:\n privatae-bastion run\n", cfg.ListenAddr) + return } if !cfg.active() { - fmt.Printf("Paired with %s — PENDING.\n\n", cfg.HostName) - fmt.Println("This machine is anchored, so a new host waits until the anchor host approves it.") - fmt.Println("Nothing is started for it yet: no tunnel, no lanes. It shows on the anchor host's") - fmt.Println("Network app as a pending pairing; once approved the daemon opens its lanes by itself.") - return nil + fmt.Fprintf(w, "Paired with %s — PENDING.\n\n", cfg.HostName) + fmt.Fprintln(w, "This machine is anchored, so a new host waits until the anchor host approves it.") + fmt.Fprintln(w, "Nothing is started for it yet: no tunnel, no lanes. It shows on the anchor host's") + fmt.Fprintln(w, "Network app as a pending pairing; once approved the daemon opens its lanes by itself.") + return } - fmt.Printf("Paired with %s.\n\nOne `privatae-bastion run` serves every pairing:\n", cfg.HostName) + fmt.Fprintf(w, "Paired with %s.\n\nOne `privatae-bastion run` serves every pairing:\n", cfg.HostName) for _, c := range all { // The listen address is the CLI's loopback control port for that host // (add-folder, git-credential). The local secret that guards it is read // from the pairing file by those verbs and is never shown: there is no // operator client to hand it to since the MCP relay was removed. - fmt.Printf("\n %s\n local: http://%s\n", c.HostName, c.ListenAddr) + fmt.Fprintf(w, "\n %s\n local: http://%s\n", c.HostName, c.ListenAddr) } - fmt.Printf("\nStart (or restart) the tunnels:\n privatae-bastion run\n") - fmt.Printf("\ngit to your forge goes through this bastion — one-time setup:\n privatae-bastion git-setup\n") - return nil + fmt.Fprintf(w, "\nStart (or restart) the tunnels:\n privatae-bastion run\n") + fmt.Fprintf(w, "\ngit to your forge goes through this bastion — one-time setup:\n privatae-bastion git-setup\n") +} + +// pairEnrolNote says what a pairing with no second factor means on this kind +// of machine, and how to enrol one. hint "" says nothing. +func pairEnrolNote(w io.Writer, host, hint string) { + if hint == "" { + return + } + if currentAnchor() != nil { + fmt.Fprintf(w, "\nNo second factor is enrolled here for %s, so this machine cannot approve what it\n"+ + "asks for until you enrol one. Enrol it now:\n %s\n", host, hint) + return + } + fmt.Fprintf(w, "\nNo second factor is enrolled here for %s. Until you enrol one, a folder grant or a\n"+ + "move plan from it installs on its signed grant alone, and model pulls and root\n"+ + "sessions are refused. Enrol it now, so each needs a code from this machine:\n %s\n", host, hint) } // anchorFirstNetworkedPairing anchors a networked machine that has no anchor @@ -486,6 +522,10 @@ func pairWithTokenVia(token, redeemURL, hostCAFile string, replace, forceActive if _, err := hostTunnelAllowedIP(rr.HostTunnelIP); err != nil { return nil, nil, fmt.Errorf("redeem response: %w", err) } + // endpoint and endpoints are written into the tunnel's configuration text. + if err := checkEndpoints(append([]string{rr.Endpoint}, rr.Endpoints...)...); err != nil { + return nil, nil, fmt.Errorf("redeem response: %w", err) + } // Belt: the redeemed host_name is authoritative — if it lands on a pairing the // pre-redeem handle check missed, still refuse to overwrite it. if prev := pairingFor(cfgs, rr.HostName); prev != nil { diff --git a/client-go/pair_enrol_test.go b/client-go/pair_enrol_test.go new file mode 100644 index 0000000..21b13d4 --- /dev/null +++ b/client-go/pair_enrol_test.go @@ -0,0 +1,137 @@ +//go:build !windows + +// Tagged !windows: it borrows setupBox (anchor_test.go) and +// withMachineAuthority (claims_unix_test.go). + +package main + +// `pair` says to enrol a second factor after every pairing that leaves this +// machine without one, not only after it anchors a networked machine. On a +// local machine a pairing with no factor installs a folder grant on the host's +// signed grant alone, so the person has to be told at the moment they pair. + +import ( + "crypto/ed25519" + "encoding/base64" + "strings" + "testing" +) + +func pairReportOf(cfg *Config, all []*Config, anchored bool) string { + var b strings.Builder + pairReport(&b, cfg, all, anchored) + return b.String() +} + +func pairedHere(t *testing.T, host string, set func(*Config)) (*Config, []*Config) { + t.Helper() + all, err := loadPairings() + if err != nil { + t.Fatal(err) + } + cfg := pairingFor(all, host) + if cfg == nil { + t.Fatalf("%s is not paired", host) + } + if set != nil { + set(cfg) + } + return cfg, all +} + +func enrolFactor(t *testing.T, cfg *Config) { + t.Helper() + if err := stepupSave(cfg.dcnDir(), &stepupFile{SecretB32: "INVENTEDTESTAAAA", EnrolledAt: "2026-01-02T03:04:05Z"}); err != nil { + t.Fatal(err) + } + if !stepupEnrolled(cfg.dcnDir()) { + t.Fatal("the fixture factor does not read as enrolled") + } +} + +// MUTATION: print the hint only when the pairing anchored the machine. +// RESULT: red — the local, TCP, unix and pending forms say nothing. +func TestEveryPairingWithoutAFactorIsToldToEnrolOne(t *testing.T) { + for _, f := range []struct { + name string + set func(*Config) + }{ + {"over the tunnel", nil}, + {"over TCP", func(c *Config) { c.Transport, c.LaneAddr = transportTCP, "127.0.0.1:8811" }}, + {"over a unix socket", func(c *Config) { c.Transport, c.LaneSocket = transportUnix, "/run/privatae-host/lane.sock" }}, + } { + t.Run("local machine, "+f.name, func(t *testing.T) { + withMachineAuthority(t, false) + t.Setenv("XDG_CONFIG_HOME", t.TempDir()) + writePairing(t, tMember) + cfg, all := pairedHere(t, tMember, f.set) + out := pairReportOf(cfg, all, false) + if !strings.Contains(out, "\n privatae-bastion stepup-enroll\n") { + t.Fatalf("a local pairing with no factor must be told to enrol one, as its own user:\n%s", out) + } + if !strings.Contains(out, "signed grant alone") { + t.Fatalf("the hint says what no factor means here:\n%s", out) + } + enrolFactor(t, cfg) + if out := pairReportOf(cfg, all, false); strings.Contains(out, "stepup-enroll") { + t.Fatalf("a pairing that keeps its factor (a --replace) is not told to enrol again:\n%s", out) + } + }) + } + + t.Run("anchored by this pairing: said once", func(t *testing.T) { + withMachineAuthority(t, true) + setupBox(t) + cfg, all := pairedHere(t, tAnchor, nil) + out := pairReportOf(cfg, all, true) + if strings.Count(out, "stepup-enroll") != 1 || !strings.Contains(out, "\n sudo -H privatae-bastion stepup-enroll\n") { + t.Fatalf("the anchoring pairing names the factor once, with machine authority:\n%s", out) + } + }) + + t.Run("a pending host on an anchored machine", func(t *testing.T) { + withMachineAuthority(t, true) + setupBox(t) + writePairingState(t, tMember, pairingPending) + cfg, all := pairedHere(t, tMember, nil) + out := pairReportOf(cfg, all, false) + if !strings.Contains(out, "sudo -H privatae-bastion stepup-enroll --host "+tMember) || + !strings.Contains(out, "cannot approve") { + t.Fatalf("a second host on an anchored machine is told to enrol a factor for it, by name:\n%s", out) + } + }) + + t.Run("an approval device pinned is a way to approve", func(t *testing.T) { + withMachineAuthority(t, true) + setupBox(t) + writePairingState(t, tMember, pairingPending) + pub, _, _ := ed25519.GenerateKey(nil) + if err := pinDeviceKey(base64.StdEncoding.EncodeToString(pub)); err != nil { + t.Fatal(err) + } + cfg, all := pairedHere(t, tMember, nil) + if out := pairReportOf(cfg, all, false); strings.Contains(out, "stepup-enroll") { + t.Fatalf("with a device pinned there is a way to approve; no hint:\n%s", out) + } + }) +} + +// The line is run as whoever ran `pair`, on each platform. +func TestStepupEnrollHintIsRunAsWhoeverPaired(t *testing.T) { + for _, c := range []struct { + goos string + authority bool + paired int + want string + }{ + {"linux", false, 1, "privatae-bastion stepup-enroll"}, + {"linux", true, 1, "sudo -H privatae-bastion stepup-enroll"}, + {"darwin", true, 2, "sudo -H privatae-bastion stepup-enroll --host h1.example.test"}, + {"windows", true, 1, "privatae-bastion stepup-enroll (from this Administrator terminal)"}, + {"windows", false, 2, "privatae-bastion stepup-enroll --host h1.example.test"}, + } { + if got := stepupEnrollHintFor(c.goos, c.authority, "h1.example.test", c.paired); got != c.want { + t.Errorf("%s authority=%v paired=%d: %q, want %q", c.goos, c.authority, c.paired, got, c.want) + } + } +} diff --git a/client-go/pair_fp_test.go b/client-go/pair_fp_test.go index 37ca7fd..a98b58b 100644 --- a/client-go/pair_fp_test.go +++ b/client-go/pair_fp_test.go @@ -61,8 +61,8 @@ func TestTheOperatorCSRNamesTheFingerprintForItsSigner(t *testing.T) { // platformSignerRefusal is the platform's rule for the SANs of an operator // CSR it signs (the bridge's _checked_operator_csr): one name, a URI, -// urn:skema:hwfp: and a fingerprint of 16 to 128 base64url characters. -// Anything else is refused before anything is signed. +// the device-fingerprint prefix (fpSANPrefix) and a fingerprint of 16 to 128 +// base64url characters. Anything else is refused before anything is signed. var platformFPRe = regexp.MustCompile(`^[A-Za-z0-9_-]{16,128}$`) func platformSignerRefusal(csr *x509.CertificateRequest) string { diff --git a/client-go/pairing_ops.go b/client-go/pairing_ops.go index 97ac8f1..cac220e 100644 --- a/client-go/pairing_ops.go +++ b/client-go/pairing_ops.go @@ -114,6 +114,14 @@ func handlePairingGrantSync(ws *websocket.Conn, _ string, args []string) { pairingRefuse(ws, "grant_sync", t.Subject, "subject pairing was revoked — pair it again to re-approve") return } + // Approving would mark it active and start it with the endpoint its file + // holds, which this build refuses. The reason names a local file, so the + // host is told only what to do. + if c.invalid != "" { + anchorMu.Unlock() + pairingRefuse(ws, "grant_sync", t.Subject, "subject pairing is not served on this box: its file holds an endpoint this bastion refuses — pair it again to approve it") + return + } if a := currentAnchor(); a != nil && a.AnchorHost == c.HostName { anchorMu.Unlock() pairingRefuse(ws, "grant_sync", t.Subject, "subject is the anchor itself; the anchor holds no grant") @@ -143,6 +151,10 @@ func handlePairingGrantSync(ws *websocket.Conn, _ string, args []string) { started := false if wasPending { started = supervisorStart(c) + // Every Host on this box reads this pairing's state from a beat, so + // each beats now rather than up to a minute from now (beatnow.go). A + // re-sync of an active pairing changes no state, so it beats nothing. + kickBeats() } pairingResult(ws, "grant_sync", c.HostName, map[string]any{ "state": "ok", "subject": c.HostName, "caps": sortedCaps(t.Caps), @@ -209,6 +221,9 @@ func handlePairingGrantDrop(ws *websocket.Conn, _ string, args []string) { } anchorMu.Unlock() stopped := stopPairing(subject) + // Every other Host on this box learns of the revoke on its next beat, which + // is now rather than up to a minute from now (beatnow.go). + kickBeats() // A revoked pairing keeps no folder the person shared with it. sharesDropped := dropSharesOf(subject) == nil pairingResult(ws, "grant_drop", subject, map[string]any{ diff --git a/client-go/peers.go b/client-go/peers.go index 7150635..c84c321 100644 --- a/client-go/peers.go +++ b/client-go/peers.go @@ -57,7 +57,10 @@ func peersFor(selfHostName string) []peerAnnounce { } out := make([]peerAnnounce, 0, len(cfgs)) for _, c := range cfgs { - if c.HostName == "" || c.HostName == selfHostName { + // A pairing whose file this build refuses is never served, so it is + // no peer; pairingsReport (anchor.go) says why it is left out rather + // than sent as "invalid". + if c.HostName == "" || c.HostName == selfHostName || c.invalid != "" { continue } // "Since" is first_seen when the pairing carries it (0.16.0+), else the diff --git a/client-go/privilege.go b/client-go/privilege.go index f607b15..d3462d3 100644 --- a/client-go/privilege.go +++ b/client-go/privilege.go @@ -139,13 +139,28 @@ func restartHintFor(goos string) string { return "Restart the service so the change takes: systemctl restart privatae-bastion" } -// stepupEnrollHint is the stepup-enroll line to print after a networked -// machine is anchored: approvals on it need a second factor or a device. -func stepupEnrollHint() string { return stepupEnrollHintFor(runtime.GOOS) } - -func stepupEnrollHintFor(goos string) string { - if goos == "windows" { - return "privatae-bastion stepup-enroll (from this Administrator terminal)" - } - return "sudo -H privatae-bastion stepup-enroll" +// stepupEnrollHint is the stepup-enroll line `pair` prints for a pairing with +// no second factor. It is run as whoever ran `pair`, since that is whose state +// the pairing is in: with machine authority (an anchored or networked machine, +// a root install) through sudo -H or this Administrator terminal, and as the +// person's own account on a local install. --host is named when more than one +// host is paired, as stepup-enroll then requires. +func stepupEnrollHint(host string, paired int) string { + return stepupEnrollHintFor(runtime.GOOS, machineAuthorityHeld(), host, paired) +} + +func stepupEnrollHintFor(goos string, authority bool, host string, paired int) string { + cmd := "privatae-bastion stepup-enroll" + if paired > 1 { + cmd += " --host " + host + } + switch { + case goos == "windows" && authority: + return cmd + " (from this Administrator terminal)" + case goos == "windows": + return cmd + case authority: + return "sudo -H " + cmd + } + return cmd } diff --git a/client-go/publicname_test.go b/client-go/publicname_test.go index b3b6a5d..46b9c84 100644 --- a/client-go/publicname_test.go +++ b/client-go/publicname_test.go @@ -1,7 +1,6 @@ package main import ( - "bufio" "bytes" "os" "os/exec" @@ -18,7 +17,10 @@ import ( // harness images, or the host still read or write under that name. Each is // listed in keptOldNames with what holds it in place; removing one means // changing the other side first. Prose, comments and the changelog are never -// exceptions. +// exceptions: a kept identifier is set aside only in Go code, outside its +// comments (goCodeAndComments). In a comment, a Markdown file, a script or +// any other file it is a hit like the bare word, and is described instead +// (the device-fingerprint prefix, fpSANPrefix in pair.go). // // The forbidden word is assembled from bytes so that this file does not // contain it outside the exception list. @@ -75,6 +77,118 @@ func isNameByte(c byte) bool { return c == '_' || c >= '0' && c <= '9' || c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' } +// hasOldName reports whether text carries the old name at all, kept +// identifiers included. +func hasOldName(text string) bool { + return strings.Contains(strings.ToLower(text), oldName) +} + +// goCodeAndComments splits Go source into its lines twice over: code[i] is +// line i with its comment text blanked, comments[i] the same line with +// everything but comment text blanked. String, raw string and rune literals +// are code, so a "//" inside one does not start a comment, and a raw string +// spanning lines stays code on every line it covers. +func goCodeAndComments(src string) (code, comments []string) { + const ( + inCode = iota + inLineComment + inBlockComment + inString + inRawString + inRune + ) + codeB := []byte(src) + commB := []byte(src) + state := inCode + for i := 0; i < len(src); i++ { + c := src[i] + isComment := false + switch state { + case inCode: + switch { + case c == '/' && i+1 < len(src) && src[i+1] == '/': + state, isComment = inLineComment, true + case c == '/' && i+1 < len(src) && src[i+1] == '*': + // Both opener bytes are taken here: the opener's '*' cannot + // also start the closer, so "/*/" leaves the comment open. + codeB[i], codeB[i+1] = ' ', ' ' + i++ + state = inBlockComment + continue + case c == '"': + state = inString + case c == '`': + state = inRawString + case c == '\'': + state = inRune + } + case inLineComment: + if c == '\n' { + state = inCode + } else { + isComment = true + } + case inBlockComment: + isComment = true + if c == '*' && i+1 < len(src) && src[i+1] == '/' { + codeB[i], codeB[i+1] = ' ', ' ' + i++ + state = inCode + continue + } + case inString, inRune: + end := byte('"') + if state == inRune { + end = '\'' + } + switch { + case c == '\\' && i+1 < len(src) && src[i+1] != '\n': + commB[i], commB[i+1] = ' ', ' ' + i++ + continue + case c == end, c == '\n': + state = inCode + } + case inRawString: + if c == '`' { + state = inCode + } + } + if c == '\n' { + continue + } + if isComment { + codeB[i] = ' ' + } else { + commB[i] = ' ' + } + } + return strings.Split(string(codeB), "\n"), strings.Split(string(commB), "\n") +} + +// oldNameHits lists, as "line: text", every line of the published file rel +// that carries the old name. Kept identifiers are set aside only in the code +// of a Go file; in its comments and in every other file they count. +func oldNameHits(rel, src string) []string { + lines := strings.Split(src, "\n") + var hits []string + if !strings.HasSuffix(rel, ".go") { + for i, l := range lines { + if hasOldName(l) { + hits = append(hits, strconv.Itoa(i+1)+": "+strings.TrimSpace(l)) + } + } + return hits + } + code, comments := goCodeAndComments(src) + for i, l := range lines { + if lineHasOldName(code[i]) || hasOldName(comments[i]) { + hits = append(hits, strconv.Itoa(i+1)+": "+strings.TrimSpace(l)) + } + } + return hits +} + // publishedFiles lists the files the member repository receives: what git // tracks at the repository root (the publisher stages `git archive` of the // release commit). Outside a git checkout it walks the tree instead. @@ -127,19 +241,14 @@ func TestPublishedSourceCarriesNoOldInternalName(t *testing.T) { if err != nil { continue // a tracked file deleted in the working tree } - sc := bufio.NewScanner(bytes.NewReader(data)) - sc.Buffer(make([]byte, 0, 1<<20), 1<<24) - n := 0 - for sc.Scan() { - n++ - if lineHasOldName(sc.Text()) { - hits = append(hits, rel+":"+strconv.Itoa(n)+": "+strings.TrimSpace(sc.Text())) - } + for _, h := range oldNameHits(rel, string(data)) { + hits = append(hits, rel+":"+h) } } if len(hits) > 0 { - t.Errorf("the old internal name appears in %d published lines. Reword prose; an identifier "+ - "may be added to keptOldNames only when something outside this repository still reads it:\n %s", + t.Errorf("the old internal name appears in %d published lines. Reword prose and comments; an "+ + "identifier may be added to keptOldNames only when something outside this repository still "+ + "reads it, and is set aside only in Go code:\n %s", len(hits), strings.Join(hits, "\n ")) } } @@ -150,7 +259,7 @@ func TestOldNameCheckCatchesProseAndPassesKeptIdentifiers(t *testing.T) { up := strings.ToUpper(oldName) title := strings.ToUpper(oldName[:1]) + oldName[1:] for _, line := range []string{ - "// see " + oldName + " #649", + "// see " + oldName + " for the rest", "// the " + title + " server", up + "_NEW_VAR", keptOldNames[0] + "X", // a kept name with a suffix is a different name @@ -173,3 +282,34 @@ func TestOldNameCheckCatchesProseAndPassesKeptIdentifiers(t *testing.T) { } } } + +// Where a kept identifier is set aside: in Go code only. In a Go comment, the +// changelog, any other Markdown file or a script it is a hit like the bare +// word. A "//" inside a string, a raw string spanning lines and an escaped +// quote do not move the line between code and comment. +func TestKeptIdentifiersAreSetAsideOnlyInGoCode(t *testing.T) { + prefix := keptOldNames[len(keptOldNames)-1] // the fingerprint prefix + env := keptOldNames[0] + for _, c := range []struct { + rel, src string + hit bool + }{ + {"CHANGELOG.md", "- the fingerprint goes under `" + prefix + "`, as before\n", true}, + {"docs/claims.md", "the agent token is read from " + env + "\n", true}, + {"deploy/run.sh", "export " + env + "=x\n", true}, + {"a.go", "package a\n\n// names it " + prefix + "\n", true}, + {"a.go", "x := 1 /* " + env + " */\n", true}, + {"a.go", "x := 1 /*/ " + env + " */\n", true}, + {"a.go", "s := \"a\\\"b\" // " + env + "\n", true}, + {"a.go", "const p = \"" + prefix + "\"\n", false}, + {"a.go", "v := []string{\"" + env + "\"} // the host mints this tag\n", false}, + {"a.go", "u := \"https://example.com/" + env + "\"\n", false}, + {"a.go", "y := `\nname: x // " + env + "\n`\n", false}, + {"a.go", "r := '\"' + 1 // " + env + "\n", true}, + } { + got := oldNameHits(c.rel, c.src) + if (len(got) > 0) != c.hit { + t.Errorf("%s %q: hits %q, want a hit: %v", c.rel, c.src, got, c.hit) + } + } +} diff --git a/client-go/publishedrefs_test.go b/client-go/publishedrefs_test.go new file mode 100644 index 0000000..c3a3429 --- /dev/null +++ b/client-go/publishedrefs_test.go @@ -0,0 +1,63 @@ +package main + +import ( + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + "testing" +) + +// A "#NNN" in this repository names a pull request or an issue of the +// development repository, which a reader of the published source cannot open. +// On a forge that links such numbers it points at an unrelated item of the +// published repository instead. A comment or a changelog entry says why +// something is so in the sentence itself. +// Six or more digits is a colour (#000000), not a number anyone cites. +var prNumberRe = regexp.MustCompile(`(^|[^&A-Za-z0-9_])#[0-9]{2,5}\b`) + +func TestPublishedSourceCitesNoPullRequestNumber(t *testing.T) { + root, err := filepath.Abs("..") + if err != nil { + t.Fatal(err) + } + files := publishedFiles(t, root) + if len(files) < 50 { + t.Fatalf("found only %d files under %s; the scan is not looking at the repository", len(files), root) + } + var hits []string + for _, rel := range files { + data, err := os.ReadFile(filepath.Join(root, rel)) + if err != nil { + continue // a tracked file deleted in the working tree + } + for i, line := range strings.Split(string(data), "\n") { + if prNumberRe.MatchString(line) { + hits = append(hits, rel+":"+strconv.Itoa(i+1)+": "+strings.TrimSpace(line)) + } + } + } + if len(hits) > 0 { + t.Errorf("%d published lines cite a pull request or issue by number; state the reason instead:\n %s", + len(hits), strings.Join(hits, "\n ")) + } +} + +func TestPullRequestNumberCheck(t *testing.T) { + for line, want := range map[string]bool{ + "// refused here (review of #" + "100).": true, + "see #" + "1234 for the rest": true, + "#" + "42 at the start": true, + "## 0.55.0": false, + "&#" + "123; an HTML entity": false, + "color: #fff": false, + `stop-color="#000000"`: false, + "step #1 of 3": false, + "x#" + "12 glued to a name": false, + } { + if got := prNumberRe.MatchString(line); got != want { + t.Errorf("%q: matched %v, want %v", line, got, want) + } + } +} diff --git a/client-go/review_fixes_test.go b/client-go/review_fixes_test.go index a8e7338..d6ab69a 100644 --- a/client-go/review_fixes_test.go +++ b/client-go/review_fixes_test.go @@ -125,6 +125,10 @@ func setupBoxKeep(t *testing.T) ed25519.PrivateKey { // before any dial; the anchor's request is dialled. func TestEgressLaneGatedByCaps(t *testing.T) { grantMember(t, []string{"observe"}) + // The name and this machine's addresses are answered here, so the anchor's + // request is vetted the same way on a box with no network or DNS. + answerEgressNames(t, map[string][]string{"example.com": {"203.0.113.10"}}) + stubOwnAddrs(t, []string{"127.0.0.1/8", "::1/128"}, nil) dialed := false dialOut := func(ctx context.Context, network, addr string) (net.Conn, error) { dialed = true diff --git a/client-go/rootexec.go b/client-go/rootexec.go index 11cbd47..61d13c5 100644 --- a/client-go/rootexec.go +++ b/client-go/rootexec.go @@ -85,7 +85,9 @@ var ( rootExecMaxOutput = 4 << 20 rootExecMinOutput = 1 << 10 rootExecTranscriptMax = int64(64 << 20) // the local record keeps far more than the host sees - rootExecMaxGrantTTL = 365 * 24 * time.Hour + // The longest a root-command grant lasts before it has to be given again + // at this machine with the code. + rootExecMaxGrantTTL = 30 * 24 * time.Hour ) // rootExecCommandMax bounds the command text itself: a script, not a payload. @@ -134,6 +136,24 @@ type rootExecGrant struct { RevokedAt string `json:"revoked_at,omitempty"` } +// ends is when the grant stops admitting: its stated expiry, and never later +// than the longest term after it was given, so a grant written when the +// longest term was longer ends on today's rule. +func (g *rootExecGrant) ends() (time.Time, error) { + exp, err := time.Parse(time.RFC3339, g.Expires) + if err != nil { + return time.Time{}, err + } + at, err := time.Parse(time.RFC3339, g.GrantedAt) + if err != nil { + return time.Time{}, err + } + if limit := at.Add(rootExecMaxGrantTTL); exp.After(limit) { + exp = limit + } + return exp, nil +} + var rootExecMu sync.Mutex func rootExecGrantPath(dir string) string { return filepath.Join(dir, "root_exec_grant.json") } @@ -189,12 +209,12 @@ func rootExecGrantCheck(dir, self, server string, now time.Time) string { if g.Server == "" || g.Server != server { return fmt.Sprintf("the root-command grant is for machine '%s', this machine is '%s'", g.Server, server) } - exp, err := time.Parse(time.RFC3339, g.Expires) + exp, err := g.ends() if err != nil { - return "the root-command grant states no expiry (open-ended terms are refused)" + return "the root-command grant states no expiry or no grant time (open-ended terms are refused)" } if !exp.After(now) { - return "the root-command grant expired at " + g.Expires + " — " + rootExecGrantHowTo(self) + return "the root-command grant expired at " + exp.UTC().Format(time.RFC3339) + " — " + rootExecGrantHowTo(self) } return "" } @@ -310,7 +330,11 @@ func cmdRootExec(args []string) error { } else if why := rootExecGrantCheck(dir, cfg.HostName, server, time.Now()); why != "" { state = why } - fmt.Printf("root commands for %s on %s: %s\n granted %s, until %s\n", g.Host, g.Server, state, g.GrantedAt, g.Expires) + until := g.Expires + if exp, err := g.ends(); err == nil { + until = exp.UTC().Format(time.RFC3339) + } + fmt.Printf("root commands for %s on %s: %s\n granted %s, until %s\n", g.Host, g.Server, state, g.GrantedAt, until) fmt.Println(" every command: " + filepath.Join(cfg.contextRoot(), "sessions") + "/.log and audit.jsonl") fmt.Println(" " + rootExecOwnHostsLine()) fmt.Println(" take it back: privatae-bastion root-exec revoke --host " + cfg.HostName) diff --git a/client-go/rootexec_test.go b/client-go/rootexec_test.go index dd201de..ed12d05 100644 --- a/client-go/rootexec_test.go +++ b/client-go/rootexec_test.go @@ -697,7 +697,7 @@ func TestRootExecGrantCeremony(t *testing.T) { if _, err := rootExecGrantWithCode(dir, "h", "box1", code, 0, now); err == nil { t.Fatal("a grant with no expiry must be refused") } - if _, err := rootExecGrantWithCode(dir, "h", "box1", code, 400*24*time.Hour, now); err == nil { + if _, err := rootExecGrantWithCode(dir, "h", "box1", code, 31*24*time.Hour, now); err == nil { t.Fatal("a grant past the longest term must be refused") } wrong := "000000" @@ -744,6 +744,71 @@ func TestRootExecGrantCeremony(t *testing.T) { } } +// The longest root-command grant is 30 days: a 31-day one is refused at the +// ceremony, and a grant held from when the longest term was a year ends 30 +// days after it was given. +// +// MUTATION: set rootExecMaxGrantTTL back to 365 days. +// RESULT: red — the 31-day grant is written and the held year-long grant admits. +func TestRootExecGrantIsAtMostThirtyDays(t *testing.T) { + now := time.Now() + enrolled := func(t *testing.T) string { + t.Helper() + dir := t.TempDir() + if err := stepupSave(dir, &stepupFile{SecretB32: rootExecTestSecret, EnrolledAt: "2026-10-11T00:00:00Z", LastCounter: -1}); err != nil { + t.Fatal(err) + } + return dir + } + + dir := enrolled(t) + d, err := parseGrantFor("31d") + if err != nil { + t.Fatal(err) + } + _, err = rootExecGrantWithCode(dir, "h", "box1", rootExecCode(t, now), d, now) + if err == nil || !strings.Contains(err.Error(), "at most 30 days") { + t.Fatalf("a 31-day grant must be refused and say the longest term: %v", err) + } + if _, err := os.Stat(rootExecGrantPath(dir)); err == nil { + t.Fatal("a refused grant writes nothing") + } + if _, err := rootExecGrantWithCode(dir, "h", "box1", rootExecCode(t, now), 30*24*time.Hour, now); err != nil { + t.Fatalf("30 days is the longest term and is granted: %v", err) + } + + held := func(t *testing.T, given, expires time.Time) string { + t.Helper() + dir := t.TempDir() + g := &rootExecGrant{ID: "g1", Host: "h", Server: "box1", + GrantedAt: given.UTC().Format(time.RFC3339), Expires: expires.UTC().Format(time.RFC3339)} + if err := rootExecGrantSave(dir, g); err != nil { + t.Fatal(err) + } + return dir + } + old := held(t, now.Add(-31*24*time.Hour), now.Add(334*24*time.Hour)) + if why := rootExecGrantCheck(old, "h", "box1", now); !strings.Contains(why, "expired") { + t.Fatalf("a year-long grant given 31 days ago has ended: %q", why) + } + recent := held(t, now.Add(-10*24*time.Hour), now.Add(355*24*time.Hour)) + if why := rootExecGrantCheck(recent, "h", "box1", now); why != "" { + t.Fatalf("a year-long grant given 10 days ago still admits: %s", why) + } + if why := rootExecGrantCheck(recent, "h", "box1", now.Add(21*24*time.Hour)); !strings.Contains(why, "expired") { + t.Fatalf("and it ends 30 days after it was given: %q", why) + } + noTime := held(t, now, now.Add(24*time.Hour)) + g, _ := rootExecGrantLoad(noTime) + g.GrantedAt = "" + if err := rootExecGrantSave(noTime, g); err != nil { + t.Fatal(err) + } + if why := rootExecGrantCheck(noTime, "h", "box1", now); why == "" { + t.Fatal("a grant with no grant time cannot be held to the longest term, so it is refused") + } +} + // The op is advertised with the other session ops, so a grant can name it // and a deny can refuse it. func TestRootExecOpAdvertised(t *testing.T) { diff --git a/client-go/rootexec_unix.go b/client-go/rootexec_unix.go index 7d2509a..bbd6b3d 100644 --- a/client-go/rootexec_unix.go +++ b/client-go/rootexec_unix.go @@ -18,6 +18,8 @@ import ( "sync" "syscall" "time" + + "golang.org/x/sys/unix" ) type shellProc struct { @@ -32,7 +34,10 @@ func startRootExec(command string, stdout, stderr io.Writer, cgroupFD int) (root if u, err := user.Lookup("root"); err == nil && u.HomeDir != "" { home = u.HomeDir } - if st, err := os.Stat(home); err != nil || !st.IsDir() { + // A home this process cannot enter is no working folder for the command. + // The daemon is root here, so that is only a home that is not there; the + // tests also run this as an ordinary user, for whom /root is shut. + if st, err := os.Stat(home); err != nil || !st.IsDir() || unix.Access(home, unix.X_OK) != nil { home = "/" } cmd := exec.Command("/bin/sh", "-c", command) diff --git a/client-go/rotate.go b/client-go/rotate.go index 1b8b44e..bf8563b 100644 --- a/client-go/rotate.go +++ b/client-go/rotate.go @@ -337,6 +337,11 @@ func parseRotateTarget(heartbeatResponse []byte) (candidates []string, pubkey st if json.Unmarshal(heartbeatResponse, &hb) != nil { return nil, "" } + // A directive naming any entry that is not one host:port is not acted + // on at all, as the pairing response is refused for the same. + if checkEndpoints(append([]string{hb.Rotate.Endpoint}, hb.Rotate.Endpoints...)...) != nil { + return nil, "" + } return candidateList(hb.Rotate.Endpoints, hb.Rotate.Endpoint), strings.TrimSpace(hb.Rotate.EndpointPubKey) } @@ -474,20 +479,10 @@ func probeAndCommit(cfg *Config, dev *device.Device, endpoint, pubkey string, li // swapPeer replaces the device's single peer in one UAPI transaction — same shape as // the boot-time config in run.go. AllowedIPs stays the host's tunnel alias ONLY. func swapPeer(cfg *Config, dev *device.Device, endpoint, pubkey string) error { - privHex, err := b64ToHex(cfg.WGPrivateKey) - if err != nil { - return fmt.Errorf("bad private key in config: %w", err) - } - pubHex, err := b64ToHex(pubkey) - if err != nil { - return fmt.Errorf("bad endpoint pubkey: %w", err) - } - hostCIDR, err := hostTunnelAllowedIP(cfg.HostTunnelIP) + uapi, err := tunnelPeerUAPI(cfg, endpoint, pubkey, true) if err != nil { return err } - uapi := fmt.Sprintf("private_key=%s\nreplace_peers=true\npublic_key=%s\nendpoint=%s\npersistent_keepalive_interval=25\nallowed_ip=%s\n", - privHex, pubHex, endpoint, hostCIDR) return dev.IpcSet(uapi) } diff --git a/client-go/rotate_test.go b/client-go/rotate_test.go index b85cd8a..08657e1 100644 --- a/client-go/rotate_test.go +++ b/client-go/rotate_test.go @@ -16,8 +16,8 @@ import ( ) func TestCandidateList(t *testing.T) { - got := candidateList([]string{" [2604:2dc0:100:589b::36]:51820 ", "203.0.113.10:52054", ""}, "203.0.113.10:52054") - want := []string{"[2604:2dc0:100:589b::36]:51820", "203.0.113.10:52054"} + got := candidateList([]string{" [2001:db8:100::36]:51820 ", "203.0.113.10:52054", ""}, "203.0.113.10:52054") + want := []string{"[2001:db8:100::36]:51820", "203.0.113.10:52054"} if !sameList(got, want) { t.Fatalf("got %v want %v", got, want) } @@ -26,20 +26,20 @@ func TestCandidateList(t *testing.T) { t.Fatalf("single: %v", got) } // The v4 entry is appended when the list omits it, never dropped. - if got := candidateList([]string{"[2604::1]:51820"}, "1.2.3.4:52001"); !sameList(got, []string{"[2604::1]:51820", "1.2.3.4:52001"}) { + if got := candidateList([]string{"[2001:db8::1]:51820"}, "203.0.113.20:52001"); !sameList(got, []string{"[2001:db8::1]:51820", "203.0.113.20:52001"}) { t.Fatalf("append: %v", got) } } func TestParseRotateTarget(t *testing.T) { // New host: ordered list. - c, pk := parseRotateTarget([]byte(`{"ok":true,"rotate_endpoint":{"endpoint":"1.2.3.4:52054","endpoints":["[2604::36]:51820","1.2.3.4:52054"],"endpoint_pubkey":"PK="}}`)) - if pk != "PK=" || !sameList(c, []string{"[2604::36]:51820", "1.2.3.4:52054"}) { + c, pk := parseRotateTarget([]byte(`{"ok":true,"rotate_endpoint":{"endpoint":"203.0.113.20:52054","endpoints":["[2001:db8::36]:51820","203.0.113.20:52054"],"endpoint_pubkey":"PK="}}`)) + if pk != "PK=" || !sameList(c, []string{"[2001:db8::36]:51820", "203.0.113.20:52054"}) { t.Fatalf("new host: %v %q", c, pk) } // Old host: only endpoint. - c, pk = parseRotateTarget([]byte(`{"rotate_endpoint":{"endpoint":"1.2.3.4:52054","endpoint_pubkey":"PK="}}`)) - if pk != "PK=" || !sameList(c, []string{"1.2.3.4:52054"}) { + c, pk = parseRotateTarget([]byte(`{"rotate_endpoint":{"endpoint":"203.0.113.20:52054","endpoint_pubkey":"PK="}}`)) + if pk != "PK=" || !sameList(c, []string{"203.0.113.20:52054"}) { t.Fatalf("old host: %v %q", c, pk) } // No directive. @@ -134,7 +134,7 @@ func TestEndpointRoutableRefusesWhenTheKernelHasNoRoute(t *testing.T) { } return nil, &net.OpError{Op: "dial", Net: network, Err: errors.New("connect: network is unreachable")} } - err := endpointRoutable("[2604:2dc0:100:589b::4a]:51820") + err := endpointRoutable("[2001:db8:100::4a]:51820") if !errors.Is(err, errNoLocalPath) { t.Fatalf("want errNoLocalPath, got %v", err) } @@ -150,7 +150,7 @@ func TestEndpointRoutableRefusesWhenTheKernelHasNoRoute(t *testing.T) { c, _ := net.Pipe() return c, nil } - if err := endpointRoutable("15.204.198.155:20074"); err != nil { + if err := endpointRoutable("203.0.113.55:20074"); err != nil { t.Fatalf("routable endpoint refused: %v", err) } } @@ -158,7 +158,7 @@ func TestEndpointRoutableRefusesWhenTheKernelHasNoRoute(t *testing.T) { // A standing local condition is stated once, not once a minute. func TestNotePathlessReportsOnlyTheFirstTime(t *testing.T) { s := rotateState{failed: map[string]probeFailure{}} // noPath deliberately nil, as the package var is - ep := "[2604:2dc0:100:589b::4a]:51820" + ep := "[2001:db8:100::4a]:51820" if !s.notePathless(ep) { t.Fatal("the first sighting is the one worth logging") } @@ -200,7 +200,7 @@ func failuresRecordedFor(ep string) []string { // the guard after the swap this panics instead of quietly passing. func TestProbeAndCommitRefusesUnroutableBeforeTouchingTheLivePeer(t *testing.T) { orig := dialForRouteCheck - ep := "[2604:2dc0:100:589b::4a]:51820" + ep := "[2001:db8:100::4a]:51820" defer func() { dialForRouteCheck = orig rotation.clearPathless(ep) @@ -212,7 +212,7 @@ func TestProbeAndCommitRefusesUnroutableBeforeTouchingTheLivePeer(t *testing.T) return nil, &net.OpError{Op: "dial", Net: network, Err: errors.New("connect: network is unreachable")} } - cfg := &Config{Endpoint: "15.204.198.155:20074", EndpointPubKey: "OLD="} + cfg := &Config{Endpoint: "203.0.113.55:20074", EndpointPubKey: "OLD="} err := probeAndCommit(cfg, nil, ep, "NEW=", []string{ep, cfg.Endpoint}) if !errors.Is(err, errNoLocalPath) { t.Fatalf("want errNoLocalPath before any peer swap, got %v", err) @@ -225,7 +225,7 @@ func TestProbeAndCommitRefusesUnroutableBeforeTouchingTheLivePeer(t *testing.T) t.Fatalf("an unroutable candidate must not consume the probe backoff, recorded %q", got) } // The live endpoint is untouched: nothing was committed. - if cfg.Endpoint != "15.204.198.155:20074" || cfg.EndpointPubKey != "OLD=" { + if cfg.Endpoint != "203.0.113.55:20074" || cfg.EndpointPubKey != "OLD=" { t.Fatalf("config moved on a refused candidate: %s %s", cfg.Endpoint, cfg.EndpointPubKey) } } diff --git a/client-go/run.go b/client-go/run.go index dc2f8b9..b30d9e9 100644 --- a/client-go/run.go +++ b/client-go/run.go @@ -16,6 +16,7 @@ import ( "os" "os/signal" "runtime" + "strconv" "strings" "sync" "syscall" @@ -31,11 +32,17 @@ import ( // deploy/publish.sh + deploy/VERSION). The "dev" default flags an unstamped local build. var clientVersion = "dev" +// b64ToHex turns a base64 WireGuard key into the hex the device's +// configuration text takes. A key is 32 bytes; anything else is refused here +// rather than by the device. func b64ToHex(b64 string) (string, error) { raw, err := base64.StdEncoding.DecodeString(b64) if err != nil { return "", err } + if len(raw) != 32 { + return "", fmt.Errorf("a WireGuard key is 32 bytes, this one is %d", len(raw)) + } return hex.EncodeToString(raw), nil } @@ -131,6 +138,11 @@ func cmdRun() error { } } if started == 0 { + for _, cfg := range cfgs { + if cfg.invalid != "" { + return fmt.Errorf("no pairing can be served — %s", cfg.invalid) + } + } return fmt.Errorf("no active pairing — every pairing is pending or revoked; the anchor host must approve one") } go superviseLoop(ctx) @@ -204,6 +216,10 @@ type pairingRun struct { // pairing — it is a file), when it is already running, or when the daemon is // not running (a CLI process handling a grant has nothing to start). func startPairing(cfg *Config) bool { + if cfg.invalid != "" { + fmt.Printf("[%s] not started (no tunnel, no lanes, no dirs): %s\n", cfg.HostName, cfg.invalid) + return false + } if !cfg.active() { fmt.Printf("[%s] %s — not started (no tunnel, no lanes, no dirs) until the anchor host approves it\n", cfg.HostName, cfg.stateName()) return false @@ -648,20 +664,12 @@ func runPairing(ctx context.Context, cfg *Config) error { // may ever wait on that. go runDeltaRefresh(ctx) beat() // first beat: flip online + bind fp now, don't wait a full interval - t := time.NewTicker(60 * time.Second) - defer t.Stop() - for { - select { - case <-ctx.Done(): - return - case <-t.C: - // Path failover BEFORE the beat: if the current relay entry has gone dead - // (no handshake for RejectAfterTime) the beat could not reach the host - // anyway; move to the next candidate first, no platform round trip. - maybeFailover(cfg, dev, startedAt) - beat() - } - } + // Every 60 s, and at once after a pairing on this box changes state + // (beatnow.go), so this Host learns of it without waiting for the tick. + // Path failover BEFORE a ticked beat: if the current relay entry has gone + // dead (no handshake for RejectAfterTime) the beat could not reach the + // host anyway; move to the next candidate first, no platform round trip. + runBeatLoop(ctx, 60*time.Second, func() { maybeFailover(cfg, dev, startedAt) }, beat) }() srv := &http.Server{Addr: cfg.ListenAddr, Handler: mux} @@ -852,16 +860,9 @@ func runUnixPairing(ctx context.Context, cfg *Config) error { go runOtherBastionRefresh(ctx) go runDeltaRefresh(ctx) beat() - t := time.NewTicker(60 * time.Second) - defer t.Stop() - for { - select { - case <-ctx.Done(): - return - case <-t.C: - beat() - } - } + // Every 60 s, and at once after a pairing on this box changes state + // (beatnow.go). + runBeatLoop(ctx, 60*time.Second, nil, beat) }() srv := &http.Server{Addr: cfg.ListenAddr, Handler: mux} @@ -901,27 +902,112 @@ func heartbeatPayload(cfg *Config, dev interface{ IpcGet() (string, error) }, ho // function of its own so a test can read back what a device holds after // exactly the configuration runPairing applies. func configureTunnelPeer(dev interface{ IpcSet(string) error }, cfg *Config) error { - privHex, err := b64ToHex(cfg.WGPrivateKey) - if err != nil { - return fmt.Errorf("wg private key: %w", err) - } - endpointPubHex, err := b64ToHex(cfg.EndpointPubKey) - if err != nil { - return fmt.Errorf("endpoint pubkey: %w", err) - } - hostCIDR, err := hostTunnelAllowedIP(cfg.HostTunnelIP) + uapi, err := tunnelPeerUAPI(cfg, cfg.Endpoint, cfg.EndpointPubKey, false) if err != nil { return err } - // AllowedIPs is the host's tunnel alias ONLY — the tunnel can reach nothing else. - uapi := fmt.Sprintf("private_key=%s\npublic_key=%s\nendpoint=%s\npersistent_keepalive_interval=25\nallowed_ip=%s\n", - privHex, endpointPubHex, cfg.Endpoint, hostCIDR) if err := dev.IpcSet(uapi); err != nil { return fmt.Errorf("configure wireguard: %w", err) } return nil } +// tunnelPeerUAPI is the configuration text that gives the device its one +// peer: the host at endpoint under pubkey, with the host's tunnel alias as +// the only allowed address. Both the boot-time configuration and the live +// endpoint swap build it here. The text is one setting per line and every +// value in it comes from the host, so each is checked before it is written: +// a newline in any of them would add settings of its own. +func tunnelPeerUAPI(cfg *Config, endpoint, pubkey string, replacePeers bool) (string, error) { + privHex, err := b64ToHex(cfg.WGPrivateKey) + if err != nil { + return "", fmt.Errorf("wg private key: %w", err) + } + pubHex, err := b64ToHex(pubkey) + if err != nil { + return "", fmt.Errorf("endpoint pubkey: %w", err) + } + if err := checkEndpoint(endpoint); err != nil { + return "", err + } + hostCIDR, err := hostTunnelAllowedIP(cfg.HostTunnelIP) + if err != nil { + return "", err + } + replace := "" + if replacePeers { + replace = "replace_peers=true\n" + } + // AllowedIPs is the host's tunnel alias ONLY — the tunnel can reach nothing else. + return fmt.Sprintf("private_key=%s\n%spublic_key=%s\nendpoint=%s\npersistent_keepalive_interval=25\nallowed_ip=%s\n", + privHex, replace, pubHex, endpoint, hostCIDR), nil +} + +// checkEndpoint accepts one tunnel endpoint: an IP address and port +// ("203.0.113.7:51820", "[2001:db8::7]:51820") or an RFC 1123 host name and +// port, the port 1-65535. Anything else is refused, any space or control +// character included: the value is written into the configuration text. +func checkEndpoint(s string) error { + bad := fmt.Errorf("the endpoint %q is not a host:port — refusing to configure the tunnel to it", s) + for i := 0; i < len(s); i++ { + if s[i] <= ' ' || s[i] >= 0x7f { + return bad + } + } + if ap, err := netip.ParseAddrPort(s); err == nil { + if ap.Port() == 0 || ap.Addr().Zone() != "" || ap.Addr().IsUnspecified() { + return bad + } + return nil + } + if strings.HasPrefix(s, "[") { + return bad // brackets hold an IPv6 address, and this one did not parse + } + host, port, err := net.SplitHostPort(s) + if err != nil || !rfc1123Host(host) || len(port) > 5 || strings.Trim(port, "0123456789") != "" { + return bad + } + if n, err := strconv.Atoi(port); err != nil || n < 1 || n > 65535 { + return bad + } + return nil +} + +// checkEndpoints applies checkEndpoint to each non-empty entry: a pairing +// over a socket has no endpoint, and an empty list entry is dropped by +// candidateList. +func checkEndpoints(eps ...string) error { + for _, e := range eps { + if e == "" { + continue + } + if err := checkEndpoint(e); err != nil { + return err + } + } + return nil +} + +// rfc1123Host is a DNS host name: dot-separated labels of 1-63 letters, +// digits and hyphens, no label starting or ending with a hyphen. +func rfc1123Host(h string) bool { + if h == "" || len(h) > 253 { + return false + } + for _, label := range strings.Split(h, ".") { + if label == "" || len(label) > 63 || label[0] == '-' || label[len(label)-1] == '-' { + return false + } + for i := 0; i < len(label); i++ { + c := label[i] + if !(c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '-') { + return false + } + } + } + return true +} + // hostTunnelAllowedIP is the one allowed address the tunnel's peer gets: the // host's tunnel alias as a single-address prefix (/128, or /32 for IPv4). The // value arrives in the pairing response, and the platform sends one bare @@ -991,6 +1077,10 @@ func cmdStatus() error { } fmt.Println() for _, cfg := range cfgs { + if cfg.invalid != "" { + fmt.Printf("paired with: %s [%s]\n problem: %s\n", cfg.HostName, cfg.stateName(), cfg.invalid) + continue + } live := "not answering" if bastionListening(cfg.ListenAddr) { live = "answering" diff --git a/client-go/runguard_darwin.go b/client-go/runguard_darwin.go index 660f035..82a5b1f 100644 --- a/client-go/runguard_darwin.go +++ b/client-go/runguard_darwin.go @@ -5,11 +5,14 @@ package main import ( "fmt" "os" + "sync/atomic" ) // runGuardFile holds the run lock for the life of the process; the kernel -// releases it when the process ends. -var runGuardFile *os.File +// releases it when the process ends. It is a package-level holder so the +// handle is never collected (a collected *os.File is closed, which would +// release the lock while we are still serving). +var runGuardFile atomic.Pointer[os.File] // guardSingleRun on macOS: the rule is guardRunBesideService (runlock_unix.go). // The launchd daemon takes the lock first, so an administrator's `run` beside @@ -19,7 +22,7 @@ func guardSingleRun() error { if err != nil { return err } - runGuardFile = f + runGuardFile.Store(f) // The daemon, holding the lock, keeps its log in bounds. launchd has no // size limit of its own for it. if systemService.installed && processIsRoot() { diff --git a/client-go/runguard_windows.go b/client-go/runguard_windows.go index 88220a3..da48387 100644 --- a/client-go/runguard_windows.go +++ b/client-go/runguard_windows.go @@ -7,12 +7,13 @@ import ( "fmt" "os" "path/filepath" + "sync/atomic" ) // runGuardFile holds the "one running bastion" lock for the life of this -// process. It is a package var so the handle is never collected and the lock is -// never released while we are still serving. -var runGuardFile *os.File +// process. It is a package-level holder so the handle is never collected and +// the lock is never released while we are still serving. +var runGuardFile atomic.Pointer[os.File] // guardSingleRun refuses to start a SECOND bastion on this machine. // @@ -100,8 +101,8 @@ func guardSingleRun() error { "your hosts, and a second would fight it for the same local ports.\n"+ " Nothing is wrong: you can close this window.\n"+ " If you believe the running one is dead, stop it (Task Manager → "+ - "privatae-bastion.exe) and run this again.")) + "privatae-bastion.exe) and run this again")) } - runGuardFile = f + runGuardFile.Store(f) return nil } diff --git a/client-go/service_mirror.go b/client-go/service_mirror.go index c17d12d..a8de186 100644 --- a/client-go/service_mirror.go +++ b/client-go/service_mirror.go @@ -246,11 +246,12 @@ func mirrorVetHost(ctx context.Context, host string) (string, error) { return ips[0].String(), nil } -// runGitEnv runs git with a clean, locked-down config and extra env. +// runGitEnv runs git with a clean, locked-down config and extra env, and +// with harnessGitSafe ahead of the caller's arguments. func runGitEnv(timeout time.Duration, env []string, args ...string) (string, error) { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() - cmd := exec.CommandContext(ctx, "git", args...) + cmd := exec.CommandContext(ctx, "git", append(append([]string{}, harnessGitSafe...), args...)...) hideConsole(cmd) home, _ := os.MkdirTemp("", "privatae-mirror-home-") defer os.RemoveAll(home) diff --git a/client-go/service_mirror_e2e_test.go b/client-go/service_mirror_e2e_test.go index 6e93ee4..a647523 100644 --- a/client-go/service_mirror_e2e_test.go +++ b/client-go/service_mirror_e2e_test.go @@ -11,6 +11,7 @@ package main // go test -c -o /tmp/b.test . && BASTION_MIRROR_E2E=1 /tmp/b.test -test.run TestMirrorEndToEnd -test.v import ( + "context" "crypto/ed25519" "encoding/base64" "encoding/json" @@ -89,7 +90,9 @@ func TestMirrorEndToEnd(t *testing.T) { } // Read the server's own view over its socket: both repos exist, with commits. - hc := &http.Client{Transport: &http.Transport{Dial: func(_, _ string) (net.Conn, error) { return net.Dial("unix", sock) }}} + hc := &http.Client{Transport: &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, "unix", sock) + }}} // Gitea indexes a pushed repo in a background queue, so poll for a moment. for _, repo := range []string{"rig/nocode", "rig/example"} { var last string diff --git a/client-go/service_mirror_test.go b/client-go/service_mirror_test.go index ba8f9f8..0611ece 100644 --- a/client-go/service_mirror_test.go +++ b/client-go/service_mirror_test.go @@ -248,7 +248,7 @@ func TestMirrorListRefusal(t *testing.T) { "space": {{URL: "https://github.com/a/b c", Repo: "m/b"}}, "tab": {{URL: "https://github.com/a/b", Repo: "m/\tb"}}, "unicode": {{URL: "https://gíthub.com/a/b", Repo: "m/b"}}, - "rtl override": {{URL: "https://github.com/a/‮b", Repo: "m/b"}}, + "rtl override": {{URL: "https://github.com/a/\u202eb", Repo: "m/b"}}, "duplicate": {good[0], {URL: "https://github.com/x/flask.git", Repo: "mirrors/flask"}}, "dup any case": {good[0], {URL: "https://github.com/x/flask.git", Repo: "Mirrors/Flask"}}, } { diff --git a/client-go/session_test.go b/client-go/session_test.go index 271d6a8..4642c77 100644 --- a/client-go/session_test.go +++ b/client-go/session_test.go @@ -228,7 +228,7 @@ func TestSessionFramingResizeAndTranscript(t *testing.T) { if err != nil { t.Fatalf("transcript missing: %v", err) } - if runtime.GOOS != "windows" && st.Mode().Perm() != 0o600 { + if st.Mode().Perm() != 0o600 { t.Fatalf("transcript mode %o, want 0600", st.Mode().Perm()) } raw, _ := os.ReadFile(p) @@ -613,9 +613,6 @@ func TestSessionWorkerLaneRouting(t *testing.T) { // TestSessionRefusesTerminalServers: tmux/screen are servers that outlive the // session's kill, so they are never the session shell even when /etc/shells lists them. func TestSessionRefusesTerminalServers(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip("unix spawner") - } shells := filepath.Join(t.TempDir(), "shells") _ = os.WriteFile(shells, []byte("/usr/bin/tmux\n/usr/bin/screen\n/bin/sh\n"), 0o644) old := sessionShellsPath diff --git a/client-go/singleton.go b/client-go/singleton.go index 995d5b3..f528242 100644 --- a/client-go/singleton.go +++ b/client-go/singleton.go @@ -434,7 +434,7 @@ func singletonGuardAgainst(m *machineMarker) error { "A machine runs one bastion. If you are the owner, run this as that\n"+ "bastion's user (or with `sudo -H`) so it pairs into the existing one.\n"+ "If someone else owns the bastion here, this machine cannot serve your\n"+ - "host yet — say so to whoever runs it.", + "host yet — say so to whoever runs it", errForkedState, m.StateDir, mine) } diff --git a/client-go/ui.go b/client-go/ui.go index 58f1cec..bb098b7 100644 --- a/client-go/ui.go +++ b/client-go/ui.go @@ -231,17 +231,16 @@ func collectUIStatus() uiStatus { running := 0 for _, c := range cfgs { p := uiPairing{ - Host: c.HostName, - State: c.State, + Host: c.HostName, + // As the setup page and `status` read it: a pairing whose file + // cannot be served is "invalid" whatever its stored state says. + State: c.stateName(), ListenAddr: c.ListenAddr, Listening: bastionListening(c.ListenAddr), TunnelIP: c.BastionTunnelIP, FirstSeen: c.FirstSeen, IsAnchor: s.AnchorHost != "" && c.HostName == s.AnchorHost, } - if p.State == "" { - p.State = pairingActive - } if p.Listening { running++ } @@ -601,7 +600,7 @@ func approvalTarget(host string) (*Config, error) { // below reach cfg.dcnDir(), which is MkdirAll-on-read. if !c.active() { return nil, fmt.Errorf("this machine does not serve %q yet — that pairing is %s", - host, c.State) + host, c.stateName()) } return c, nil } @@ -869,7 +868,7 @@ func localUIEntryURLTo(to string) (entry, addr string, err error) { } if !bastionListening(addr) { return "", addr, fmt.Errorf("the bastion is not running on this machine, so "+ - "there is nothing to show.\n Start it with `%s`, then run this again.", + "there is nothing to show.\n Start it with `%s`, then run this again", runCommandHint()) } secret, err := loadOrCreateUISecret() diff --git a/client-go/ui_approvals_test.go b/client-go/ui_approvals_test.go index 88500f5..66ebf39 100644 --- a/client-go/ui_approvals_test.go +++ b/client-go/ui_approvals_test.go @@ -232,7 +232,7 @@ func TestAnUndescribableRequestDoesNotGetAPlainApprove(t *testing.T) { // It must still be POSSIBLE to approve — this is a warning, not a wall. A // machine that cannot be told "yes" would strand a fleet mid-upgrade. - w = uiPost(t, u.handler(), "/approve", url.Values{ + uiPost(t, u.handler(), "/approve", url.Values{ "host": {host}, "id": {id}, "code": {uiCurrentCode(t)}}, cookie) if got := requestsLoad(dcn)[id].State; got != "approved" { t.Fatalf("an informed approval was blocked: state %q", got) diff --git a/client-go/ui_pairing_state_test.go b/client-go/ui_pairing_state_test.go new file mode 100644 index 0000000..4e63235 --- /dev/null +++ b/client-go/ui_pairing_state_test.go @@ -0,0 +1,41 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +// A pairing whose file this build will not serve reads "invalid" on the +// console, as it does on the setup page and in `status`, even when the state +// stored in the file is empty, the old spelling of active. +func TestTheConsoleShowsAPairingThatIsNotServedAsInvalid(t *testing.T) { + const host = "unserved.host.privatae.ai" + uiTestConfigDir(t) + pd, err := pairingsDir() + if err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(pd, 0o700); err != nil { + t.Fatal(err) + } + b, _ := json.Marshal(Config{HostName: host, ListenAddr: "127.0.0.1:59999", + Endpoint: "203.0.113.7:51820\nallowed_ip=::/0"}) // one the endpoint check refuses + if err := os.WriteFile(filepath.Join(pd, host+".json"), b, 0o600); err != nil { + t.Fatal(err) + } + var got []string + for _, p := range collectUIStatus().Pairings { + got = append(got, p.Host+"="+p.State) + } + if want := host + "=" + pairingInvalid; len(got) != 1 || got[0] != want { + t.Errorf("the console reads %v, want [%s]", got, want) + } + u := uiTestServer(t) + body := uiSetupGet(t, u.handler(), "/fragments/status", uiSignedIn(t, u)).Body.String() + if !strings.Contains(body, "invalid, not served") { + t.Errorf("the console page does not say the pairing is not served: %q", body) + } +} diff --git a/client-go/uiassets/status.html b/client-go/uiassets/status.html index d9ebde2..8875e71 100644 --- a/client-go/uiassets/status.html +++ b/client-go/uiassets/status.html @@ -64,6 +64,7 @@ {{if .IsAnchor}}anchor{{end}} {{if eq .State "pending"}}waiting for approval{{end}} {{if eq .State "revoked"}}revoked{{end}} + {{if eq .State "invalid"}}invalid, not served{{end}}
Local port
diff --git a/client-go/uidevice.go b/client-go/uidevice.go index 51707e4..b4661a7 100644 --- a/client-go/uidevice.go +++ b/client-go/uidevice.go @@ -16,7 +16,7 @@ package main // machine itself — everything that is waiting on the person, in one place. Each // device ask has Approve and Deny; several have "Approve all shown". The tray // (tray_windows.go) and the notification (notify_windows.go) open this page -// directly, as a small app window where the system has one (uipopup_*.go), so a +// directly, as a small app window where the system has one (uipopup_windows.go), so a // waiting approval is one click from the taskbar with no menu in between. // // WHAT IS NOT CHANGED. Signing is deviceApprove's, exactly as the console does @@ -347,29 +347,6 @@ func deviceAskServer(id string) string { return "that machine" } -// openApprovalsPopup is what the tray's approvals line and a click on the -// approval notification do: open /approvals, signed in, as a small window. -// -// It goes through localUIEntryURLTo exactly as openLocalUI goes through -// localUIEntryURL — a one-use ticket minted against the running daemon — so it -// cannot hand out a link that does not authenticate, and the ticket's redirect -// lands on the page itself rather than on a menu. -// approvalsOpener is what a click on an approval notification opens: the -// daemon's own popup, or, in the tray beside a system service, the tray's page -// (tray_pipe_windows.go sets it before the notifier starts). -var approvalsOpener = openApprovalsPopup - -func openApprovalsPopup(from string) { - entry, _, err := localUIEntryURLTo(uiApprovalsPath) - if err != nil { - uilogf("%s: %v\n", from, err) - return - } - if err := openPopupWindow(entry); err != nil { - uilogf("%s: %v\n", from, err) - } -} - // uiApprovalsPath is the one place a sign-in ticket may land other than "/". // root() redirects only to paths in uiLandingPaths — a ticket link is handed to // a browser, and an open redirect on it would be a phishing tool pointed at the diff --git a/client-go/uipopup_other.go b/client-go/uipopup_other.go deleted file mode 100644 index ebb63b1..0000000 --- a/client-go/uipopup_other.go +++ /dev/null @@ -1,8 +0,0 @@ -//go:build !windows - -package main - -// uipopup_other.go — off Windows there is no tray to open the approvals popup -// from (tray_windows.go says why), so the page opens in the default browser, -// the same way `privatae-bastion ui` opens the front page. -func openPopupWindow(url string) error { return openBrowser(url) } diff --git a/client-go/uipopup_windows.go b/client-go/uipopup_windows.go index 5426d80..060f8fe 100644 --- a/client-go/uipopup_windows.go +++ b/client-go/uipopup_windows.go @@ -45,6 +45,29 @@ func openPopupWindow(url string) error { return openBrowser(url) } +// approvalsOpener is what a click on an approval notification opens: the +// daemon's own popup, or, in the tray beside a system service, the tray's page +// (tray_pipe_windows.go sets it before the notifier starts). +var approvalsOpener = openApprovalsPopup + +// openApprovalsPopup is what the tray's approvals line and a click on the +// approval notification do: open /approvals, signed in, as a small window. +// +// It goes through localUIEntryURLTo exactly as openLocalUI goes through +// localUIEntryURL — a one-use ticket minted against the running daemon — so it +// cannot hand out a link that does not authenticate, and the ticket's redirect +// lands on the page itself rather than on a menu. +func openApprovalsPopup(from string) { + entry, _, err := localUIEntryURLTo(uiApprovalsPath) + if err != nil { + uilogf("%s: %v\n", from, err) + return + } + if err := openPopupWindow(entry); err != nil { + uilogf("%s: %v\n", from, err) + } +} + // findEdge looks where Edge installs: machine-wide under both Program Files // roots, and per-user under LOCALAPPDATA. "" when it is in none of them. func findEdge() string { diff --git a/deploy/VERSION b/deploy/VERSION index 1942d77..9dd7871 100644 --- a/deploy/VERSION +++ b/deploy/VERSION @@ -1 +1 @@ -0.54.1 +0.55.3 diff --git a/deploy/build.sh b/deploy/build.sh index 7f3a9cc..95bfd52 100755 --- a/deploy/build.sh +++ b/deploy/build.sh @@ -23,10 +23,10 @@ set -euo pipefail HERE="$(cd "$(dirname "$0")/.." && pwd)" VER="${1:?usage: deploy/build.sh }" -# golang:1.25-alpine as of the digest below (go1.25.14). Move it deliberately, -# in its own commit, and a release built after that commit is rebuilt with the -# new one. -GO_IMAGE="golang@sha256:1ae0735f00daffa3aaf1363a5184c0d2dc55c78e3db4ec70241cdac97bf84b59" +# golang:1.26.10-alpine3.24, the multi-arch index digest below (go1.26.10). +# Move it deliberately, in its own commit, and a release built after that +# commit is rebuilt with the new one. +GO_IMAGE="golang@sha256:27f2d3b83cfc5803c8078f8e730af0f7c431bf067afaec8f60a36fb763187cba" # The module cache is only a download cache: go.sum checks everything read from # it. It is kept between runs so a rebuild does not fetch every module again. diff --git a/deploy/check-claims.sh b/deploy/check-claims.sh index 14bf1a2..4633197 100755 --- a/deploy/check-claims.sh +++ b/deploy/check-claims.sh @@ -18,8 +18,47 @@ # # It checks that the test exists, not what it asserts; that part is reviewed # by reading the test when a row is added or changed. +# +# deploy/check-claims.sh --skipped [repo root] +# +# reads a `go test -json` run instead and names every cited test that +# skipped in it, with the reason it gave. It exits non-zero if any did: a CI job +# that runs a claim's test where it cannot run proves nothing about the claim. set -euo pipefail HERE="$(cd "$(dirname "$0")/.." && pwd)" +if [ "${1:-}" = "--skipped" ]; then + RUN="$(cd "$(dirname "${2:?usage: deploy/check-claims.sh --skipped [repo root]}")" && pwd)/$(basename "$2")" + ROOT="${3:-$HERE}" + cd "$ROOT" + [ -f docs/claims.md ] || { echo "no docs/claims.md in $ROOT" >&2; exit 2; } + python3 - "$RUN" <<'PY' +import json, re, sys + +cited = set(re.findall(r"`[\w./-]+_test\.go`: `(\w+)`", open("docs/claims.md", encoding="utf-8").read())) +skipped, why, ran = {}, {}, set() +for line in open(sys.argv[1], encoding="utf-8", errors="replace"): + try: + e = json.loads(line) + except ValueError: + continue + t = e.get("Test") + if not t: + continue + top = t.split("/")[0] + if top not in cited: + continue + ran.add(top) + if e.get("Action") == "output" and "_test.go:" in e.get("Output", ""): + why.setdefault(t, e["Output"].strip()) + if e.get("Action") == "skip": + skipped[t] = True +for t in sorted(skipped): + print("SKIPPED " + t + ": " + why.get(t, "(no reason printed)")) +print("%d cited tests ran here, %d of them skipped" % (len(ran), len(skipped))) +sys.exit(1 if skipped else 0) +PY + exit $? +fi ROOT="${1:-$HERE}" cd "$ROOT" [ -f docs/claims.md ] && [ -d client-go ] || { echo "not a repository root (no docs/claims.md or client-go): $ROOT" >&2; exit 2; } diff --git a/deploy/github-release.py b/deploy/github-release.py new file mode 100755 index 0000000..670ea8e --- /dev/null +++ b/deploy/github-release.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +"""Attach a release's checked files to its GitHub release. + + GH_TOKEN= python3 deploy/github-release.py v + +.github/workflows/release.yml runs this after deploy/release-assets.sh has +filled with files it checked against the signed manifest. The +release for the tag is made if it does not exist yet, named +`bastion `, with that version's section of as its +notes; an existing release keeps its name and notes. Each file in + is uploaded, replacing an asset of the same name, so a re-run +leaves the same set. Standard library only; the token is read from the +environment and sent only to the GitHub API. +""" + +import json +import os +import re +import sys +import urllib.error +import urllib.parse +import urllib.request + +API = os.environ.get("GITHUB_API_URL", "https://api.github.com") +UPLOADS = os.environ.get("GITHUB_UPLOADS_URL", "https://uploads.github.com") + + +def call(method, url, token, body=None, ctype="application/json"): + data = None + if body is not None: + data = body if isinstance(body, bytes) else json.dumps(body).encode() + req = urllib.request.Request(url, data=data, method=method, headers={ + "Authorization": "Bearer " + token, "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", "Content-Type": ctype, + "User-Agent": "bastion-release-assets"}) + try: + with urllib.request.urlopen(req, timeout=300) as r: + raw = r.read() + return r.status, (json.loads(raw) if raw else None) + except urllib.error.HTTPError as e: + raw = e.read() + try: + return e.code, json.loads(raw) + except ValueError: + return e.code, {"message": raw.decode(errors="replace")[:300]} + + +def notes(changelog, ver): + """The CHANGELOG section `## `, up to the next `## ` heading.""" + text = open(changelog, encoding="utf-8").read() + m = re.search(r"^## " + re.escape(ver) + r"[ \t]*$", text, re.M) + if m is None: + raise SystemExit("CHANGELOG has no section for %s" % ver) + nxt = re.search(r"^## ", text[m.end():], re.M) + body = text[m.end():m.end() + nxt.start() if nxt else len(text)].strip() + return (body + "\n\n---\nThe binaries attached here were rebuilt from this tag by this repository's " + "release workflow, and their digests are the ones in the signed manifest `version.json` " + "attached beside them. To check them yourself, see " + "[Verify a release](https://github.com/Privatae/bastion#verify-a-release).") + + +def main(argv): + if len(argv) != 5: + print(__doc__.strip().splitlines()[2].strip(), file=sys.stderr) + return 2 + repo, tag, assets, changelog = argv[1:] + if not re.fullmatch(r"v[0-9]+\.[0-9]+\.[0-9]+", tag): + raise SystemExit("not a release tag: " + tag) + token = os.environ.get("GH_TOKEN", "") + if not token: + raise SystemExit("GH_TOKEN is not set") + ver = tag[1:] + files = sorted(os.listdir(assets)) + if not files: + raise SystemExit("no assets in " + assets) + + st, rel = call("GET", "%s/repos/%s/releases/tags/%s" % (API, repo, tag), token) + if st == 404: + st, rel = call("POST", "%s/repos/%s/releases" % (API, repo), token, { + "tag_name": tag, "name": "bastion " + ver, "body": notes(changelog, ver), + "draft": False, "prerelease": False}) + if st != 201: + raise SystemExit("could not make the release for %s: HTTP %d %s" % (tag, st, rel)) + print("made the release for %s" % tag) + elif st != 200: + raise SystemExit("could not read the release for %s: HTTP %d %s" % (tag, st, rel)) + + have = {a["name"]: a["id"] for a in rel.get("assets", [])} + for name in files: + if name in have: + st, _ = call("DELETE", "%s/repos/%s/releases/assets/%d" % (API, repo, have[name]), token) + if st != 204: + raise SystemExit("could not replace %s: HTTP %d" % (name, st)) + with open(os.path.join(assets, name), "rb") as fh: + data = fh.read() + url = "%s/repos/%s/releases/%d/assets?name=%s" % (UPLOADS, repo, rel["id"], urllib.parse.quote(name)) + st, up = call("POST", url, token, data, "application/octet-stream") + if st != 201 or up.get("size") != len(data): + raise SystemExit("upload of %s failed: HTTP %d %s" % (name, st, up)) + print("attached %s (%d bytes)" % (name, len(data))) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/deploy/manifest.py b/deploy/manifest.py new file mode 100755 index 0000000..5d70150 --- /dev/null +++ b/deploy/manifest.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +"""Write a bastion release manifest (version.json) to standard output. + + python3 deploy/manifest.py + +publish.sh writes every release manifest with this, and release-assets.sh +writes it again from a rebuild of the release's tag. The manifest is a function +of the version, the commit it was built from and the bytes of the binaries in +, so a rebuild of the same release writes the same bytes, and the +signature published for the release (named by the manifest's digest) still +verifies against it. + +The format: {version, built_from, binaries: {-[.exe]: {url, sha256}}}, +keys in that order, the binaries sorted by file name, indented by two spaces, +no newline at the end. +""" + +import hashlib +import json +import os +import sys + + +def manifest(ver, built_from, dist): + binaries = {} + for f in sorted(os.listdir(dist)): + if not f.startswith("privatae-bastion-"): + continue + with open(os.path.join(dist, f), "rb") as fh: + h = hashlib.sha256(fh.read()).hexdigest() + # ?v= IS LOAD-BEARING, not decoration. dl.privatae.ai sits + # behind a cache that keys on the full URL including the query string. + # Without it a publish replaces the bytes on every edge, every edge + # verifies, the manifest is fresh, and the cache keeps serving the + # PREVIOUS binary to anyone whose request lands on a warm edge: the + # client downloads the old bytes, compares them with the new digest, + # and aborts with "sha256 mismatch", which reads like a corrupt build + # and is a stale cache. It breaks one platform at a time, for whoever + # downloaded last. A new version string is a new cache key and so a + # guaranteed miss; update.go downloads binaries[key].url verbatim, so + # bastions already in the field pick this up with no client change. + binaries[f[len("privatae-bastion-"):]] = { + "url": f"https://dl.privatae.ai/dl/{f}?v={ver}", "sha256": h} + return json.dumps({"version": ver, "built_from": built_from, "binaries": binaries}, indent=2) + + +def main(argv): + if len(argv) != 4: + print("usage: deploy/manifest.py ", file=sys.stderr) + return 2 + sys.stdout.write(manifest(argv[1], argv[2], argv[3])) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/deploy/publish.sh b/deploy/publish.sh index 1b7dbd6..eda082e 100755 --- a/deploy/publish.sh +++ b/deploy/publish.sh @@ -85,37 +85,11 @@ fi # same script and compares every digest. "$HERE/deploy/build.sh" "$VER" -# version manifest: {version, built_from, binaries:{os-arch:{url,sha256}}} — built with python -# (clean JSON; the host + the bastion self-update read this to decide "out of date"). -python3 - "$VER" "$GITSHA" "$HERE/client-go/dist" > "$HERE/client-go/dist/version.json" <<'PY' -import sys, os, hashlib, json -ver, sha, dist = sys.argv[1], sys.argv[2], sys.argv[3] -binaries = {} -for f in sorted(os.listdir(dist)): - if not f.startswith("privatae-bastion-"): - continue - h = hashlib.sha256(open(os.path.join(dist, f), "rb").read()).hexdigest() - # ?v= IS LOAD-BEARING, not decoration. dl.privatae.ai sits behind - # Cloudflare, which caches these objects and keys on the full URL including - # the query string. Without it a publish replaces the bytes on both edges, - # both edges verify, the manifest is fresh — and Cloudflare keeps serving the - # PREVIOUS binary to anyone whose request lands on a warm edge. The client - # downloads the old bytes, compares them against the new digest, and aborts - # with "sha256 mismatch" — which reads like a corrupt build and is actually a - # stale cache. Measured publishing 0.23.0: windows-amd64.exe came - # back cf-cache-status HIT, age 3917, carrying 0.22.0's bytes. It was the - # ONLY stale one, because it was the only object anything had requested since - # the previous publish — so this breaks one platform at a time, at random, - # for whoever happens to have downloaded last. The worst possible shape. - # - # A new version string is a new cache key is a guaranteed miss. Needs no - # Cloudflare credential and cannot be forgotten at publish time. update.go - # downloads binaries[key].url verbatim, so bastions already in the field pick - # this up from the manifest with no client change. - binaries[f[len("privatae-bastion-"):]] = { - "url": f"https://dl.privatae.ai/dl/{f}?v={ver}", "sha256": h} -json.dump({"version": ver, "built_from": sha, "binaries": binaries}, sys.stdout, indent=2) -PY +# The version manifest, written by deploy/manifest.py: release-assets.sh writes +# it again from a rebuild of the release's tag, and the two must give the same +# bytes for the signature to verify there. The host and every bastion's +# self-update read it to decide "out of date". +python3 "$HERE/deploy/manifest.py" "$VER" "$GITSHA" "$HERE/client-go/dist" > "$HERE/client-go/dist/version.json" MAN="$HERE/client-go/dist/version.json" MAN_SHA="$(python3 -c 'import hashlib,sys;print(hashlib.sha256(open(sys.argv[1],"rb").read()).hexdigest())' "$MAN")" diff --git a/deploy/release-assets.sh b/deploy/release-assets.sh new file mode 100755 index 0000000..833d57b --- /dev/null +++ b/deploy/release-assets.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +# Rebuild a published release from its tag and collect the files its GitHub +# release carries, refusing unless they are exactly what was signed. +# +# deploy/release-assets.sh v +# +# Needs git, docker, curl and python3, and a clone of this repository that has +# the tag. It needs no key or account of ours. .github/workflows/release.yml +# runs it for every release tag; anyone can run it to check a release. +# +# The tag must be annotated, with the line `built from ` that names +# the commit the release was built from in the development repository. The +# tag's own deploy/build.sh rebuilds the binaries (a pinned toolchain image, +# so the bytes are reproducible), deploy/manifest.py writes the release +# manifest from them, and the signature published beside the manifest, which +# is named by the manifest's digest and never removed, must verify against +# the release keys pinned in this checkout and at the tag. A rebuild that +# differs by one byte gives another digest, no signature by that name, and +# nothing in . When the tag is the release the public manifest names +# now, the rebuilt manifest must also equal that one byte for byte. +# +# gets the six binaries, version.json (the signed manifest), +# version.json..sig (its signature, the file every +# bastion's self-update checks) and SHA256SUMS (the binaries and the manifest, +# in the format `sha256sum -c` reads; it is not signed itself). +set -euo pipefail +HERE="$(cd "$(dirname "$0")/.." && pwd)" +TAG="${1:?usage: deploy/release-assets.sh v }" +OUT="${2:?usage: deploy/release-assets.sh v }" +BASE="${MANIFEST_BASE:-https://privatae.ai/bastion}" +[[ "$TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] || { echo "not a release tag: $TAG" >&2; exit 2; } +VER="${TAG#v}" +[ ! -e "$OUT" ] || [ -z "$(ls -A "$OUT")" ] || { echo "$OUT is not empty" >&2; exit 2; } + +git -C "$HERE" rev-parse -q --verify "refs/tags/$TAG" >/dev/null || { echo "no tag $TAG in this clone" >&2; exit 1; } +[ "$(git -C "$HERE" cat-file -t "refs/tags/$TAG")" = tag ] || { echo "$TAG is not an annotated tag" >&2; exit 1; } +FROM="$(git -C "$HERE" tag -l --format='%(contents)' "$TAG" | sed -n 's/^built from \([0-9a-f]\{7,40\}\)$/\1/p')" +[ "$(printf '%s\n' "$FROM" | grep -c .)" = 1 ] || { echo "$TAG does not carry exactly one 'built from ' line" >&2; exit 1; } +SRC="$(git -C "$HERE" rev-parse "refs/tags/$TAG^{commit}")" +[ "$(git -C "$HERE" show "$SRC:deploy/VERSION" | tr -d '[:space:]')" = "$VER" ] \ + || { echo "deploy/VERSION at $TAG is not $VER" >&2; exit 1; } +echo "release: $TAG at $SRC, built from $FROM" + +WORK="$(mktemp -d)" +trap 'git -C "$HERE" worktree remove --force "$WORK/src" >/dev/null 2>&1 || true; rm -rf "$WORK"' EXIT +git -C "$HERE" worktree add -q --detach "$WORK/src" "$SRC" +"$WORK/src/deploy/build.sh" "$VER" +DIST="$WORK/src/client-go/dist" +python3 "$HERE/deploy/manifest.py" "$VER" "$FROM" "$DIST" > "$WORK/version.json" +H="$(python3 -c 'import hashlib,sys;print(hashlib.sha256(open(sys.argv[1],"rb").read()).hexdigest()[:16])' "$WORK/version.json")" +SIG="version.json.$H.sig" +curl -fsS --max-time 30 "$BASE/$SIG" -o "$WORK/$SIG" \ + || { echo "no signature is published for the rebuilt manifest ($SIG): the rebuild is not the signed release" >&2; exit 1; } +SIGNED_BY="$("$HERE/deploy/release-verify.sh" bastion "$WORK/version.json" "$WORK/$SIG" "$SRC")" \ + || { echo "the signature $SIG does not verify against the rebuilt manifest" >&2; exit 1; } +echo "signature: ok ($SIGNED_BY)" + +if curl -fsS --max-time 30 "$BASE/version.json" -o "$WORK/live.json"; then + LIVE="$(python3 -c 'import json,sys;print(json.load(open(sys.argv[1]))["version"])' "$WORK/live.json")" + if [ "$LIVE" = "$VER" ]; then + cmp -s "$WORK/live.json" "$WORK/version.json" \ + || { echo "the public manifest names $VER but differs from the rebuilt one" >&2; exit 1; } + echo "the public manifest names $VER and is byte for byte the rebuilt one" + else + echo "the public manifest names $LIVE; $VER is checked by its signature alone" + fi +fi + +mkdir -p "$OUT" +cp "$DIST"/privatae-bastion-* "$WORK/version.json" "$WORK/$SIG" "$OUT/" +(cd "$OUT" && python3 - <<'PY' +import hashlib, os +names = sorted(f for f in os.listdir(".") if f.startswith("privatae-bastion-")) + ["version.json"] +with open("SHA256SUMS", "w") as fh: + for n in names: + fh.write("%s %s\n" % (hashlib.sha256(open(n, "rb").read()).hexdigest(), n)) +PY +) +n="$(ls "$OUT" | grep -c '^privatae-bastion-')" +[ "$n" = 6 ] || { echo "expected 6 binaries, built $n" >&2; exit 1; } +echo "assets for $TAG in $OUT:" +ls -l "$OUT" diff --git a/deploy/staticcheck.sh b/deploy/staticcheck.sh new file mode 100755 index 0000000..e839d22 --- /dev/null +++ b/deploy/staticcheck.sh @@ -0,0 +1,35 @@ +#!/bin/sh +# staticcheck on both Go modules, for every OS the release is built for. +# +# deploy/staticcheck.sh +# +# Each module is checked once per GOOS and the runs are merged, which is how +# staticcheck reports code unused on every OS rather than unused on one: a +# function only the Windows build calls is used. Any finding fails. The +# checker is the pinned release below, built with this toolchain; set +# STATICCHECK to use one already installed. +set -eu +HERE="$(cd "$(dirname "$0")/.." && pwd)" +VERSION=v0.8.2 # staticcheck 2026.2.2 +SC="${STATICCHECK:-}" +if [ -z "$SC" ]; then + BIN="$(mktemp -d)" + GOBIN="$BIN" GOFLAGS= go install "honnef.co/go/tools/cmd/staticcheck@$VERSION" + SC="$BIN/staticcheck" +fi +OUT="$(mktemp -d)" +bad=0 +for mod in client-go deploy/release-sign; do + for os in linux darwin windows; do + (cd "$HERE/$mod" && GOOS=$os "$SC" -f binary ./... > "$OUT/$os.bin") + done + found="$(cd "$HERE/$mod" && "$SC" -merge "$OUT/linux.bin" "$OUT/darwin.bin" "$OUT/windows.bin" || true)" + if [ -n "$found" ]; then + printf '%s\n' "$found" | sed "s#^#$mod/#" + bad=1 + else + echo "ok $mod: no findings on linux, darwin and windows" + fi +done +rm -rf "$OUT" +exit "$bad" diff --git a/deploy/test_github_release.py b/deploy/test_github_release.py new file mode 100644 index 0000000..961d021 --- /dev/null +++ b/deploy/test_github_release.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +"""deploy/github-release.py against a fake GitHub API on loopback. + + python3 -I deploy/test_github_release.py + +A release that does not exist is made with the version's CHANGELOG section +and every file attached; a re-run replaces each asset of the same name and +leaves an existing release's name and notes alone; the token goes only in +the Authorization header. +""" + +import importlib.util +import json +import os +import sys +import tempfile +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +sys.dont_write_bytecode = True # leave no __pycache__ beside the script under test +HERE = os.path.dirname(os.path.abspath(__file__)) +spec = importlib.util.spec_from_file_location("ghrel", os.path.join(HERE, "github-release.py")) + + +class Fake: + def __init__(self): + self.release = None + self.calls = [] + self.next_id = 100 + + +def handler(fake): + class H(BaseHTTPRequestHandler): + def log_message(self, *a): + pass + + def reply(self, code, obj=None): + raw = json.dumps(obj).encode() if obj is not None else b"" + self.send_response(code) + self.send_header("Content-Length", str(len(raw))) + self.end_headers() + self.wfile.write(raw) + + def body(self): + return self.rfile.read(int(self.headers.get("Content-Length", 0))) + + def do_GET(self): + fake.calls.append(("GET", self.path, self.headers.get("Authorization"))) + if self.path == "/repos/o/r/releases/tags/v1.2.3": + return self.reply(200, fake.release) if fake.release else self.reply(404, {"message": "Not Found"}) + self.reply(404, {}) + + def do_POST(self): + data = self.body() + fake.calls.append(("POST", self.path, self.headers.get("Authorization"))) + if self.path == "/repos/o/r/releases": + req = json.loads(data) + fake.release = {"id": 7, "tag_name": req["tag_name"], "name": req["name"], "body": req["body"], "assets": []} + return self.reply(201, fake.release) + if self.path.startswith("/repos/o/r/releases/7/assets?name="): + name = self.path.split("name=", 1)[1] + fake.next_id += 1 + fake.release["assets"].append({"id": fake.next_id, "name": name, "size": len(data)}) + return self.reply(201, {"id": fake.next_id, "name": name, "size": len(data)}) + self.reply(404, {}) + + def do_DELETE(self): + fake.calls.append(("DELETE", self.path, self.headers.get("Authorization"))) + aid = int(self.path.rsplit("/", 1)[1]) + fake.release["assets"] = [a for a in fake.release["assets"] if a["id"] != aid] + self.reply(204) + return H + + +def main(): + fake = Fake() + srv = ThreadingHTTPServer(("127.0.0.1", 0), handler(fake)) + threading.Thread(target=srv.serve_forever, daemon=True).start() + base = "http://127.0.0.1:%d" % srv.server_address[1] + os.environ.update({"GITHUB_API_URL": base, "GITHUB_UPLOADS_URL": base, "GH_TOKEN": "t0ken"}) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + with tempfile.TemporaryDirectory() as d: + assets = os.path.join(d, "assets") + os.mkdir(assets) + for name, data in (("privatae-bastion-linux-amd64", b"bin"), ("version.json", b"{}")): + open(os.path.join(assets, name), "wb").write(data) + cl = os.path.join(d, "CHANGELOG.md") + open(cl, "w").write("# Changelog\n\n## 1.2.4\n\n- later\n\n## 1.2.3\n\n- **One change.**\n\n## 1.2.2\n\n- earlier\n") + + assert mod.main(["x", "o/r", "v1.2.3", assets, cl]) == 0 + rel = fake.release + assert rel["name"] == "bastion 1.2.3", rel + assert rel["body"].startswith("- **One change.**") and "later" not in rel["body"] and "earlier" not in rel["body"], rel["body"] + assert sorted(a["name"] for a in rel["assets"]) == ["privatae-bastion-linux-amd64", "version.json"] + + rel["name"], rel["body"] = "kept", "kept notes" + open(os.path.join(assets, "version.json"), "wb").write(b'{"v":1}') + assert mod.main(["x", "o/r", "v1.2.3", assets, cl]) == 0 + assert (rel["name"], rel["body"]) == ("kept", "kept notes") + sizes = {a["name"]: a["size"] for a in rel["assets"]} + assert sizes == {"privatae-bastion-linux-amd64": 3, "version.json": 7}, sizes + assert sum(1 for c in fake.calls if c[0] == "DELETE") == 2 + assert all(c[2] == "Bearer t0ken" for c in fake.calls) + assert not any("t0ken" in c[1] for c in fake.calls) + + try: + mod.main(["x", "o/r", "v9.9.9", assets, cl]) + except SystemExit as e: + assert "no section for 9.9.9" in str(e), e + else: + raise AssertionError("a version with no CHANGELOG section must not get a release") + srv.shutdown() + print("ok - github-release.py makes, fills and refills a release") + + +if __name__ == "__main__": + main() diff --git a/docs/claims.md b/docs/claims.md index 75ba6e4..31135b4 100644 --- a/docs/claims.md +++ b/docs/claims.md @@ -67,6 +67,7 @@ says so. Those parts are enforced only by the code as written. | | `ops_linux_test.go`: `TestOpDryRunShowsTheChangeAndApplyCommitsExactlyThat` | A dry run commits nothing; an apply must name the dry run's change list and carry an approval bound to that op, its parameters, the machine and that change list. Skipped where the sandbox is unavailable. | | | `ops_test.go`: `TestOpTermsMutationsAreEachRefused` | Each altered field of signed op terms is refused. | | | `ops_linux_test.go`: `TestOpSandboxShowsOnlyTheBaseAndWhatTheOpDeclared` | In a real sandbox, an op reads the file it declared and the system base; a file beside its own folder, a folder beside it and `/etc/shadow` read as empty; a declared path that has since become a link onto `/etc/ssh` refuses the run. Skipped where the sandbox cannot run. | +| | `ops_taskcap_linux_test.go`: `TestARootOpIsStoppedAtItsTaskCap` | A root op starting 100 background tasks is stopped at its scope's cap, and its report says the cap held. The scope is made by a stand-in for `systemd-run` that creates the cgroup directly. Skipped where the sandbox, systemd as init or a pids controller is unavailable. | | | `ops_view_test.go`: `TestOpSensitivePathsAreRefusedAtUnderOrAboveAndThroughLinks` | A declared path at, under or above a credential store, as written or through a link, is refused; a malformed `sensitive_paths` refuses every op. | | | `fuzz_paths_unix_test.go`: `FuzzAnOpEffectNeverCoversTheMachinesOwnRecord` | No accepted effect covers the bastion's own records or the system directories the sandbox masks. | | An action op's program reaches only what it declared | `ops_action_linux_test.go`: `TestActionProgramReachesOnlyItsDestinations` | In a real unit, an apply reaches its declared destination by name; another port on that address, the declared port on another address, this machine's loopback and the internet are refused. A dry run reaches nothing, its declared destination included, and a name that resolves to the metadata service is refused. Skipped where action ops cannot run. | @@ -76,7 +77,8 @@ says so. Those parts are enforced only by the code as written. | | `ops_action_linux_test.go`: `TestActionVerbCredentialNeverReachesTheRealProgram` | In a real unit, a credential a `tree.checkout` names never reaches the program, while the verb uses it and the program's own credential is handed and scrubbed from its output. Skipped where action ops cannot run. | | An action op is never capped, and yields first | `ops_action_linux_test.go`: `TestActionMemoryIsNotCappedAndTheOpYieldsFirst` | In a real unit, a run holds 1500 MiB resident with nothing limiting it, and the kernel would kill it first (`oom_score_adj` 1000). When the machine's free memory falls to the floor, the op is stopped, with a refusal that names its peak. A machine already under the floor starts no op and spends no uid. Skipped where action ops cannot run or 10 GiB is not free. | | | `ops_action_linux_test.go`: `TestActionYieldsWhenTheServicesWaitOnMemoryNotForItsOwnStalls` | Against a stand-in tree of pressures: neither the op's own slice waiting nor a pod group already waiting stops it; the services' slice rising to 12.5% during the run stops it within seconds; a group that stays at a level does not stop the next run; a group that spiked as the run began and fell back does not hide a later rise. Skipped where action ops cannot run. | -| | `ops_action_linux_test.go`: `TestActionProgramSeesNothingItWasNotGiven` | (also) The real unit's program sits in `privatae_ops.slice`, the slice the pressure reading leaves out. | +| | `ops_action_linux_test.go`: `TestActionProgramSeesNothingItWasNotGiven` | (also) The real unit's program sits in the ops' slice (`privatae_ops.slice`; in the suite, a slice of its own), whose own figure the pressure reading leaves out. | +| | `ops_memory_stalls_linux_test.go`: `TestAnActionOpYieldsToARootOpBesideItNeverToItself` | Against stand-in pressures: a root op's scope in the ops' slice rising 12.4% beside an action op counts as another program waiting, while the action op's own unit and the slice's own figure rising never count. | | | `ops_action_linux_test.go`: `TestTheMemoryWatchReportsOnlyAKillThatLanded` | A kill that finds no unit is not reported as the op stopped. | | | `ops_action_linux_test.go`: `TestTheMemoryWatchKeepsTryingUntilTheKillLands` | With the machine short before the op's unit exists, the first kill finds nothing; the real unit that starts a second later is still stopped. Skipped where action ops cannot run. | | | `ops_action_linux_test.go`: `TestMemoryFloorSitsAboveTheKubeletsEviction` | The floor reads the kubelet's eviction line from the k3s config, its drop-ins, the k3s unit's flags, the kubelet config k3s writes, and any kubelet config file or folder a flag names (one named and missing counts as unreadable): percentages, decimals, binary and decimal units, soft as well as hard, and a kubelet config file's form, never a comment. It sits 512 MiB above that line, or at 5% of memory, or at 1 GiB, whichever is largest. A value it cannot read raises the floor to 10% plus 512 MiB. | @@ -131,8 +133,16 @@ says so. Those parts are enforced only by the code as written. | | `ownaddr_test.go`: `TestCoderEgressRefusesAnAddressTheKernelDeliversHere` | A coder run is refused an address the kernel delivers here, in every spelling, and refused anything when the lookup fails; `allow_lan` stays the exception. | | | `ownaddr_netns_linux_test.go`: `TestAnyIPDestinationsAreRefusedInARealNamespace` | In a private network namespace with `local 192.0.2.0/24` and `local 2001:db8:a::/64` on `lo`, plus a broadcast route (198.51.101.255), anycast routes (198.51.102.0/24, 2001:db8:d::/64) and a multicast route: the real route lookup reads each as this machine, and a unicast route or no route as not; a 0.0.0.0 listener answers on both local ranges. A coder run's proxy and the anchored egress lane each refuse those addresses (local, broadcast and anycast), the IPv4-mapped and NAT64 forms of each IPv4 one, and a name resolving to each, with the lane's this-machine reason (a name the lane could not resolve does not count), and pass the unicast and no-route addresses and a name resolving to each, dialling the vetted address; each answers 403 end to end with a real dialer, the proxy over its socket and the anchored lane through its connection handler as the anchor host, to CONNECT by address and by name and to plain HTTP. The backup lanes' URL check refuses the same addresses, forms and names as this machine's or its network's and passes the unicast and no-route ones, and their dial wall refuses the local, broadcast and anycast addresses. Nothing reaches the listener. Skipped where a namespace cannot be made. | | | `ownaddr_netns_linux_test.go`: `TestOnlyAUnicastRouteLeavesThisMachine` | The kernel's route type is read so that only unicast leaves this machine; local, broadcast, anycast, multicast, unspec, nat and xresolve are this machine. | -| | `hostops_test.go`: `TestAnchoredEgressRefusesLoopbackAndPrivateDestinations` | On an anchored machine the egress lane refuses loopback, `localhost`, private, ULA, link-local, CGNAT and metadata addresses, dials the vetted address, and a local machine keeps the lane it had. | -| | `ownaddr_test.go`: `TestAnchoredEgressRefusesThisMachinesOwnAddresses` | On an anchored machine the egress lane refuses an address on this machine's interface and one a `local` route delivers here, in every spelling, and refuses when the lookup fails; a neighbour passes; a local machine's lane is unchanged. | +| | `hostops_test.go`: `TestAnchoredEgressRefusesLoopbackAndPrivateDestinations` | On an anchored machine the egress lane refuses loopback, `localhost`, private, ULA, link-local, CGNAT and metadata addresses and dials the vetted address; a local machine refuses its own ollama and keeps its LAN. | +| | `ownaddr_test.go`: `TestAnchoredEgressRefusesThisMachinesOwnAddresses` | On an anchored machine the egress lane refuses an address on this machine's interface and one a `local` route delivers here, in every spelling, and refuses when the lookup fails; a neighbour passes; a local machine refuses those addresses too. | +| | `egress_local_test.go`: `TestLocalEgressRefusesThisMachineAndKeepsTheLAN` | On a local machine the egress lane refuses loopback, `localhost`, unspecified, `0.0.0.0/8`, link-local and metadata addresses, this machine's own LAN address and AnyIP range, each also inside an IPv4-mapped, NAT64 or 6to4 address, and a name with one such answer; it dials private, ULA, CGNAT and public addresses, and a name's vetted addresses, never the name. | +| | `ownaddr_inside_linux_test.go`: `TestNetworksInsideThisMachineAreThisMachine` | In a private network namespace with two bridges whose only port is a veth (one with an IPv4 and an IPv6 range, one with a public range), a bridge whose port is a dummy, a bridge with no port, a veth routed to directly, a range routed through a veth with this machine's own address on it as the gateway, a veth whose only default route is in another table, a bridge over a VLAN, a route over a dummy link, the network on three veths that carry this machine's main-table default routes (an IPv4 multipath default over two, an IPv6 default over the third: this machine as a container), addresses through those defaults' gateways and a route through a gateway on the dummy-port bridge (a LAN behind a router VM): the real route lookup reads the first seven as this machine and the rest as leaving; a local machine's egress lane refuses the inside addresses, by address and by name, and dials the others; a coder run's proxy refuses the public range on an inside network and dials a public address through the gateway; the backup lanes allow the IPv4 and IPv6 addresses through the gateway. | +| | `ownaddr_inside_linux_test.go`: `TestInsideMachineReadsTheLinkAndABridgesPorts` | A veth, and a bridge whose every port is a veth, a tap or a dummy (or that has none), keep a packet on this machine; a bridge with a network card, a VLAN or a bridge among its ports, a network card, loopback, a VPN, a tunnel and a macvlan do not. | +| | `egress_local_test.go`: `TestLocalEgressLaneNeverDialsThisMachine` | Through the lane on a local machine with a real dialer: a request for a loopback listener gets 403 over CONNECT and plain HTTP and never reaches it; a LAN name is dialled at its vetted address; a dual-stack name falls back to its second address. | +| | `egress_dial_test.go`: `TestDialEachDoesNotWaitOutAnAddressThatGoesNowhere` | The dial both egress lanes share tries only the vetted addresses, in order; an address that never answers holds up the next for well under its share of the budget, and is cancelled once the next answers. | +| | `egress_local_test.go`: `TestLocalModelsSwitchLeavesNoEgressRouteToOllama` | On a local machine with `models` switched off for the host, the ollama forward filters, and the egress lane refuses `/api/pull` to ollama on loopback, `localhost` and this machine's LAN address without a dial. | +| | `egress_local_test.go`: `TestAnchoredEgressStillRefusesTheLAN` | An anchored machine's egress lane still refuses the LAN addresses a local machine reaches, with its own wording, and passes the public internet. | +| | `egress_obsolete_test.go`: `TestEveryLaneRefusesTheDeprecatedIPv6Forms` | The anchored and the local egress lane each refuse IPv4-compatible (`::/96`, with a public, a loopback and a LAN address inside), IPv4-translated (`::ffff:0:0/96`) and site-local (`fec0::/10`) addresses, and a name with one such answer among public ones, and pass the public internet; a coder run's proxy refuses the same addresses. | | | `ownaddr_test.go`: `TestBackupLanesRefuseThisMachinesOwnAddresses` | The backup lanes' URL check refuses an address on this machine's interface and one a `local` route delivers here, in every spelling and through a name, and their dial wall refuses those addresses; a failed lookup refuses both; a neighbour on the subnet passes. With the kernel's answer supplied. | | | `backupfetch_test.go`: `TestBackupWallJudgesTheIPv4InsideAndTheReservedRanges` | The backup lanes' classifier, address wall and URL check refuse 10.0.0.1 and 127.0.0.1 inside NAT64, 10.0.0.1 inside 6to4 and in the IPv4-compatible and IPv4-translated forms, the metadata address inside 6to4, a local-use NAT64 address, `fec0::1`, `198.18.0.0/15` (also inside an IPv4-mapped address), `240.0.0.0/4` (also inside NAT64) and `255.255.255.255`; public addresses in the NAT64, 6to4 and IPv4-mapped forms, and the addresses either side of `198.18.0.0/15`, pass the classifier. | | | not covered by a test | A public address that reaches this machine only through a 1:1 NAT in front of it. Nothing refuses it. | @@ -153,7 +163,8 @@ says so. Those parts are enforced only by the code as written. | | `session_test.go`: `TestSessionRootApprovalBindsToSession` | The purpose a root approval is bound to differs between two session ids on the same machine. Compared as strings, not through the terminal handler. | | | `session_test.go`: `TestSessionFramingResizeAndTranscript` | An opened session writes a transcript and start and stop lines to the audit log. | | | `hostops_test.go`: `TestTheDeviceSignsOnlyWhatItDerivedItself` | The approval device refuses to take a root-terminal ask. | -| Root commands run only under a grant given here with the code | `rootexec_test.go`: `TestRootExecGrantCeremony` | The grant exists only after a code from the factor enrolled here checks out, once per code, with an expiry no longer than a year; past its expiry or after a revoke it refuses. | +| Root commands run only under a grant given here with the code | `rootexec_test.go`: `TestRootExecGrantCeremony` | The grant exists only after a code from the factor enrolled here checks out, once per code, with an expiry no longer than 30 days; past its expiry or after a revoke it refuses. | +| | `rootexec_test.go`: `TestRootExecGrantIsAtMostThirtyDays` | A 31-day grant is refused and writes nothing, 30 days is granted, and a grant held with a longer term ends 30 days after it was given; one with no grant time is refused. | | | `rootexec_test.go`: `TestRootExecRefusals` | Through the terminal handler: with no grant, an expired, revoked, other-host or other-machine grant, from a member pairing holding the `session` family, or with the bastion not root, the command is refused, never starts, leaves no transcript and is audited with its command. | | | `rootexec_test.go`: `TestRootExecLocalSwitchRefuses` | With `session` switched off here, a granted command is refused. | | | `rootexec_test.go`: `TestRootExecRoundTrip` | A granted command returns its stdout, stderr and exit status, and is written to a transcript and to the audit log at start and stop with the command. | diff --git a/docs/context.md b/docs/context.md index c990eea..22958bd 100644 --- a/docs/context.md +++ b/docs/context.md @@ -17,3 +17,13 @@ audited on this machine. The vocabulary is reads plus **one write lane**: a signed move plan that you confirmed on this machine, which archives files and never deletes them. Dropping a grant removes the grant, not your files. **No delete op exists.** + +A grant with `search` runs the bastion agent on the folder. Its search +results, read ranges and catalog cards come from the agent's sanitized text, +and each reply says how that text was sanitized: `sanitize_mode` is the mode +in the agent's manifest, `sanitized` is true only for `sterilize`, and +`sanitize_note` says what the mode leaves in. The bastion deploys the agent in +`flag` mode, which replaces e-mail addresses, phone numbers, card, bank and ID +numbers, IP and MAC addresses and keys, and does **not** find names, street +addresses or dates of birth. So a Host reading your folder through search +sees those. diff --git a/docs/host-on-this-machine.md b/docs/host-on-this-machine.md index 4c04ac1..0bea42a 100644 --- a/docs/host-on-this-machine.md +++ b/docs/host-on-this-machine.md @@ -60,14 +60,29 @@ carries a tag of the named repository. ## A Host in a VM on this machine A Host with no platform that runs in a VM on this machine shares no kernel and -no directory with it. Its installer forwards two ports of the VM to this -machine's loopback and hands over a pairing token with the Host's CA: +no directory with it. A C-Series Host +([Privatae/cseries](https://github.com/Privatae/cseries)) is one: it runs in +its own Linux virtual machine. Its installer, `npx cseries`, forwards two ports +of the VM to this machine's loopback, installs this bastion as the machine's +system service, and on every pairing runs these in one elevated step (on +Windows, from an Administrator PowerShell instead of `sudo -H`): ```sh -privatae-bastion pair --redeem-url https://127.0.0.1:8810/v1/local/redeem --host-ca host-ca.pem -privatae-bastion run +# 1. The service's state, networked. +sudo -H privatae-bastion set-mode networked +# 2. Pair over the Host's enroll port, trusting its CA alone; the token is read +# from standard input, never from the arguments. +sudo -H privatae-bastion pair --redeem-url https://127.0.0.1:8810/v1/local/redeem --host-ca host-ca.pem - +# 3. Anchor the machine to that Host: every write and terminal waits for your approval. +sudo -H privatae-bastion claim-anchor +# 4. Your authenticator, enrolled on that console and nowhere else. +sudo -H privatae-bastion stepup-enroll --console-only --host ``` +No Privatae account is needed. A plain `pair` and `run` as your own user also +pair with such a Host, but leave the machine local: not anchored, and with no +second factor until you enrol one. + | port | what | |---|---| | `8810` | `pair` redeems its token here (`POST /v1/local/redeem`, the platform redeem's body) | diff --git a/docs/lanes.md b/docs/lanes.md index 74b4dfe..3112ab2 100644 --- a/docs/lanes.md +++ b/docs/lanes.md @@ -49,8 +49,11 @@ privatae-bastion root-exec revoke [--host ] # take ``` The grant names one host (the anchor; a member pairing never gets it) and -this machine, ends at the time you chose, and is written only by that command -with the code; nothing the host sends can create, widen or extend it. Each +this machine, ends at the time you chose, at most 30 days after you gave it +(`--for 31d` is refused; grant again before it ends), and is written only by +that command with the code; nothing the host sends can create, widen or +extend it. A grant given before 0.55.0, when the longest term was a year, +ends 30 days after it was given. Each command runs as `/bin/sh -c` in its own process group, with a deadline (10 minutes unless the host asks for less or more, never more than 8 hours) and with stdout and stderr returned up to a size cap (256 KiB each unless the host diff --git a/docs/models.md b/docs/models.md index 61a8128..32f3577 100644 --- a/docs/models.md +++ b/docs/models.md @@ -13,11 +13,26 @@ op. A pull is reported available only after a behavioural probe, because a completed download proves only that bytes arrived. Note the boundary: those rules govern the `model:*` ops. The host's direct -traffic to the ollama forward inside the tunnel is filtered (store-changing -requests such as `/api/pull` and `/api/delete` refused, model names checked) -only when the pairing is **scoped to named models**, when you have switched -the `models` capability off in the local console, or when a networked -bastion's grant withholds model management. Otherwise, on a local bastion -with an unscoped pairing, that forward is a raw byte-for-byte splice to your -ollama, and the pairing itself is the authorization. Scope the pairing on -your host, or switch `models` off here, if you want the filter. +traffic to the ollama forward inside the tunnel is a separate path, and it is +filtered (store-changing requests such as `/api/pull` and `/api/delete` +refused) in these cases: + +- **On an anchored machine**, for every pairing, the anchor included. Changing + which models are there goes through `model:pull` and `model:drop`. A member + host whose grant from the anchor is **scoped to named models** also has + every request's model name checked against that list. +- **On a local machine**, once you switch the `models` capability off for + that host in the local console (`privatae-bastion ui`). On a root install + kept local (`--local`), `sudo -H privatae-bastion switch models off` + does the same. On an install as your own user that command reads root's + state, where your host is not paired, so it switches nothing; use the + console. + +Otherwise, on a local machine, that forward is a raw byte-for-byte splice to +your ollama, and the pairing itself is the authorization. A local machine +reads no grant for this, so a scope set on the host does nothing there: +switching `models` off is how you stop a host there changing your models. +The egress lane is no way around that switch: on a local machine it never +reaches this machine's own addresses (see +[the security model](security-model.md)), so ollama on loopback or on this +machine's LAN address cannot be reached through it either. diff --git a/docs/ops.md b/docs/ops.md index 2ce6354..b4bdb83 100644 --- a/docs/ops.md +++ b/docs/ops.md @@ -61,8 +61,8 @@ a dry run first, then an apply bound to the change list the dry run showed. kubelet's own eviction line (hard or soft) plus 512 MiB, so the op goes before any pod is evicted. A kubelet setting it cannot read raises the floor, never lowers it; - - the bastion also stops the op when the machine's services, pods or users - start waiting on memory: 7 points more of the time than the least they + - the bastion also stops the op when the machine's services, pods, users or + another op running beside it start waiting on memory: 7 points more of the time than the least they had during the run. The op's own waits don't count, and neither does a group that was already waiting, such as a pod at its own limit; - a machine already under the floor starts no op; diff --git a/docs/security-model.md b/docs/security-model.md index 4af28fd..e78d93f 100644 --- a/docs/security-model.md +++ b/docs/security-model.md @@ -64,19 +64,43 @@ - **Egress for someone else reaches the public internet, never this machine.** A coder run's proxy, always, and the egress lane on an anchored machine refuse loopback, link-local (the cloud metadata address included), - private, CGNAT and reserved addresses, and any address this machine takes + private, CGNAT and reserved addresses, the deprecated IPv4-compatible + (`::/96`), IPv4-translated (`::ffff:0:0/96`) and site-local (`fec0::/10`) + IPv6 forms, and any address this machine takes as its own: one on its interfaces (a public one included), and on Linux any address the kernel's routing table delivers here (a `local`, broadcast, anycast or multicast route, such as an AnyIP range on `lo`), asked of the - kernel per connection. The IPv4 address inside an IPv4-mapped, NAT64 or - 6to4 address is checked too, every address a name resolves to is checked, - and the address checked is the one dialled. When the machine cannot tell, + kernel per connection, and on Linux any address the kernel routes straight + onto a link that stays on this machine: a container's veth, or a bridge + whose every port is a veth, a tap or a dummy, which is the bridge network + Docker, Podman, LXD and libvirt each make for their containers and VMs. A + bridge with a network card, a bond or a VLAN among its ports is the LAN, a + VPN or a tunnel to other machines leaves, and so does a route through a + gateway, which sends the packet on. A link an IPv4 or IPv6 default route + of the main table goes out of is never inside: it is this machine's way + out, so a bastion that is itself in a container dials the network on its + own veth, its LAN included. The IPv4 address inside an + IPv4-mapped, NAT64 or 6to4 address is checked too, every address a name + resolves to is checked, and the address checked is the one dialled. When the machine cannot tell, it refuses. A coder run reaches a LAN host only when you named it in `coders set --allow-lan`; the anchored egress lane has no such exception. The anchored lane's refusal of this machine's own non-private addresses is - new in 0.50.0; before it, that lane refused only the internal ranges. A - local (unanchored) machine's egress lane is unchanged and vets no - destination. The backup lanes (`backup_fetch` and `backup_push`, anchored + new in 0.50.0; before it, that lane refused only the internal ranges. On a + local (unanchored) machine the egress lane refuses this machine too: + `localhost` names, loopback and unspecified addresses (`0.0.0.0/8` and + `::` included), link-local addresses (the cloud metadata address included), + the rest of `64:ff9b::/32`, the three deprecated IPv6 forms, and any + address this machine takes as its own, checked as above and also inside an + IPv4-mapped, NAT64 or 6to4 address. It + keeps your LAN: a private, CGNAT or reserved address that is not this + machine's is dialled. A container or VM on a network made on this machine + is this machine, not the LAN; on macOS and Windows, which have no route + lookup here, such an address is dialled. Every address a name resolves to is checked there + too, and only those addresses are dialled, in order, the next as soon as + one fails or none has answered within 300 ms. This is new in + 0.55.0; before it, a local machine's egress lane vetted no destination, so + a host could reach this machine's own services through it, ollama's + `/api/pull` among them, whatever the `models` switch said. The backup lanes (`backup_fetch` and `backup_push`, anchored or not) apply the same check of this machine's own addresses, to every address the URL's name resolves to, to each redirect a fetch follows and to the address dialled; this too is new in 0.50.0, and before it they refused diff --git a/scripts/test_no_personal_data.py b/scripts/test_no_personal_data.py index 074bae1..ca60b18 100644 --- a/scripts/test_no_personal_data.py +++ b/scripts/test_no_personal_data.py @@ -17,31 +17,41 @@ lands here is published far past where it came from. This fails on: - term a word or phrase from TERMS: the owner's and members' details that - were once copied into a repository (bank, employer, account - digits, places, names, addresses, a device, a customer, a person's - project, a Host's content, record ids). They are held as keyed + term a word or phrase from TERMS, a list of details about real people + that must not appear in this repository. They are held as keyed digests, so this file does not publish what it keeps out, and a guess cannot be tested against the table without the key: every 1-3 token n-gram of a line is normalised (lowercase letters and digits, joined by one space), HMAC-SHA256'd with the maintainers' - key and looked up. The key lives outside the repository + key and looked up. A backslash escape (\\n, \\t, \\x41, \\u00e9...) + separates tokens as whitespace would, so a term written right + after one is still found. The key lives outside the repository (NO_PERSONAL_DATA_KEY_FILE, default /etc/privatae/keys/no-personal-data.hmac), as do the plain list and the script that hashes it. Where the key is absent (a CI job, which holds no secrets) this class is skipped and the run says - so; the other classes always run. A key other than the one TERMS + so; the other classes still run. A key other than the one TERMS was made with fails the run (KEY_CHECK), since it would match nothing. - entity-id a real entity id that labels a person's Host (RESERVED_ENTITY_IDS) - in entity-N, entity N, entity_id=N, "entity_id": N, entity id N, - Host N, Host #N or host-N form; examples and tests use 9000 and up + entity-id a reserved entity id, one that labels a person's Host, in + entity-N, entity N, entity_id=N, "entity_id": N, entity id N, + Host N, Host #N or host-N form; examples and tests use 9000 and + up. The ids live outside the repository, one per line + (RESERVED_ENTITY_IDS_FILE, default + /etc/privatae/reserved-entity-ids). Where that file is absent (a + CI job) this class is skipped and the run says so; a file that + holds no id, or a line that is not one, fails the run, since it + would match nothing. provenance a line that says its data is real: "real rows" followed by "that" or "from", an all-caps "REAL" row, "verbatim" from a transcript or a run, a row "read off" an entity... email a mailbox at a personal mail provider (gmail, yahoo, proton...) phone a North American number outside the 555 fiction range +A run of 24 or more base64 or base64url characters that decodes to mostly +printable UTF-8 is decoded and its text checked for every class as well, so a +value is not let through by being encoded. + Every tracked text file is scanned, file names included, and every text member of each zip-format bundle (an app bundle is one), since a bundle is what installs. Lockfiles and other binary files are not. @@ -63,6 +73,8 @@ Exit 0 = clean; exit 1 + FAIL lines otherwise. from __future__ import annotations +import base64 +import binascii import hmac import os import re @@ -118,13 +130,43 @@ TERMS: frozenset[str] = frozenset({ }) _NGRAM_MAX = 3 -# Entity ids of real Hosts that belong to people (the operator's own, members', -# the platform's Security entity and the two security-test Hosts). -RESERVED_ENTITY_IDS = (3, 102, 122, 123, 128, 163, 164) -_IDS = "|".join(str(i) for i in RESERVED_ENTITY_IDS) -ENTITY_ID = re.compile( - r"\b(?:entit(?:y|ies)(?:[-_ ]|_id\s*==?\s*|_id[\"']?\s*:\s*|\s+id\s+|\s+#)" - r"|Host\s+#?|host-)(?:%s)\b" % _IDS, re.I) +# The reserved entity ids are read from a file outside the repository, one id +# per line; blank lines and lines starting with # are ignored. +RESERVED_ENTITY_IDS_FILE = os.environ.get("RESERVED_ENTITY_IDS_FILE", "/etc/privatae/reserved-entity-ids") + + +def load_reserved_ids(path: str = RESERVED_ENTITY_IDS_FILE): + """(ids, error). A missing file is (None, None): the class is skipped. A + file that cannot be read, holds a line that is not an id, or holds no id + at all is (None, error): a list that matches nothing must not pass.""" + try: + with open(path, encoding="utf-8") as f: + lines = f.read().splitlines() + except FileNotFoundError: + return None, None + except (OSError, UnicodeDecodeError) as e: + return None, "the reserved entity ids at %s cannot be read: %s" % (path, e) + ids = [] + for n, raw in enumerate(lines, 1): + s = raw.strip() + if not s or s.startswith("#"): + continue + if not s.isdigit(): + return None, "%s:%d is not an entity id (one id per line)" % (path, n) + ids.append(int(s)) + if not ids: + return None, "%s holds no entity id, so the entity-id check would match nothing" % path + return tuple(ids), None + + +def entity_id_pattern(ids): + """The entity-id class for these ids, or None (the class is off).""" + if not ids: + return None + alts = "|".join(str(i) for i in sorted(set(ids), key=lambda i: -len(str(i)))) + return re.compile( + r"\b(?:entit(?:y|ies)(?:[-_ ]|_id\s*==?\s*|_id[\"']?\s*:\s*|\s+id\s+|\s+#)" + r"|Host\s+#?|host-)(?:%s)\b" % alts, re.I) PROVENANCE = re.compile( r"\breal rows? (?:that|from|read|copied|taken|off)\b" @@ -181,13 +223,25 @@ def key_matches(key: bytes) -> bool: return hmac.compare_digest(digest(KEY_CHECK_TEXT, key), KEY_CHECK) +# A backslash escape as source and data files write it: a named control +# character, a hex or unicode escape, or an octal one. "a\nb" is two words, but +# the tokenizer alone reads "a" and "nb". The line is read once as written and +# once with each escape turned into a space; a hit in either counts. Reading it +# as written too keeps a term that follows a backslash which is not an escape +# (a Windows path: C:\name) whole. +_ESCAPE = re.compile(r"\\(?:x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8}|[0-7]{1,3}|[abefnrtv])") + + class TermScanner: """Finds TERMS in a line by hashing its 1-3 token n-grams. Digests are - cached, so a repeated n-gram is hashed once.""" + cached, so a repeated n-gram is hashed once. It also carries the + entity-id pattern for the reserved ids (None: that class is off).""" - def __init__(self, terms, key): + def __init__(self, terms, key, entity_ids=None): self.terms = terms self.key = key + self.entity_ids = entity_ids + self.entity_re = entity_id_pattern(entity_ids) self._cache = {} def _hit(self, gram: str) -> bool: @@ -195,16 +249,64 @@ class TermScanner: self._cache[gram] = digest(gram, self.key) in self.terms return self._cache[gram] + def _grams(self, text: str) -> dict: + toks = [t.lower() for t in _TOKEN.findall(text)] + hits = {} + for n in range(1, _NGRAM_MAX + 1): + for i in range(len(toks) - n + 1): + gram = " ".join(toks[i:i + n]) + if self._hit(gram): + hits[gram] = hits.get(gram, 0) + 1 + return hits + def scan(self, line: str) -> int: if self.key is None: return 0 - toks = [t.lower() for t in _TOKEN.findall(line)] - found = 0 - for n in range(1, _NGRAM_MAX + 1): - for i in range(len(toks) - n + 1): - if self._hit(" ".join(toks[i:i + n])): - found += 1 - return found + hits = self._grams(line) + if "\\" in line: + for gram, n in self._grams(_ESCAPE.sub(" ", line)).items(): + hits[gram] = max(hits.get(gram, 0), n) + return sum(hits.values()) + + +# ── values inside base64 ───────────────────────────────────────────────────── +# A run of base64 or base64url characters, with its padding. The run is decoded +# at each of the four offsets, since it may be glued to the text before it or +# cut by a window edge; an offset whose bytes are mostly printable UTF-8 is the +# text. Random bytes almost never are, so identifiers, paths and digests that +# happen to be base64-shaped decode to nothing worth reading. +_B64_RUN = re.compile(r"[A-Za-z0-9+/_-]{24,}={0,2}") +_B64_MIN = 24 +_B64_PRINTABLE = 0.85 +# A decoded text may itself hold base64; it is followed this many levels down. +_B64_DEPTH = 2 + + +def _printable_share(text: str) -> float: + if not text: + return 0.0 + ok = sum(1 for ch in text if ch in "\t\n\r" or (ch.isprintable() and ch != "\ufffd")) + return ok / len(text) + + +def b64_texts(line: str): + """The text of every base64 run in line that decodes to mostly printable + UTF-8, one per run.""" + for m in _B64_RUN.finditer(line): + run = m.group(0).rstrip("=").replace("-", "+").replace("_", "/") + for off in range(4): + body = run[off:] + body = body[:len(body) - len(body) % 4] if len(body) % 4 == 1 else body + if len(body) < _B64_MIN: + break + try: + raw = base64.b64decode(body + "=" * (-len(body) % 4), validate=True) + except (ValueError, binascii.Error): + continue + text = raw.decode("utf-8", "replace") + if _printable_share(text) >= _B64_PRINTABLE: + yield text + break # A minified asset or a bundle member can hold one line of megabytes. It is read @@ -241,16 +343,66 @@ def line_hits(line: str, scanner: TermScanner): # Classes whose line would carry the personal value itself: their FAIL line # names the file, the line and the class, never the text, so a CI log does not -# publish what the gate keeps out. +# publish what the gate keeps out. A line that holds one shows no text for any +# finding, since another class's excerpt of the line would carry the value too. _QUIET = {"term", "email", "phone"} -def line_findings(line: str, scanner: TermScanner): +_PATH_SEP = re.compile(r"([/!])") + + +def shown_path(label: str, scanner: TermScanner) -> str: + """label as a FAIL line shows it. A path component (a directory, a file + name, a bundle member's name) that holds a quiet value is replaced by a + placeholder giving its depth, so the line says where without the text. A + value that runs across components withholds them all.""" + def quiet(text): + return any(cls in _QUIET for cls, _ in line_findings(text, scanner)) + + if not quiet(label): + return label + parts = _PATH_SEP.split(label) + + def masked(every): + out, depth = [], 0 + for p in parts: + if p in ("/", "!"): + out.append(p) + continue + depth += 1 + out.append("" % depth if every or quiet(p) else p) + return "".join(out) + + shown = masked(False) + return masked(True) if quiet(shown) else shown + + +def line_report(label: str, n: int, line: str, scanner: TermScanner, allow=(), used=None, + skip_entity_ids: bool = False): + """(class, FAIL text) for each finding on line n of label that no allow + entry covers.""" + used = set() if used is None else used + hits = list(line_hits(line, scanner)) + quiet = any(cls in _QUIET for _part, cls, _what in hits) + out = [] + where = None + for part, cls, what in hits: + if (cls == "entity-id" and skip_entity_ids) or allowed(allow, label, cls, part, used): + continue + if where is None: + where = shown_path(label, scanner) + shown = "" if quiet else part.strip()[:140] + out.append((cls, ("%s:%d [%s: %s] %s" % (where, n, cls, what, shown)).rstrip())) + return out + + +def line_findings(line: str, scanner: TermScanner, _depth: int = 0): out = [] for _ in range(scanner.scan(line)): out.append(("term", "a listed term")) - for m in ENTITY_ID.finditer(line): - out.append(("entity-id", m.group(0))) + if scanner.entity_re is not None: + for m in scanner.entity_re.finditer(line): + out.append(("entity-id", m.group(0))) for m in PROVENANCE.finditer(line): out.append(("provenance", m.group(0))) for m in EMAIL.finditer(line): @@ -258,6 +410,10 @@ def line_findings(line: str, scanner: TermScanner): for m in PHONE.finditer(line): if (m.group(2) or m.group(5)) != "555": out.append(("phone", "a phone number")) + if _depth < _B64_DEPTH: + for text in b64_texts(line): + for cls, what in line_findings(text, scanner, _depth + 1): + out.append((cls, what if what.endswith("(in base64)") else what + " (in base64)")) return out @@ -363,32 +519,61 @@ def self_test(): fails = [] probe_key = os.urandom(32) + # Invented ids stand in for the reserved ones, which this file never holds. + probe_ids = (9101, 9202, 9303) sc = TermScanner(frozenset(digest(g, probe_key) for g in ("zorblaxquent", "quilmar vesk", "tam orvel prax")), - probe_key) + probe_key, probe_ids) def classes(line): return {c for c, _ in line_findings(line, sc)} + def b64(s: str, url: bool = False) -> str: + enc = base64.urlsafe_b64encode if url else base64.b64encode + return enc(live(s).encode("utf-8")).decode("ascii").rstrip("=" if url else "") + positives = [(c, live(s)) for c, s in [ ("term", "pays into the Zorblaxquent ledger"), ("term", "QUILMAR-vesk sent it"), ("term", "a tam_orvel.prax file"), - ("entity-id", "entity_id=|3,"), ("entity-id", '"entity_id": |128'), ("entity-id", "Entity |164's app"), - ("entity-id", "kubectl -n entity-|3 exec"), ("entity-id", "pod entity-|3-0 restarted"), ("entity-id", "measured on Host #|163"), - ("entity-id", "gitea_user = 'host-|3'"), ("entity-id", "entity id |122 knocked"), + # A term right after a backslash escape, as a string literal or a JSON + # value writes it, and after a backslash that is not one. + ("term", r'"line one\nZorblax|quent"'), ("term", r"col\tzorblax|quent\tcol"), + ("term", r"end\r\nquilmar| vesk"), ("term", r"a\x41zorblax|quent"), ("term", r"x\u00e9zorblax|quent"), + ("term", r"C:\Users\zorblax|quent\notes"), + ("entity-id", "entity_id=|9101,"), ("entity-id", '"entity_id": |9202'), ("entity-id", "Entity |9303's app"), + ("entity-id", "kubectl -n entity-|9101 exec"), ("entity-id", "pod entity-|9101-0 restarted"), + ("entity-id", "measured on Host #|9303"), ("entity-id", "gitea_user = 'host-|9101'"), + ("entity-id", "entity id |9202 knocked"), ("provenance", "the real| rows that fired on the Host"), ("provenance", "the REAL| row 9"), ("provenance", "verbatim| from the transcript"), ("provenance", "read| off entity 9001"), ("email", "mail ana.b@gm|ail.com today"), ("email", "x@proto|nmail.com"), ("phone", "call +1 (2|02) 555-|0100 or +1 30|1 444 1234"), ("phone", "+1|3014441234"), ]] + # Every class is read inside base64 and base64url, padded or not, whole or + # glued to the text before it. + positives += [ + ("term", 'data: "%s"' % b64("a note that names Zorblax|quent outright")), + ("term", "token=%s" % b64("pays QUILMAR| vesk ~~ été ok??", url=True)), + ("term", "xyz" + b64("a note that names Zorblax|quent outright")), + ("term", b64(b64("twice over: zorblax|quent, still found"))), + ("entity-id", "blob %s end" % b64("the pod entity-|9101-0 restarted twice")), + ("email", "%s" % b64("write to ana.b@gm|ail.com about it")), + ] negatives = [live(s) for s in [ - "zorblax quent", "quilmar", "a vesk quilmar", "orvel prax tam", - "entity_id=9003", "entity_id=30", "entity 1023", "entity-3x", "host-31", "Host 7", - "entity-9310-0.svc", "the v0.3.128 tag", "entity_id = ", + "zorblax quent", "quilmar", "a vesk quilmar", "orvel prax tam", r"zorblax\nquent", + "entity_id=91011", "entity_id=910", "entity 92021", "entity-9101x", "host-91012", "Host 9104", + "entity-9310-0.svc", "the v0.9.9101 tag", "entity_id = ", "real rows in a throwaway container", "the real row replaces the echo", "returns the text verbatim", "ana@example.com", "a•••@gmail.com shows the first letter", "+1 415 555 0199", "+1|5555550100", "ids 2026091812345", "port 8800, 4096 MiB", "MemoryHigh=15032385536 MemoryMax=17179869184", "2026-10-07 12:00", "widths 278 584 556 1015 667 722", + # Base64-shaped text that is not base64 of text: identifiers, paths, + # digests and random bytes are not decoded into findings. + "TestPublishedSourceCarriesNoOldInternalNameAnywhere", + "/usr/local/lib/python3/site-packages/zorbtool/__init__", + "sha256:1ae0735f00daffa3aaf1363a5184c0d2dc55c78e3db4ec70241cdac97bf84b59", + base64.b64encode(bytes(range(200, 256))).decode("ascii"), + b64("a plain note with nothing listed in it at all"), ]] for cls, line in positives: if cls not in classes(line): @@ -408,11 +593,41 @@ def self_test(): # The real table is wired: every entry is a digest of the right shape. if not TERMS or not all(re.fullmatch(r"[0-9a-f]{32}", k) for k in TERMS): fails.append("self-test: TERMS is empty or malformed") + # A line holding a quiet value shows no text for any finding on it: here a + # provenance phrase beside a mailbox, and one base64 run holding both an + # entity id and a term. + blob = b64("the pod entity-|9101-0 belongs to zorblax|quent") + for line, want, secret in ((live("the real| rows that came from ana.b@gm|ail.com"), + {"provenance", "email"}, live("ana.b@gm|ail")), + ("note %s end" % blob, {"entity-id", "term"}, blob[:24])): + got = line_report("x.md", 1, line, sc) + if {c for c, _ in got} != want or any(secret in t for _c, t in got): + fails.append("self-test: a line with a quiet value is reported without its text, got %s" % got) + # A path that holds a quiet value is shown with that component withheld, + # by depth; one split across components withholds them all. + for label, want in (("docs/zorblaxquent-notes.md", "docs/"), + ("app.bundle!notes/" + live("ana.b@gm|ail.com") + ".txt", "app.bundle!notes/"), + ("quilmar/vesk.md", "/"), + ("docs/readme.md", "docs/readme.md")): + if shown_path(label, sc) != want: + fails.append("self-test: path %r shown as %r, want %r" % (label, shown_path(label, sc), want)) + got = line_report("docs/" + live("ana.b@gm|ail.com") + ".md", 1, live("the real| rows that"), sc) + if [t for _c, t in got] != [live("docs/:1 [provenance: real| rows that] the real| rows that")]: + fails.append("self-test: a FAIL line's path is shown withheld, got %s" % got) + # A term is counted once per place it stands, however many ways it is read. + got = [c for c, _ in line_findings(live(r"a\nzorblax|quent b"), sc)] + if got != ["term"]: + fails.append("self-test: a term after an escape is one hit, got %s" % got) # Without a key the term class is off and nothing else changes. - keyless = TermScanner(sc.terms, None) - got = {c for c, _ in line_findings(live("the zorblax|quent ledger, entity_id=|3, x@proto|nmail.com"), keyless)} + planted = live("the zorblax|quent ledger, entity_id=|9101, x@proto|nmail.com") + keyless = TermScanner(sc.terms, None, probe_ids) + got = {c for c, _ in line_findings(planted, keyless)} if got != {"entity-id", "email"}: fails.append("self-test: without a key only the term class is off, got %s" % sorted(got)) + # Without the ids the entity-id class is off and nothing else changes. + got = {c for c, _ in line_findings(planted, TermScanner(sc.terms, probe_key))} + if got != {"term", "email"}: + fails.append("self-test: without the ids only the entity-id class is off, got %s" % sorted(got)) # A key other than the one TERMS was made with is told apart from it. if key_matches(probe_key) or key_matches(b""): fails.append("self-test: a key TERMS was not made with passes the key check") @@ -426,6 +641,18 @@ def self_test(): if len(entries) != 1 or len(errors) != 2: fails.append("self-test: the allow file reader kept %d entries and %d errors, want 1 and 2" % (len(entries), len(errors))) + # The reserved ids are read one per line; a missing file turns the + # class off, and a file that would match nothing is an error. + idf = os.path.join(td, "ids") + for body, want in (("# reserved\n9101\n\n 9202 \n", ((9101, 9202), False)), + ("9101\nentity-9202\n", (None, True)), ("# none\n\n", (None, True)), ("", (None, True))): + with open(idf, "w", encoding="utf-8") as f: + f.write(body) + ids, err = load_reserved_ids(idf) + if (ids, err is not None) != want: + fails.append("self-test: reserved ids %r read as %r, %r" % (body, ids, err)) + if load_reserved_ids(os.path.join(td, "absent")) != (None, None): + fails.append("self-test: a missing reserved-ids file is not (None, None)") # A zip-format bundle is opened by its signature, whatever it is named # (an app bundle, or a .zip such as a browser extension a site serves), # and its text members are read. @@ -469,13 +696,20 @@ def main(argv) -> int: key = load_key() if key is None: print(" NOTE term check skipped: no key at %s (set NO_PERSONAL_DATA_KEY_FILE); " - "the entity-id, provenance, email and phone checks still run" % KEY_FILE) + "the other checks still run" % KEY_FILE) elif not key_matches(key): fails.append("the key at %s is not the one TERMS was made with, so it would match " "nothing: point NO_PERSONAL_DATA_KEY_FILE at that key, or remake TERMS and KEY_CHECK" % KEY_FILE) print(" FAIL %s" % fails[-1]) key = None - scanner = TermScanner(TERMS, key) + ids, ids_error = load_reserved_ids() + if ids_error: + fails.append(ids_error) + print(" FAIL %s" % ids_error) + elif ids is None: + print(" NOTE entity-id check skipped: no reserved ids at %s (set RESERVED_ENTITY_IDS_FILE); " + "the other checks still run" % RESERVED_ENTITY_IDS_FILE) + scanner = TermScanner(TERMS, key, ids) used = set() findings = [] counts = {} @@ -488,21 +722,24 @@ def main(argv) -> int: continue for cls, what in line_findings(rel, scanner): if not allowed(allow, rel, cls, rel, used): - findings.append("%s [%s: %s] (file name)" % (rel, cls, what)) + findings.append("%s [%s: %s] (file name)" % (shown_path(rel, scanner), cls, what)) counts[cls] = counts.get(cls, 0) + 1 for label, text in texts(root, rel): n_texts += 1 if text is None: fails.append("%s has a zip signature but does not open as a zip, so its " - "members cannot be read" % label) + "members cannot be read" % shown_path(label, scanner)) print(" FAIL %s" % fails[-1]) continue + # A bundle member's name installs with it, like a file name. + member = label[len(rel) + 1:] + for cls, what in (line_findings(member, scanner) if member else ()): + if not allowed(allow, label, cls, member, used): + findings.append("%s [%s: %s] (bundle member name)" % (shown_path(label, scanner), cls, what)) + counts[cls] = counts.get(cls, 0) + 1 for i, line in enumerate(text.splitlines(), 1): - for part, cls, what in line_hits(line, scanner): - if (cls == "entity-id" and label == allow_rel) or allowed(allow, label, cls, part, used): - continue - shown = "" if cls in _QUIET else part.strip()[:140] - findings.append(("%s:%d [%s: %s] %s" % (label, i, cls, what, shown)).rstrip()) + for cls, finding in line_report(label, i, line, scanner, allow, used, label == allow_rel): + findings.append(finding) counts[cls] = counts.get(cls, 0) + 1 for f in findings: print(" FAIL %s" % f)