bastion 0.55.3
ci / personal-data (push) Successful in 5s
ci / test (push) Successful in 3m51s
ci / fuzz (push) Successful in 2m55s

This commit is contained in:
Privatae
2026-10-25 23:16:45 +00:00
parent 77b7853ae1
commit 8822d5df85
136 changed files with 8638 additions and 1641 deletions
+1 -1
View File
@@ -1 +1 @@
0.54.1
0.55.3
+4 -4
View File
@@ -23,10 +23,10 @@ set -euo pipefail
HERE="$(cd "$(dirname "$0")/.." && pwd)"
VER="${1:?usage: deploy/build.sh <version>}"
# golang:1.25-alpine as of the digest below (go1.25.14). Move it deliberately,
# in its own commit, and a release built after that commit is rebuilt with the
# new one.
GO_IMAGE="golang@sha256:1ae0735f00daffa3aaf1363a5184c0d2dc55c78e3db4ec70241cdac97bf84b59"
# golang:1.26.10-alpine3.24, the multi-arch index digest below (go1.26.10).
# Move it deliberately, in its own commit, and a release built after that
# commit is rebuilt with the new one.
GO_IMAGE="golang@sha256:27f2d3b83cfc5803c8078f8e730af0f7c431bf067afaec8f60a36fb763187cba"
# The module cache is only a download cache: go.sum checks everything read from
# it. It is kept between runs so a rebuild does not fetch every module again.
+39
View File
@@ -18,8 +18,47 @@
#
# It checks that the test exists, not what it asserts; that part is reviewed
# by reading the test when a row is added or changed.
#
# deploy/check-claims.sh --skipped <go test -json output> [repo root]
#
# reads a `go test -json` run instead and names every cited test that
# skipped in it, with the reason it gave. It exits non-zero if any did: a CI job
# that runs a claim's test where it cannot run proves nothing about the claim.
set -euo pipefail
HERE="$(cd "$(dirname "$0")/.." && pwd)"
if [ "${1:-}" = "--skipped" ]; then
RUN="$(cd "$(dirname "${2:?usage: deploy/check-claims.sh --skipped <go test -json output> [repo root]}")" && pwd)/$(basename "$2")"
ROOT="${3:-$HERE}"
cd "$ROOT"
[ -f docs/claims.md ] || { echo "no docs/claims.md in $ROOT" >&2; exit 2; }
python3 - "$RUN" <<'PY'
import json, re, sys
cited = set(re.findall(r"`[\w./-]+_test\.go`: `(\w+)`", open("docs/claims.md", encoding="utf-8").read()))
skipped, why, ran = {}, {}, set()
for line in open(sys.argv[1], encoding="utf-8", errors="replace"):
try:
e = json.loads(line)
except ValueError:
continue
t = e.get("Test")
if not t:
continue
top = t.split("/")[0]
if top not in cited:
continue
ran.add(top)
if e.get("Action") == "output" and "_test.go:" in e.get("Output", ""):
why.setdefault(t, e["Output"].strip())
if e.get("Action") == "skip":
skipped[t] = True
for t in sorted(skipped):
print("SKIPPED " + t + ": " + why.get(t, "(no reason printed)"))
print("%d cited tests ran here, %d of them skipped" % (len(ran), len(skipped)))
sys.exit(1 if skipped else 0)
PY
exit $?
fi
ROOT="${1:-$HERE}"
cd "$ROOT"
[ -f docs/claims.md ] && [ -d client-go ] || { echo "not a repository root (no docs/claims.md or client-go): $ROOT" >&2; exit 2; }
+105
View File
@@ -0,0 +1,105 @@
#!/usr/bin/env python3
"""Attach a release's checked files to its GitHub release.
GH_TOKEN=<token> python3 deploy/github-release.py <owner/repo> v<version> <assets dir> <CHANGELOG.md>
.github/workflows/release.yml runs this after deploy/release-assets.sh has
filled <assets dir> with files it checked against the signed manifest. The
release for the tag is made if it does not exist yet, named
`bastion <version>`, with that version's section of <CHANGELOG.md> as its
notes; an existing release keeps its name and notes. Each file in
<assets dir> is uploaded, replacing an asset of the same name, so a re-run
leaves the same set. Standard library only; the token is read from the
environment and sent only to the GitHub API.
"""
import json
import os
import re
import sys
import urllib.error
import urllib.parse
import urllib.request
API = os.environ.get("GITHUB_API_URL", "https://api.github.com")
UPLOADS = os.environ.get("GITHUB_UPLOADS_URL", "https://uploads.github.com")
def call(method, url, token, body=None, ctype="application/json"):
data = None
if body is not None:
data = body if isinstance(body, bytes) else json.dumps(body).encode()
req = urllib.request.Request(url, data=data, method=method, headers={
"Authorization": "Bearer " + token, "Accept": "application/vnd.github+json",
"X-GitHub-Api-Version": "2022-11-28", "Content-Type": ctype,
"User-Agent": "bastion-release-assets"})
try:
with urllib.request.urlopen(req, timeout=300) as r:
raw = r.read()
return r.status, (json.loads(raw) if raw else None)
except urllib.error.HTTPError as e:
raw = e.read()
try:
return e.code, json.loads(raw)
except ValueError:
return e.code, {"message": raw.decode(errors="replace")[:300]}
def notes(changelog, ver):
"""The CHANGELOG section `## <ver>`, up to the next `## ` heading."""
text = open(changelog, encoding="utf-8").read()
m = re.search(r"^## " + re.escape(ver) + r"[ \t]*$", text, re.M)
if m is None:
raise SystemExit("CHANGELOG has no section for %s" % ver)
nxt = re.search(r"^## ", text[m.end():], re.M)
body = text[m.end():m.end() + nxt.start() if nxt else len(text)].strip()
return (body + "\n\n---\nThe binaries attached here were rebuilt from this tag by this repository's "
"release workflow, and their digests are the ones in the signed manifest `version.json` "
"attached beside them. To check them yourself, see "
"[Verify a release](https://github.com/Privatae/bastion#verify-a-release).")
def main(argv):
if len(argv) != 5:
print(__doc__.strip().splitlines()[2].strip(), file=sys.stderr)
return 2
repo, tag, assets, changelog = argv[1:]
if not re.fullmatch(r"v[0-9]+\.[0-9]+\.[0-9]+", tag):
raise SystemExit("not a release tag: " + tag)
token = os.environ.get("GH_TOKEN", "")
if not token:
raise SystemExit("GH_TOKEN is not set")
ver = tag[1:]
files = sorted(os.listdir(assets))
if not files:
raise SystemExit("no assets in " + assets)
st, rel = call("GET", "%s/repos/%s/releases/tags/%s" % (API, repo, tag), token)
if st == 404:
st, rel = call("POST", "%s/repos/%s/releases" % (API, repo), token, {
"tag_name": tag, "name": "bastion " + ver, "body": notes(changelog, ver),
"draft": False, "prerelease": False})
if st != 201:
raise SystemExit("could not make the release for %s: HTTP %d %s" % (tag, st, rel))
print("made the release for %s" % tag)
elif st != 200:
raise SystemExit("could not read the release for %s: HTTP %d %s" % (tag, st, rel))
have = {a["name"]: a["id"] for a in rel.get("assets", [])}
for name in files:
if name in have:
st, _ = call("DELETE", "%s/repos/%s/releases/assets/%d" % (API, repo, have[name]), token)
if st != 204:
raise SystemExit("could not replace %s: HTTP %d" % (name, st))
with open(os.path.join(assets, name), "rb") as fh:
data = fh.read()
url = "%s/repos/%s/releases/%d/assets?name=%s" % (UPLOADS, repo, rel["id"], urllib.parse.quote(name))
st, up = call("POST", url, token, data, "application/octet-stream")
if st != 201 or up.get("size") != len(data):
raise SystemExit("upload of %s failed: HTTP %d %s" % (name, st, up))
print("attached %s (%d bytes)" % (name, len(data)))
return 0
if __name__ == "__main__":
sys.exit(main(sys.argv))
+56
View File
@@ -0,0 +1,56 @@
#!/usr/bin/env python3
"""Write a bastion release manifest (version.json) to standard output.
python3 deploy/manifest.py <version> <built_from> <dist dir>
publish.sh writes every release manifest with this, and release-assets.sh
writes it again from a rebuild of the release's tag. The manifest is a function
of the version, the commit it was built from and the bytes of the binaries in
<dist dir>, so a rebuild of the same release writes the same bytes, and the
signature published for the release (named by the manifest's digest) still
verifies against it.
The format: {version, built_from, binaries: {<os>-<arch>[.exe]: {url, sha256}}},
keys in that order, the binaries sorted by file name, indented by two spaces,
no newline at the end.
"""
import hashlib
import json
import os
import sys
def manifest(ver, built_from, dist):
binaries = {}
for f in sorted(os.listdir(dist)):
if not f.startswith("privatae-bastion-"):
continue
with open(os.path.join(dist, f), "rb") as fh:
h = hashlib.sha256(fh.read()).hexdigest()
# ?v=<version> IS LOAD-BEARING, not decoration. dl.privatae.ai sits
# behind a cache that keys on the full URL including the query string.
# Without it a publish replaces the bytes on every edge, every edge
# verifies, the manifest is fresh, and the cache keeps serving the
# PREVIOUS binary to anyone whose request lands on a warm edge: the
# client downloads the old bytes, compares them with the new digest,
# and aborts with "sha256 mismatch", which reads like a corrupt build
# and is a stale cache. It breaks one platform at a time, for whoever
# downloaded last. A new version string is a new cache key and so a
# guaranteed miss; update.go downloads binaries[key].url verbatim, so
# bastions already in the field pick this up with no client change.
binaries[f[len("privatae-bastion-"):]] = {
"url": f"https://dl.privatae.ai/dl/{f}?v={ver}", "sha256": h}
return json.dumps({"version": ver, "built_from": built_from, "binaries": binaries}, indent=2)
def main(argv):
if len(argv) != 4:
print("usage: deploy/manifest.py <version> <built_from> <dist dir>", file=sys.stderr)
return 2
sys.stdout.write(manifest(argv[1], argv[2], argv[3]))
return 0
if __name__ == "__main__":
sys.exit(main(sys.argv))
+5 -31
View File
@@ -85,37 +85,11 @@ fi
# same script and compares every digest.
"$HERE/deploy/build.sh" "$VER"
# version manifest: {version, built_from, binaries:{os-arch:{url,sha256}}} — built with python
# (clean JSON; the host + the bastion self-update read this to decide "out of date").
python3 - "$VER" "$GITSHA" "$HERE/client-go/dist" > "$HERE/client-go/dist/version.json" <<'PY'
import sys, os, hashlib, json
ver, sha, dist = sys.argv[1], sys.argv[2], sys.argv[3]
binaries = {}
for f in sorted(os.listdir(dist)):
if not f.startswith("privatae-bastion-"):
continue
h = hashlib.sha256(open(os.path.join(dist, f), "rb").read()).hexdigest()
# ?v=<version> IS LOAD-BEARING, not decoration. dl.privatae.ai sits behind
# Cloudflare, which caches these objects and keys on the full URL including
# the query string. Without it a publish replaces the bytes on both edges,
# both edges verify, the manifest is fresh — and Cloudflare keeps serving the
# PREVIOUS binary to anyone whose request lands on a warm edge. The client
# downloads the old bytes, compares them against the new digest, and aborts
# with "sha256 mismatch" — which reads like a corrupt build and is actually a
# stale cache. Measured publishing 0.23.0: windows-amd64.exe came
# back cf-cache-status HIT, age 3917, carrying 0.22.0's bytes. It was the
# ONLY stale one, because it was the only object anything had requested since
# the previous publish — so this breaks one platform at a time, at random,
# for whoever happens to have downloaded last. The worst possible shape.
#
# A new version string is a new cache key is a guaranteed miss. Needs no
# Cloudflare credential and cannot be forgotten at publish time. update.go
# downloads binaries[key].url verbatim, so bastions already in the field pick
# this up from the manifest with no client change.
binaries[f[len("privatae-bastion-"):]] = {
"url": f"https://dl.privatae.ai/dl/{f}?v={ver}", "sha256": h}
json.dump({"version": ver, "built_from": sha, "binaries": binaries}, sys.stdout, indent=2)
PY
# The version manifest, written by deploy/manifest.py: release-assets.sh writes
# it again from a rebuild of the release's tag, and the two must give the same
# bytes for the signature to verify there. The host and every bastion's
# self-update read it to decide "out of date".
python3 "$HERE/deploy/manifest.py" "$VER" "$GITSHA" "$HERE/client-go/dist" > "$HERE/client-go/dist/version.json"
MAN="$HERE/client-go/dist/version.json"
MAN_SHA="$(python3 -c 'import hashlib,sys;print(hashlib.sha256(open(sys.argv[1],"rb").read()).hexdigest())' "$MAN")"
+82
View File
@@ -0,0 +1,82 @@
#!/usr/bin/env bash
# Rebuild a published release from its tag and collect the files its GitHub
# release carries, refusing unless they are exactly what was signed.
#
# deploy/release-assets.sh v<version> <out dir>
#
# Needs git, docker, curl and python3, and a clone of this repository that has
# the tag. It needs no key or account of ours. .github/workflows/release.yml
# runs it for every release tag; anyone can run it to check a release.
#
# The tag must be annotated, with the line `built from <commit>` that names
# the commit the release was built from in the development repository. The
# tag's own deploy/build.sh rebuilds the binaries (a pinned toolchain image,
# so the bytes are reproducible), deploy/manifest.py writes the release
# manifest from them, and the signature published beside the manifest, which
# is named by the manifest's digest and never removed, must verify against
# the release keys pinned in this checkout and at the tag. A rebuild that
# differs by one byte gives another digest, no signature by that name, and
# nothing in <out dir>. When the tag is the release the public manifest names
# now, the rebuilt manifest must also equal that one byte for byte.
#
# <out dir> gets the six binaries, version.json (the signed manifest),
# version.json.<first 16 hex of its sha256>.sig (its signature, the file every
# bastion's self-update checks) and SHA256SUMS (the binaries and the manifest,
# in the format `sha256sum -c` reads; it is not signed itself).
set -euo pipefail
HERE="$(cd "$(dirname "$0")/.." && pwd)"
TAG="${1:?usage: deploy/release-assets.sh v<version> <out dir>}"
OUT="${2:?usage: deploy/release-assets.sh v<version> <out dir>}"
BASE="${MANIFEST_BASE:-https://privatae.ai/bastion}"
[[ "$TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] || { echo "not a release tag: $TAG" >&2; exit 2; }
VER="${TAG#v}"
[ ! -e "$OUT" ] || [ -z "$(ls -A "$OUT")" ] || { echo "$OUT is not empty" >&2; exit 2; }
git -C "$HERE" rev-parse -q --verify "refs/tags/$TAG" >/dev/null || { echo "no tag $TAG in this clone" >&2; exit 1; }
[ "$(git -C "$HERE" cat-file -t "refs/tags/$TAG")" = tag ] || { echo "$TAG is not an annotated tag" >&2; exit 1; }
FROM="$(git -C "$HERE" tag -l --format='%(contents)' "$TAG" | sed -n 's/^built from \([0-9a-f]\{7,40\}\)$/\1/p')"
[ "$(printf '%s\n' "$FROM" | grep -c .)" = 1 ] || { echo "$TAG does not carry exactly one 'built from <commit>' line" >&2; exit 1; }
SRC="$(git -C "$HERE" rev-parse "refs/tags/$TAG^{commit}")"
[ "$(git -C "$HERE" show "$SRC:deploy/VERSION" | tr -d '[:space:]')" = "$VER" ] \
|| { echo "deploy/VERSION at $TAG is not $VER" >&2; exit 1; }
echo "release: $TAG at $SRC, built from $FROM"
WORK="$(mktemp -d)"
trap 'git -C "$HERE" worktree remove --force "$WORK/src" >/dev/null 2>&1 || true; rm -rf "$WORK"' EXIT
git -C "$HERE" worktree add -q --detach "$WORK/src" "$SRC"
"$WORK/src/deploy/build.sh" "$VER"
DIST="$WORK/src/client-go/dist"
python3 "$HERE/deploy/manifest.py" "$VER" "$FROM" "$DIST" > "$WORK/version.json"
H="$(python3 -c 'import hashlib,sys;print(hashlib.sha256(open(sys.argv[1],"rb").read()).hexdigest()[:16])' "$WORK/version.json")"
SIG="version.json.$H.sig"
curl -fsS --max-time 30 "$BASE/$SIG" -o "$WORK/$SIG" \
|| { echo "no signature is published for the rebuilt manifest ($SIG): the rebuild is not the signed release" >&2; exit 1; }
SIGNED_BY="$("$HERE/deploy/release-verify.sh" bastion "$WORK/version.json" "$WORK/$SIG" "$SRC")" \
|| { echo "the signature $SIG does not verify against the rebuilt manifest" >&2; exit 1; }
echo "signature: ok ($SIGNED_BY)"
if curl -fsS --max-time 30 "$BASE/version.json" -o "$WORK/live.json"; then
LIVE="$(python3 -c 'import json,sys;print(json.load(open(sys.argv[1]))["version"])' "$WORK/live.json")"
if [ "$LIVE" = "$VER" ]; then
cmp -s "$WORK/live.json" "$WORK/version.json" \
|| { echo "the public manifest names $VER but differs from the rebuilt one" >&2; exit 1; }
echo "the public manifest names $VER and is byte for byte the rebuilt one"
else
echo "the public manifest names $LIVE; $VER is checked by its signature alone"
fi
fi
mkdir -p "$OUT"
cp "$DIST"/privatae-bastion-* "$WORK/version.json" "$WORK/$SIG" "$OUT/"
(cd "$OUT" && python3 - <<'PY'
import hashlib, os
names = sorted(f for f in os.listdir(".") if f.startswith("privatae-bastion-")) + ["version.json"]
with open("SHA256SUMS", "w") as fh:
for n in names:
fh.write("%s %s\n" % (hashlib.sha256(open(n, "rb").read()).hexdigest(), n))
PY
)
n="$(ls "$OUT" | grep -c '^privatae-bastion-')"
[ "$n" = 6 ] || { echo "expected 6 binaries, built $n" >&2; exit 1; }
echo "assets for $TAG in $OUT:"
ls -l "$OUT"
+35
View File
@@ -0,0 +1,35 @@
#!/bin/sh
# staticcheck on both Go modules, for every OS the release is built for.
#
# deploy/staticcheck.sh
#
# Each module is checked once per GOOS and the runs are merged, which is how
# staticcheck reports code unused on every OS rather than unused on one: a
# function only the Windows build calls is used. Any finding fails. The
# checker is the pinned release below, built with this toolchain; set
# STATICCHECK to use one already installed.
set -eu
HERE="$(cd "$(dirname "$0")/.." && pwd)"
VERSION=v0.8.2 # staticcheck 2026.2.2
SC="${STATICCHECK:-}"
if [ -z "$SC" ]; then
BIN="$(mktemp -d)"
GOBIN="$BIN" GOFLAGS= go install "honnef.co/go/tools/cmd/staticcheck@$VERSION"
SC="$BIN/staticcheck"
fi
OUT="$(mktemp -d)"
bad=0
for mod in client-go deploy/release-sign; do
for os in linux darwin windows; do
(cd "$HERE/$mod" && GOOS=$os "$SC" -f binary ./... > "$OUT/$os.bin")
done
found="$(cd "$HERE/$mod" && "$SC" -merge "$OUT/linux.bin" "$OUT/darwin.bin" "$OUT/windows.bin" || true)"
if [ -n "$found" ]; then
printf '%s\n' "$found" | sed "s#^#$mod/#"
bad=1
else
echo "ok $mod: no findings on linux, darwin and windows"
fi
done
rm -rf "$OUT"
exit "$bad"
+118
View File
@@ -0,0 +1,118 @@
#!/usr/bin/env python3
"""deploy/github-release.py against a fake GitHub API on loopback.
python3 -I deploy/test_github_release.py
A release that does not exist is made with the version's CHANGELOG section
and every file attached; a re-run replaces each asset of the same name and
leaves an existing release's name and notes alone; the token goes only in
the Authorization header.
"""
import importlib.util
import json
import os
import sys
import tempfile
import threading
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
sys.dont_write_bytecode = True # leave no __pycache__ beside the script under test
HERE = os.path.dirname(os.path.abspath(__file__))
spec = importlib.util.spec_from_file_location("ghrel", os.path.join(HERE, "github-release.py"))
class Fake:
def __init__(self):
self.release = None
self.calls = []
self.next_id = 100
def handler(fake):
class H(BaseHTTPRequestHandler):
def log_message(self, *a):
pass
def reply(self, code, obj=None):
raw = json.dumps(obj).encode() if obj is not None else b""
self.send_response(code)
self.send_header("Content-Length", str(len(raw)))
self.end_headers()
self.wfile.write(raw)
def body(self):
return self.rfile.read(int(self.headers.get("Content-Length", 0)))
def do_GET(self):
fake.calls.append(("GET", self.path, self.headers.get("Authorization")))
if self.path == "/repos/o/r/releases/tags/v1.2.3":
return self.reply(200, fake.release) if fake.release else self.reply(404, {"message": "Not Found"})
self.reply(404, {})
def do_POST(self):
data = self.body()
fake.calls.append(("POST", self.path, self.headers.get("Authorization")))
if self.path == "/repos/o/r/releases":
req = json.loads(data)
fake.release = {"id": 7, "tag_name": req["tag_name"], "name": req["name"], "body": req["body"], "assets": []}
return self.reply(201, fake.release)
if self.path.startswith("/repos/o/r/releases/7/assets?name="):
name = self.path.split("name=", 1)[1]
fake.next_id += 1
fake.release["assets"].append({"id": fake.next_id, "name": name, "size": len(data)})
return self.reply(201, {"id": fake.next_id, "name": name, "size": len(data)})
self.reply(404, {})
def do_DELETE(self):
fake.calls.append(("DELETE", self.path, self.headers.get("Authorization")))
aid = int(self.path.rsplit("/", 1)[1])
fake.release["assets"] = [a for a in fake.release["assets"] if a["id"] != aid]
self.reply(204)
return H
def main():
fake = Fake()
srv = ThreadingHTTPServer(("127.0.0.1", 0), handler(fake))
threading.Thread(target=srv.serve_forever, daemon=True).start()
base = "http://127.0.0.1:%d" % srv.server_address[1]
os.environ.update({"GITHUB_API_URL": base, "GITHUB_UPLOADS_URL": base, "GH_TOKEN": "t0ken"})
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
with tempfile.TemporaryDirectory() as d:
assets = os.path.join(d, "assets")
os.mkdir(assets)
for name, data in (("privatae-bastion-linux-amd64", b"bin"), ("version.json", b"{}")):
open(os.path.join(assets, name), "wb").write(data)
cl = os.path.join(d, "CHANGELOG.md")
open(cl, "w").write("# Changelog\n\n## 1.2.4\n\n- later\n\n## 1.2.3\n\n- **One change.**\n\n## 1.2.2\n\n- earlier\n")
assert mod.main(["x", "o/r", "v1.2.3", assets, cl]) == 0
rel = fake.release
assert rel["name"] == "bastion 1.2.3", rel
assert rel["body"].startswith("- **One change.**") and "later" not in rel["body"] and "earlier" not in rel["body"], rel["body"]
assert sorted(a["name"] for a in rel["assets"]) == ["privatae-bastion-linux-amd64", "version.json"]
rel["name"], rel["body"] = "kept", "kept notes"
open(os.path.join(assets, "version.json"), "wb").write(b'{"v":1}')
assert mod.main(["x", "o/r", "v1.2.3", assets, cl]) == 0
assert (rel["name"], rel["body"]) == ("kept", "kept notes")
sizes = {a["name"]: a["size"] for a in rel["assets"]}
assert sizes == {"privatae-bastion-linux-amd64": 3, "version.json": 7}, sizes
assert sum(1 for c in fake.calls if c[0] == "DELETE") == 2
assert all(c[2] == "Bearer t0ken" for c in fake.calls)
assert not any("t0ken" in c[1] for c in fake.calls)
try:
mod.main(["x", "o/r", "v9.9.9", assets, cl])
except SystemExit as e:
assert "no section for 9.9.9" in str(e), e
else:
raise AssertionError("a version with no CHANGELOG section must not get a release")
srv.shutdown()
print("ok - github-release.py makes, fills and refills a release")
if __name__ == "__main__":
main()