bastion agent 0.1.16

bastion_agent/VERSION

@@ -1 +1 @@
-0.1.15
+0.1.16

bastion_agent/__init__.py

@@ -10,3 +10,9 @@ from .manifest import (  # noqa: F401
     signing_payload,
     validate_document,
 )
+
+# The package's public names: the manifest's schema and its loader.
+__all__ = [
+    "STAGES", "MASK_OPS", "SWEEPS", "GRANT_OPS", "Manifest", "ManifestError",
+    "load_manifest", "signing_digest", "signing_payload", "validate_document",
+]

bastion_agent/catalog.py

@@ -340,7 +340,6 @@ def catalog(manifest, out_dir, env=None, summarizer=None, now=None):
             continue
         key = (w["root"], w["path"])
         counts["files"] += 1
-        name = w["path"].rsplit("/", 1)[-1]
         shown_path, changed = mask_path(w["path"], detect)
         masked = shown_path.rsplit("/", 1)[-1]
         counts["names_masked"] += changed

bastion_agent/doctor.py

@@ -7,11 +7,13 @@ tesseract, the gliner package) — their absence is a recorded gap in the
 stage output, not a failure here.
 """
 
+import importlib
 import json
 import os
 import platform
 import shutil
 import sys
+import urllib.error
 import urllib.request
 
 PY_FLOOR = (3, 8)
@@ -25,7 +27,6 @@ def _probe_http(url, path, timeout=5):
     """Reachable means the server answered — any HTTP status counts (a
     proxy that only permits the embed path answers 403 on '/'). Only a
     connection failure is 'not reachable'."""
-    import urllib.error
     try:
         with urllib.request.urlopen(url.rstrip("/") + path, timeout=timeout) as r:
             return True, "HTTP %d" % r.status
@@ -65,7 +66,7 @@ def checks(env=None):
         add(tool, bool(p), p or ("not on PATH — " + why), required=req, stage=stage)
 
     try:
-        import gliner  # noqa: F401
+        importlib.import_module("gliner")
         add("gliner package", True, "importable (PII_RECALL=gliner available)", required=False, stage="sanitize")
     except ImportError:
         add("gliner package", False, "not installed — sanitize sterilize on a laptop needs "

bastion_agent/extract.py

@@ -41,7 +41,6 @@ import os
 import re
 import shutil
 import subprocess
-import sys
 import tempfile
 import time
 import unicodedata

bastion_agent/publish.sh

@@ -192,7 +192,7 @@ installers = {n: {"url": "%s/%s" % (base_new, n), "sha256": h(os.path.join(pkg,
               for n in ("install.sh", "install.ps1")}
 json.dump({"version": ver, "built_from": sha, "python_floor": "3.8",
            "files": files, "installers": installers,
-           "notes": "https://git.privatae.ai/privatae/bastion-agent/releases/tag/v%s" % ver},
+           "notes": "https://github.com/Privatae/bastion-agent/releases/tag/v%s" % ver},
           sys.stdout, indent=2)
 PY
 

bastion_agent/sanitize.py

@@ -52,7 +52,7 @@ from .manifest import signing_digest
 
 # Part of the reuse key: bump it whenever the pipeline would sterilize the
 # same text differently, or a stored result from older code is kept as is.
-SANITIZE_VERSION = 2
+SANITIZE_VERSION = 3
 SANITIZE_CHUNK_CHARS = 20000     # per sterilize call; under the service's 2 MiB body cap
 SERVICE_TIMEOUT = 600.0
 

bastion_agent/serve.py

@@ -17,6 +17,12 @@ Ops (JSON over HTTPS):
   POST /read_range  {doc, start, end}     read_range (owner, grantee w/ op)
                                           from the SANITIZED store only,
                                           <= READ_RANGE_MAX bytes
+  Both answer with `sanitize_mode`, the mode the sanitize stage recorded for
+  the text in the reply ("flag" or "sterilize"; the weakest across a
+  search's hits; "unknown" when the record does not vouch for that text; a
+  search with no hits names the manifest's mode, or "none"), so a caller can
+  say what the text has had done to it: in flag mode names, street addresses
+  and dates of birth remain.
   GET  /catalog[?prefix=&offset=&limit=]  owner only: the file cards (paged)
   GET  /catalog/dirs[?...]                owner only: the directory cards (paged)
   GET  /status                            owner only: run.state.json
@@ -65,6 +71,83 @@ class ServeError(Exception):
     pass
 
 
+def sanitize_mode(manifest):
+    """The sanitize mode the manifest sets: "flag", "sterilize", or "none" when
+    it names no sanitize stage. A reply with no text in it (a search with no
+    hits) carries this; a reply with text carries served_mode."""
+    s = getattr(manifest, "sanitize", None)
+    return getattr(s, "mode", None) or "none"
+
+
+# Weakest first: a reply that mixes texts names the weakest mode among them.
+_MODE_RANK = {"unknown": 0, "flag": 1, "sterilize": 2}
+_index_lock = threading.Lock()
+_index_cache = {}  # sanitize.jsonl path -> ((ino, mtime_ns, size), {doc: mode})
+
+
+def _sanitize_index(out_dir):
+    """({doc sha: mode} for every document the last sanitize run wrote with
+    status ok, the index's mtime_ns), from <out>/sanitize.jsonl; ({}, None)
+    when there is none. Re-read only when the file changes."""
+    path = os.path.join(out_dir, "sanitize.jsonl")
+    try:
+        st = os.stat(path)
+    except OSError:
+        return {}, None
+    sig = (st.st_ino, st.st_mtime_ns, st.st_size)
+    with _index_lock:
+        hit = _index_cache.get(path)
+        if hit and hit[0] == sig:
+            return hit[1], st.st_mtime_ns
+        modes = {}
+        try:
+            with open(path, encoding="utf-8") as fh:
+                for line in fh:
+                    try:
+                        rec = json.loads(line)
+                    except ValueError:
+                        continue
+                    if isinstance(rec, dict) and rec.get("status") == "ok" and rec.get("sha256"):
+                        modes[rec["sha256"]] = rec.get("mode")
+        except OSError:
+            return {}, None
+        _index_cache[path] = (sig, modes)
+        return modes, st.st_mtime_ns
+
+
+def served_mode(out_dir, doc, start=None, text=None):
+    """The sanitize mode that wrote the text served for doc, as the sanitize
+    stage recorded it, or "unknown" when that record does not vouch for the
+    text. The manifest's mode says what the NEXT sweep will do; the store can
+    still hold an earlier sweep's text (the manifest switched to sterilize and
+    the sweep has not run, or stopped; a sweep is rewriting the store now; the
+    chunks a search reads were cut before the latest sanitize run).
+
+    Unknown when: no status-ok record for doc, the file is newer than the
+    index (a run that has not finished rewrote it), or, for a search hit,
+    `text` at `start` is not what the store holds now."""
+    modes, index_mtime = _sanitize_index(out_dir)
+    mode = modes.get(doc)
+    if mode not in ("flag", "sterilize"):
+        return "unknown"
+    path = sanitized_store_path(out_dir, doc)
+    try:
+        if os.stat(path).st_mtime_ns > index_mtime:
+            return "unknown"
+        if text is not None:
+            with open(path, encoding="utf-8") as fh:
+                stored = fh.read()
+            if stored[start:start + len(text)] != text:
+                return "unknown"
+    except (OSError, TypeError, ValueError):
+        return "unknown"
+    return mode
+
+
+def _weakest(modes):
+    return min(modes, key=lambda m: _MODE_RANK.get(m, 0))
+
+
 def parse_bind(bind):
     bind = bind.strip()
     if bind.startswith("["):
@@ -312,7 +395,13 @@ class Handler(BaseHTTPRequestHandler):
             except (_embed.EmbedError, OSError) as e:
                 self._send(502, {"error": "search unavailable: %s" % e}, cn, "search", {"q": qhash})
                 return
-            self._send(200, {"hits": hits, "k": k}, cn, "search", {"q": qhash, "hits": len(hits)})
+            if hits:
+                mode = _weakest([served_mode(self.out_dir, h.get("doc"), h.get("start"), h.get("text"))
+                                 for h in hits])
+            else:
+                mode = sanitize_mode(self.policy.manifest)
+            self._send(200, {"hits": hits, "k": k, "sanitize_mode": mode},
+                       cn, "search", {"q": qhash, "hits": len(hits)})
             return
         if self.path == "/read_range":
             cn, denied = self._authorize("read_range")
@@ -340,7 +429,8 @@ class Handler(BaseHTTPRequestHandler):
                 text = fh.read()
             piece = text[start:end]
             self._send(200, {"doc": doc, "start": start, "end": start + len(piece),
-                             "total": len(text), "text": piece},
+                             "total": len(text), "text": piece,
+                             "sanitize_mode": served_mode(self.out_dir, doc)},
                        cn, "read_range", {"doc": doc[:12], "chars": len(piece)})
             return
         self._send(404, {"error": "not found"}, self._cn(), None)

bastion_agent/test_run.py

@@ -12,7 +12,6 @@ import shutil
 import subprocess
 import tempfile
 import unittest
-from unittest import mock
 
 import yaml
 

bastion_agent/test_sanitize.py

@@ -153,6 +153,19 @@ class FlagModeTests(Case):
         self.assertEqual(s2["counts"]["ok"], s1["counts"]["ok"])
         self.assertFalse(idx["note.txt"].get("reused", False))
 
+    def test_text_kept_by_the_older_rules_is_redone(self):
+        # version 2 kept an international number and a lowercase IBAN in the
+        # clear; a store it wrote must not be reused by the wider rules
+        put(self.root, "intl.txt", "Mobile +44 7700 900123, iban gb82west12345698765432\n")
+        with mock.patch.object(sz, "SANITIZE_VERSION", 2):
+            self.run_stage(manifest(self.root, "flag"))
+        s, idx = self.run_stage(manifest(self.root, "flag"))
+        self.assertEqual(s["counts"]["reused"], 0)
+        out = self.text_of(idx["intl.txt"])
+        self.assertNotIn("7700 900123", out)
+        self.assertNotIn("gb82west", out)
+        self.assertEqual(idx["intl.txt"]["types"], {"PHONE": 1, "BANK_ACCOUNT": 1})
+
     def test_tokens_stable_across_files_in_scope(self):
         put(self.root, "again.txt", "ping m@example.com again\n")
         _, idx = self.run_stage(manifest(self.root, "flag"))

bastion_agent/test_seal.py

@@ -175,7 +175,7 @@ class RoundTripTests(Case):
     def test_tampered_member_refused(self):
         ident, pub = keygen(self.out)
         env = {"FREEAGENT_OWNER_RECIPIENT": pub}
-        s = sl.seal(manifest(self.root), self.out, env=env)
+        sl.seal(manifest(self.root), self.out, env=env)
         # rebuild a bundle with a modified catalog but the old inventory, seal it, and unseal
         plain, _ = sl.build_bundle(manifest(self.root), self.out, "x")
         buf = io.BytesIO()

bastion_agent/test_serve.py

@@ -22,7 +22,6 @@ from . import run as rn
 from . import serve as sv
 from .catalog import catalog
 from .extract import extract
-from .manifest import validate_document
 from .sanitize import sanitize
 from .test_embed import _Stub as EmbedStub  # deterministic bag-of-words embedder
 from .walk import walk
@@ -112,12 +111,12 @@ class ServeTests(unittest.TestCase):
         shutil.rmtree(cls.work, ignore_errors=True)
 
     @classmethod
-    def write_manifest(cls, grants):
+    def write_manifest(cls, grants, mode="flag"):
         import yaml
         doc = {"freeagent": 1, "id": "t", "owner": {"host": "owner-host"},
                "scope": {"roots": [cls.root]}, "mask": ["read", "search", "index"],
                "stages": ["walk", "extract", "sanitize", "chunk", "embed", "catalog", "seal"],
-               "sanitize": {"mode": "flag"},
+               "sanitize": {"mode": mode},
                "embed": {"model": "stub-model", "dim": 64, "chunk": {"size_tokens": 8, "overlap_tokens": 2}},
                "sweep": "manual",
                "sink": {"catalog": "host", "vectors": "local", "plaintext": "never"},
@@ -238,6 +237,93 @@ class ServeTests(unittest.TestCase):
         self.assertEqual(self.call("owner", "POST", "/search", {"query": ""})[0], 400)
         self.assertEqual(self.call("owner", "POST", "/nope", {})[0], 404)
 
+    GRANTS = [{"host": "partner-host", "ops": ["search"], "until": "2026-12-31"},
+              {"host": "old-host", "ops": ["search", "read_range"], "until": "2026-01-01"}]
+
+    def modes(self):
+        """(owner search, owner read_range, partner search) sanitize_mode for the apples doc."""
+        st, body = self.call("owner", "POST", "/search", {"query": "apples", "k": 1})
+        self.assertEqual(st, 200)
+        doc = body["hits"][0]["doc"]
+        st, rr = self.call("owner", "POST", "/read_range", {"doc": doc, "start": 0, "end": 10})
+        self.assertEqual(st, 200)
+        st, pb = self.call("partner", "POST", "/search", {"query": "apples", "k": 1})
+        self.assertEqual(st, 200)
+        return doc, (body["sanitize_mode"], rr["sanitize_mode"], pb["sanitize_mode"])
+
+    def rewrite_index(self, fn):
+        """Rewrite sanitize.jsonl through fn(record); returns the original bytes."""
+        path = os.path.join(self.out, "sanitize.jsonl")
+        with open(path, "rb") as fh:
+            orig = fh.read()
+        recs = [fn(json.loads(l)) for l in orig.decode("utf-8").splitlines() if l.strip()]
+        with open(path, "w", encoding="utf-8") as fh:
+            fh.write("".join(json.dumps(r, sort_keys=True) + "\n" for r in recs))
+        return orig
+
+    def restore_index(self, orig):
+        with open(os.path.join(self.out, "sanitize.jsonl"), "wb") as fh:
+            fh.write(orig)
+
+    def test_a_reply_names_the_mode_that_wrote_its_text_not_the_manifests(self):
+        # The store was swept in flag mode. Switching the manifest to
+        # sterilize changes what the NEXT sweep does, not the text already
+        # served: until a sweep rewrites it, every reply still says flag, so
+        # the bastion never calls flag-mode text sanitized.
+        try:
+            for mode in ("flag", "sterilize", "flag"):
+                self.write_manifest(grants=self.GRANTS, mode=mode)
+                self.assertEqual(self.modes()[1], ("flag", "flag", "flag"), mode)
+        finally:
+            self.write_manifest(grants=self.GRANTS)
+
+    def test_a_store_the_sanitize_stage_wrote_in_sterilize_mode_is_named_sterilize(self):
+        orig = self.rewrite_index(lambda r: dict(r, mode="sterilize"))
+        try:
+            self.write_manifest(grants=self.GRANTS, mode="sterilize")
+            self.assertEqual(self.modes()[1], ("sterilize", "sterilize", "sterilize"))
+        finally:
+            self.restore_index(orig)
+            self.write_manifest(grants=self.GRANTS)
+
+    def test_text_the_sanitize_record_does_not_vouch_for_is_unknown(self):
+        doc, modes = self.modes()
+        self.assertEqual(modes, ("flag", "flag", "flag"))
+        spath = sv.sanitized_store_path(self.out, doc)
+        st = os.stat(spath)
+        index = os.path.join(self.out, "sanitize.jsonl")
+        # a sweep still running has rewritten the file after the index
+        later = os.stat(index).st_mtime_ns + 10 ** 9
+        os.utime(spath, ns=(later, later))
+        try:
+            self.assertEqual(self.modes()[1], ("unknown", "unknown", "unknown"))
+        finally:
+            os.utime(spath, ns=(st.st_atime_ns, st.st_mtime_ns))
+        # the chunks a search reads were cut from text the store no longer holds
+        with open(spath, encoding="utf-8") as fh:
+            text = fh.read()
+        with open(spath, "w", encoding="utf-8") as fh:
+            fh.write("X" * len(text))
+        os.utime(spath, ns=(st.st_atime_ns, st.st_mtime_ns))
+        try:
+            got = self.modes()[1]
+            self.assertEqual((got[0], got[2]), ("unknown", "unknown"))
+        finally:
+            with open(spath, "w", encoding="utf-8") as fh:
+                fh.write(text)
+            os.utime(spath, ns=(st.st_atime_ns, st.st_mtime_ns))
+        # the last run did not write it (stopped: not_run), so the file is an older run's
+        orig = self.rewrite_index(lambda r: dict(r, status="not_run") if r.get("sha256") == doc else r)
+        try:
+            self.assertEqual(self.modes()[1], ("unknown", "unknown", "unknown"))
+        finally:
+            self.restore_index(orig)
+        self.assertEqual(self.modes()[1], ("flag", "flag", "flag"))
+
+    def test_sanitize_mode_of_a_manifest_without_the_stage_is_none(self):
+        self.assertEqual(sv.sanitize_mode(None), "none")
+        self.assertEqual(sv.sanitize_mode(type("M", (), {"sanitize": None})()), "none")
+
     def test_silent_tcp_connection_does_not_stall_other_clients(self):
         # with the handshake on the accept loop one idle raw TCP
         # connection blocked every client; now the handshake runs in the
@@ -333,7 +419,7 @@ class UnitTests(unittest.TestCase):
     def test_parse_bind(self):
         self.assertEqual(sv.parse_bind("[::]:8470"), ("::", 8470))
         self.assertEqual(sv.parse_bind("10.0.0.5:9"), ("10.0.0.5", 9))
-        self.assertEqual(sv.parse_bind("[fd00:1::31]:8470"), ("fd00:1::31", 8470))
+        self.assertEqual(sv.parse_bind("[2001:db8::31]:8470"), ("2001:db8::31", 8470))
         with self.assertRaises(sv.ServeError):
             sv.parse_bind("nope")
 

bastion_agent/test_verify_release.py

@@ -169,16 +169,26 @@ class EndToEndTests(unittest.TestCase):
                            "files": {name: {"sha256": digest or self.digest},
                                      name + ".sha256": {"sha256": "0" * 64}}}).encode()
 
-    def verify(self, body, seed=None):
+    def verify(self, body, seed=None, args=()):
         msg = vr.statement("bastion-agent", self.version, hashlib.sha256(body).hexdigest())
-        sig = base64.b64encode(_sign(seed or self.SEED, msg))
+        sig = base64.b64encode(_sign(seed or self.SEED, msg)) + b"\n"
         served = {"/version.json": body, "/" + vr.sig_name(body): sig}
+        self.served = served
         out, err = io.StringIO(), io.StringIO()
         with mock.patch.object(vr, "fetch", lambda url: served[url[len("https://test"):]]), \
                 contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
-            code = vr.main(["--base", "https://test"], repo=self.repo, keys=(self.pub,))
+            code = vr.main(["--base", "https://test"] + list(args), repo=self.repo, keys=(self.pub,))
         return code, out.getvalue() + err.getvalue()
 
+    def release_tag(self, built_from, annotated=True):
+        tag = "v" + self.version
+        if annotated:
+            self.run_git("tag", "-a", tag, "-m", "bastion agent %s\n\nbuilt from %s" % (self.version, built_from))
+        else:
+            self.run_git("tag", tag)
+        self.addCleanup(subprocess.run, ("git", "-C", self.repo, "tag", "-d", tag),
+                        stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+
     def test_tag_first_then_built_from(self):
         code, out = self.verify(self.manifest("0000000"))
         self.assertEqual(code, 1, out)
@@ -189,13 +199,57 @@ class EndToEndTests(unittest.TestCase):
         self.assertIn("source:    built_from", out)
         self.assertIn("matches %s" % self.digest, out)
 
-        self.run_git("tag", "v" + self.version)
-        try:
-            code, out = self.verify(self.manifest("0000000"))
-            self.assertEqual(code, 0, out)
-            self.assertIn("source:    tag v%s" % self.version, out)
-        finally:
-            self.run_git("tag", "-d", "v" + self.version)
+        self.release_tag("0000000")
+        code, out = self.verify(self.manifest("0000000"))
+        self.assertEqual(code, 0, out)
+        self.assertIn("source:    tag v%s" % self.version, out)
+
+    def test_a_tag_without_the_release_record_is_refused(self):
+        # A lightweight tag, or one recording another build, says nothing
+        # about which release it is: refused, not passed over for built_from.
+        self.release_tag("0000000", annotated=False)
+        code, out = self.verify(self.manifest(self.commit[:7]))
+        self.assertEqual(code, 1, out)
+        self.assertIn("does not carry the release record", out)
+        self.assertNotIn("rebuild:", out)
+        self.run_git("tag", "-d", "v" + self.version)
+        self.release_tag("1111111")
+        code, out = self.verify(self.manifest(self.commit[:7]))
+        self.assertEqual(code, 1, out)
+        self.assertIn("built from %s" % self.commit[:7], out)
+
+    def test_another_version_than_expected_is_refused_before_any_checkout(self):
+        code, out = self.verify(self.manifest(self.commit[:7]), args=["--expect-version", "v9.9.9"])
+        self.assertEqual(code, 1, out)
+        self.assertIn("names v%s, not v9.9.9" % self.version, out)
+        self.assertNotIn("source:", out)
+        code, out = self.verify(self.manifest(self.commit[:7]), args=["--expect-version", "v" + self.version])
+        self.assertEqual(code, 0, out)
+
+    def test_out_writes_what_a_release_carries(self):
+        name = "bastion-agent-%s.pyz" % self.version
+        sums = ("%s  %s\n" % (self.digest, name)).encode()
+        out_dir = os.path.join(self.tmp, "release-out")
+        self.addCleanup(shutil.rmtree, out_dir, True)
+        body = json.dumps({"version": self.version, "built_from": self.commit[:7],
+                           "files": {name: {"sha256": self.digest},
+                                     name + ".sha256": {"sha256": hashlib.sha256(sums).hexdigest()}}}).encode()
+        code, out = self.verify(body, args=["--out", out_dir])
+        self.assertEqual(code, 0, out)
+        sig = vr.sig_name(body)
+        self.assertEqual(sorted(os.listdir(out_dir)), sorted([name, name + ".sha256", "version.json", sig]))
+        with open(os.path.join(out_dir, "version.json"), "rb") as fh:
+            self.assertEqual(fh.read(), body)
+        with open(os.path.join(out_dir, sig), "rb") as fh:
+            self.assertEqual(fh.read(), self.served["/" + sig])
+        with open(os.path.join(out_dir, name), "rb") as fh:
+            self.assertEqual(hashlib.sha256(fh.read()).hexdigest(), self.digest)
+
+    def test_out_writes_nothing_when_the_rebuild_differs(self):
+        out_dir = os.path.join(self.tmp, "release-out-bad")
+        code, out = self.verify(self.manifest(self.commit[:7], digest="1" * 64), args=["--out", out_dir])
+        self.assertEqual(code, 1, out)
+        self.assertFalse(os.path.exists(out_dir))
 
     def test_a_different_build_is_reported(self):
         code, out = self.verify(self.manifest(self.commit, digest="1" * 64))

bastion_agent/test_walk.py

@@ -10,13 +10,12 @@ import hashlib
 import json
 import os
 import shutil
-import sys
 import tempfile
 import unittest
 from unittest import mock
 
 from .manifest import validate_document
-from .walk import Excluder, Walker, _glob_to_regex, load_hash_cache, walk
+from .walk import Excluder, _glob_to_regex, load_hash_cache, walk
 
 IS_WINDOWS = os.name == "nt"
 

bastion_agent/verify_release.py

@@ -15,9 +15,15 @@ and the signature says the release key holder published it.
 
 The source is the tag v<version>. The public repository is a snapshot of the
 development one, so the commit the manifest names (built_from) exists only
-there; the tag marks the same tree in the public history. When the tag is
-absent and built_from is present (a clone of the development repository),
-built_from is used instead.
+there; the tag marks the same tree in the public history, and it counts only
+as an annotated tag whose message has the line `built from <built_from>`. When
+the tag is absent and built_from is present (a clone of the development
+repository), built_from is used instead.
+
+`--expect-version` refuses a manifest that names another version, and `--out
+DIR` writes what a release carries once everything matched: the rebuilt .pyz
+and its .sha256, the manifest's exact bytes and its signature file. The
+GitHub release workflow (.github/workflows/release.yml) runs it that way.
 
 The signature check is ed25519 in pure Python, because the standard library
 has no ed25519 and this has to run where nothing else is installed. It only
@@ -228,15 +234,33 @@ def _commit(repo, ref):
     return r.stdout.strip() if r.returncode == 0 else ""
 
 
+def tag_records(repo, tag, built_from):
+    """True when tag is an annotated tag whose message has the line
+    `built from <built_from>`, the record publishing writes on every release
+    tag. A tag without it says nothing about which release it is."""
+    if not built_from:
+        return False
+    if _git(repo, "cat-file", "-t", "refs/tags/" + tag, check=False).stdout.strip() != "tag":
+        return False
+    msg = _git(repo, "tag", "-l", "--format=%(contents)", tag, check=False).stdout
+    return ("built from " + built_from) in msg.splitlines()
+
+
 def pick_source(repo, version, built_from):
     """(label, commit) of the source to rebuild: the tag v<version>, else the
-    manifest's built_from, else VerifyError."""
+    manifest's built_from, else VerifyError. The tag counts only when it
+    records the manifest's built_from (tag_records); one that does not is
+    refused rather than passed over."""
     tag = "v" + version
     # Fetch the tag if the clone lacks it; a failure (offline, no such tag)
     # just leaves the local refs to decide.
     _git(repo, "fetch", "-q", "origin", "refs/tags/%s:refs/tags/%s" % (tag, tag), check=False)
     c = _commit(repo, "refs/tags/" + tag)
     if c:
+        if not tag_records(repo, tag, built_from):
+            raise VerifyError("the tag %s does not carry the release record (an annotated tag "
+                              "with the line 'built from %s'); nothing was built"
+                              % (tag, built_from or "(none named)"))
         return "tag " + tag, c
     if built_from:
         c = _commit(repo, built_from)
@@ -249,7 +273,8 @@ def pick_source(repo, version, built_from):
 
 
 def rebuild(repo, commit, version, work):
-    """sha256 of the .pyz that commit's own build.py produces."""
+    """sha256 of the .pyz that commit's own build.py produces. The build's
+    output stays in <work>/dist."""
     src = os.path.join(work, "src")
     _git(repo, "worktree", "add", "-q", "--detach", src, commit)
     try:
@@ -269,20 +294,52 @@ def rebuild(repo, commit, version, work):
         _git(repo, "worktree", "remove", "--force", src, check=False)
 
 
+def write_release(out, dist, version, files, body, sig):
+    """The files a release carries, into out: the rebuilt .pyz and its .sha256
+    (each checked against the manifest's digest for that name), the manifest
+    as it was served, byte for byte, and its signature under the name it is
+    served by. Called only after the signature and the rebuild both passed."""
+    os.makedirs(out, exist_ok=True)
+    for name in ("bastion-agent-%s.pyz" % version, "bastion-agent-%s.pyz.sha256" % version):
+        with open(os.path.join(dist, name), "rb") as fh:
+            data = fh.read()
+        want = (files.get(name) or {}).get("sha256")
+        if hashlib.sha256(data).hexdigest() != want:
+            raise VerifyError("the rebuilt %s does not match the manifest's sha256 for it" % name)
+        with open(os.path.join(out, name), "wb") as fh:
+            fh.write(data)
+    with open(os.path.join(out, "version.json"), "wb") as fh:
+        fh.write(body)
+    with open(os.path.join(out, sig_name(body)), "wb") as fh:
+        fh.write(sig)
+    print("release:   wrote %s" % ", ".join(sorted(os.listdir(out))))
+
+
 def main(argv=None, repo=REPO, keys=None):
     ap = argparse.ArgumentParser(prog="bastion_agent.verify_release",
                                  description="Rebuild the published agent and compare it with the release manifest.")
     ap.add_argument("--base", default=os.environ.get("MANIFEST_BASE", DEFAULT_BASE),
                     help="where version.json is served (default %s)" % DEFAULT_BASE)
+    ap.add_argument("--expect-version", metavar="VERSION",
+                    help="refuse unless the published manifest names this version (v0.1.16 or 0.1.16)")
+    ap.add_argument("--out", metavar="DIR",
+                    help="when everything matches, write the rebuilt .pyz and its .sha256, the "
+                         "manifest's exact bytes and its signature file into DIR")
     args = ap.parse_args(argv)
     base = args.base.rstrip("/")
     work = tempfile.mkdtemp(prefix="agent-verify-")
     try:
         body = fetch(base + "/version.json")
-        sig = fetch(base + "/" + sig_name(body)).decode("ascii", "replace")
+        sig_raw = fetch(base + "/" + sig_name(body))
+        sig = sig_raw.decode("ascii", "replace")
         m, key = check_signature(body, sig, keys)
         ver, built_from = m["version"], m.get("built_from") or ""
         print("published: v%s built from %s" % (ver, built_from or "(not named)"))
+        want = (args.expect_version or "").strip()
+        if want.startswith("v"):
+            want = want[1:]
+        if want and want != ver:
+            raise VerifyError("the published manifest names v%s, not v%s" % (ver, want))
         print("signature: ok (release key %d of %d, %s)"
               % (key + 1, len(RELEASE_KEYS if keys is None else keys), "primary" if key == 0 else "spare"))
         files = m.get("files") if isinstance(m.get("files"), dict) else {}
@@ -295,12 +352,16 @@ def main(argv=None, repo=REPO, keys=None):
         print("source:    %s (%s)" % (label, commit[:12]))
         got = rebuild(repo, commit, ver, work)
         bad = 0
-        for name, want in pyz.items():
-            same = got == want
+        for name, digest in pyz.items():
+            same = got == digest
             bad += not same
             print("rebuild:   %-28s %s" % (name, "matches %s" % got if same
-                                             else "DIFFERS (rebuilt %s, published %s)" % (got, want)))
-        return 1 if bad else 0
+                                             else "DIFFERS (rebuilt %s, published %s)" % (got, digest)))
+        if bad:
+            return 1
+        if args.out:
+            write_release(args.out, os.path.join(work, "dist"), ver, files, body, sig_raw)
+        return 0
     except VerifyError as exc:
         print("verify: %s" % exc, file=sys.stderr)
         return 1

bastion_agent/walk.py

@@ -25,7 +25,6 @@ import json
 import os
 import platform
 import re
-import sys
 import tempfile
 
 from .manifest import AGENT_DIR, signing_digest