bastion agent 0.1.17

bastion agent 0.1.16
2026-10-29 21:42:06 +00:00 · 2026-10-25 23:16:50 +00:00
43 changed files with 1556 additions and 222 deletions
@@ -17,6 +17,8 @@ env:
  PY_CURRENT: python@sha256:7c61056e61ac89e852de05f3dc6fa51a6dd2181797bceed46aa725dd7cb2cd3b  # 3.13-slim
  # The PyYAML release bastion_agent/build.py vendors; the manifest loader imports it.
  PYYAML: pyyaml==6.0.3
  # The linter: every name a module imports is used, and none is undefined.
  PYFLAKES: pyflakes==4.0.3
  # The pull request's own head, or the pushed commit.
  WANT: ${{ gitea.event.pull_request.head.sha || gitea.sha }}
@@ -36,17 +38,27 @@ jobs:
        run: |
          docker run --rm -v "$PWD":/src:ro -w /src "$PY_CURRENT" \
            python -m unittest -v test_syntax_floor
      - name: pyflakes
        run: |
          docker run --rm -v "$PWD":/src:ro -w /src -e PYFLAKES "$PY_CURRENT" sh -euc '
            pip install -q --disable-pip-version-check --root-user-action=ignore "$PYFLAKES"
            python -m pyflakes bastion_agent pii scripts test_syntax_floor.py'
      - name: every test, on Python 3.8 and on a current Python
-        # age is installed so the seal round trips run instead of skipping.
+        # age is installed so the seal round trips run, and git so the release
        # verifier's end-to-end tests do. Every test runs here: a skipped one
        # fails the job (the only skips the suite has are for Windows, or for a
        # tool missing, and this job installs the tools).
        run: |
          set -euo pipefail
          for img in "$PY_FLOOR" "$PY_CURRENT"; do
            docker run --rm -v "$PWD":/src:ro -w /src -e PYYAML "$img" sh -euc '
              python --version
              apt-get -qq update >/dev/null
-              apt-get -qq install -y --no-install-recommends age >/dev/null
+              apt-get -qq install -y --no-install-recommends age git >/dev/null
              pip install -q --disable-pip-version-check --root-user-action=ignore "$PYYAML"
-              python -m unittest discover .'
+              python -m unittest discover -v . > /tmp/tests.log 2>&1 || { cat /tmp/tests.log; exit 1; }
              grep -E "^(Ran|OK|FAILED)" /tmp/tests.log
              if grep -E "\.\.\. skipped|^(OK|FAILED) \(.*skipped=" /tmp/tests.log; then echo "a test was skipped; CI runs every test"; exit 1; fi'
          done
  personal-data:
@@ -62,9 +74,14 @@ jobs:
          test "$(git rev-parse HEAD)" = "$WANT"
      - name: no person's data in the tree
        # Reads files, runs nothing from the tree. The term check needs the
-        # maintainers' key, which a CI job never holds, so here it is skipped and
+        # maintainers' key and the entity-id check the reserved ids, which a CI
-        # the run says so; the entity-id, provenance, email and phone checks run.
+        # job never holds, so the job names files that do not exist: both
-        run: python3 scripts/test_no_personal_data.py
+        # classes are skipped and the run prints a NOTE for each; the
        # provenance, email and phone checks run. Left at their defaults, a
        # path that exists but cannot be read fails the run instead.
        run: |
          none="$PWD/.ci-holds-none/none"
          NO_PERSONAL_DATA_KEY_FILE="$none" RESERVED_ENTITY_IDS_FILE="$none" python3 scripts/test_no_personal_data.py
  reproducible:
    needs: test
@@ -0,0 +1,116 @@
 # The checks of .gitea/workflows/ci.yml, for the copy of this repository on
 # GitHub. Gitea reads .gitea/workflows when it exists and then ignores this
 # directory, so each forge runs one of the two files, never both.
 #
 # The jobs, images and commands are the same as there. No action from the
 # marketplace is used: the commit is fetched with git from this repository's
 # public URL, so nothing outside this file and the pinned images runs. The job
 # token can only read.
 name: ci
 on:
  push:
    branches: [main]
  pull_request:
    branches: [main]
 permissions:
  contents: read
 env:
  PY_FLOOR: python@sha256:1d52838af602b4b5a831beb13a0e4d073280665ea7be7f69ce2382f29c5a613f    # 3.8-slim
  PY_CURRENT: python@sha256:7c61056e61ac89e852de05f3dc6fa51a6dd2181797bceed46aa725dd7cb2cd3b  # 3.13-slim
  # The PyYAML release bastion_agent/build.py vendors; the manifest loader imports it.
  PYYAML: pyyaml==6.0.3
  # The linter: every name a module imports is used, and none is undefined.
  PYFLAKES: pyflakes==4.0.3
  # The pull request's own head, or the pushed commit.
  WANT: ${{ github.event.pull_request.head.sha || github.sha }}
  REPO_URL: ${{ github.server_url }}/${{ github.repository }}.git
 jobs:
  test:
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    steps:
      - name: fetch the commit
        run: |
          set -euo pipefail
          git init -q .
          git fetch -q --depth=1 "$REPO_URL" "$WANT"
          git checkout -q --detach FETCH_HEAD
          test "$(git rev-parse HEAD)" = "$WANT"
      - name: the 3.8 syntax floor
        run: |
          docker run --rm -v "$PWD":/src:ro -w /src "$PY_CURRENT" \
            python -m unittest -v test_syntax_floor
      - name: pyflakes
        run: |
          docker run --rm -v "$PWD":/src:ro -w /src -e PYFLAKES "$PY_CURRENT" sh -euc '
            pip install -q --disable-pip-version-check --root-user-action=ignore "$PYFLAKES"
            python -m pyflakes bastion_agent pii scripts test_syntax_floor.py'
      - name: every test, on Python 3.8 and on a current Python
        # age is installed so the seal round trips run, and git so the release
        # verifier's end-to-end tests do. Every test runs here: a skipped one
        # fails the job (the only skips the suite has are for Windows, or for a
        # tool missing, and this job installs the tools).
        run: |
          set -euo pipefail
          for img in "$PY_FLOOR" "$PY_CURRENT"; do
            docker run --rm -v "$PWD":/src:ro -w /src -e PYYAML "$img" sh -euc '
              python --version
              apt-get -qq update >/dev/null
              apt-get -qq install -y --no-install-recommends age git >/dev/null
              pip install -q --disable-pip-version-check --root-user-action=ignore "$PYYAML"
              python -m unittest discover -v . > /tmp/tests.log 2>&1 || { cat /tmp/tests.log; exit 1; }
              grep -E "^(Ran|OK|FAILED)" /tmp/tests.log
              if grep -E "\.\.\. skipped|^(OK|FAILED) \(.*skipped=" /tmp/tests.log; then echo "a test was skipped; CI runs every test"; exit 1; fi'
          done
  personal-data:
    runs-on: ubuntu-24.04
    timeout-minutes: 10
    steps:
      - name: fetch the commit
        run: |
          set -euo pipefail
          git init -q .
          git fetch -q --depth=1 "$REPO_URL" "$WANT"
          git checkout -q --detach FETCH_HEAD
          test "$(git rev-parse HEAD)" = "$WANT"
      - name: no person's data in the tree
        # Reads files, runs nothing from the tree. The term check needs the
        # maintainers' key and the entity-id check the reserved ids, which
        # this job never holds, so both are skipped and the run prints a NOTE
        # for each; the provenance, email and phone checks run.
        run: |
          set -euo pipefail
          docker run --rm -v "$PWD":/src:ro -w /src "$PY_CURRENT" \
            python -I scripts/test_no_personal_data.py --root /src
  reproducible:
    needs: test
    runs-on: ubuntu-24.04
    timeout-minutes: 20
    steps:
      - name: fetch the commit
        run: |
          set -euo pipefail
          git init -q .
          git fetch -q --depth=1 "$REPO_URL" "$WANT"
          git checkout -q --detach FETCH_HEAD
          test "$(git rev-parse HEAD)" = "$WANT"
      - name: two builds on each Python produce one sha256
        # Anyone must be able to rebuild a release and get the published bytes,
        # so the .pyz may not depend on the build or the Python that made it.
        run: |
          set -euo pipefail
          for img in "$PY_FLOOR" "$PY_CURRENT"; do
            docker run --rm -v "$PWD":/src:ro -w /src -e PYYAML "$img" sh -euc '
              pip install -q --disable-pip-version-check --root-user-action=ignore "$PYYAML"
              python -m bastion_agent.build --out /tmp/a | awk "{print \$NF}"
              python -m bastion_agent.build --out /tmp/b | awk "{print \$NF}"'
          done | tee digests
          test "$(wc -l < digests)" -eq 4
          test "$(sort -u digests | wc -l)" -eq 1
          echo "every build is $(head -1 digests)"
@@ -0,0 +1,174 @@
 # Attach the release files to the GitHub release of a tag v<version>.
 #
 # GitHub only: Gitea reads .gitea/workflows when it exists and then ignores
 # this directory, so the forge's copy of this repository never runs this file.
 #
 # The files are not uploaded from anywhere else. This job rebuilds them from
 # the tag and attaches them only when they are the published release:
 #
 #   1. bastion_agent/verify_release.py fetches the live manifest
 #      (privatae.ai/bastion-agent/version.json) and its signature, checks the
 #      signature against the release keys the bastion pins, refuses unless the
 #      manifest names this tag's version and the tag is annotated with the line
 #      `built from <built_from>`, rebuilds the .pyz from the tag with that
 #      tag's own build.py, and compares the sha256 with the manifest;
 #   2. only then are the rebuilt .pyz, its .sha256, the manifest (its exact
 #      bytes) and the signature file attached to the release, which is created
 #      when the tag has none, with that version's CHANGELOG section.
 #
 # The live manifest names only the newest release, so a release can be given
 # its files only while it is the published one. Older releases, whose manifest
 # is no longer served, cannot be filled in afterwards. A run that failed (the
 # tag pushed before the release went live, say) is run again from the Actions
 # tab with the tag as input.
 #
 # No action from the marketplace is used, and nothing is fetched with a
 # credential: the repository is read from its public URL. The job may write
 # to the repository's releases; the token is handed to the last step only.
 name: release
 on:
  push:
    tags: ["v*.*.*"]
  workflow_dispatch:
    inputs:
      tag:
        description: "the release tag, vX.Y.Z"
        required: true
 permissions:
  contents: read
 env:
  REPO_URL: ${{ github.server_url }}/${{ github.repository }}.git
  TAG: ${{ github.event.inputs.tag || github.ref_name }}
 defaults:
  run:
    shell: bash -euo pipefail {0}
 jobs:
  release:
    runs-on: ubuntu-24.04
    timeout-minutes: 20
    permissions:
      contents: write
    steps:
      - name: the tag is a release tag
        run: |
          printf '%s\n' "$TAG" | grep -Exq 'v[0-9]+\.[0-9]+\.[0-9]+' \
            || { echo "not a release tag: $TAG"; exit 1; }
      - name: fetch the repository and its tags
        # verify_release.py is run from the commit this workflow runs from: the
        # tag itself on a tag push, the default branch on a manual run. It
        # builds the tag's own source in a separate worktree.
        run: |
          git init -q .
          git remote add origin "$REPO_URL"
          git fetch -q origin "+refs/tags/*:refs/tags/*" "$GITHUB_SHA"
          git checkout -q --detach "$GITHUB_SHA"
          test "$(git rev-parse HEAD)" = "$GITHUB_SHA"
          git rev-parse -q --verify "refs/tags/$TAG^{commit}" >/dev/null \
            || { echo "no tag $TAG in this repository"; exit 1; }
      - name: check the signed manifest and rebuild the .pyz from the tag
        run: |
          # The PyYAML release bastion_agent/build.py vendors, as a wheel only
          # and only with one of these sha256s (PyPI's, for Linux on x86_64 and
          # arm64, Python 3.12 and 3.13): the one package this job takes from
          # outside the repository, and this step runs before the upload step
          # that holds the token. A different PyYAML would only make the
          # rebuild differ and the step refuse.
          cat > "$RUNNER_TEMP/requirements.txt" <<'REQ'
          pyyaml==6.0.3 \
            --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \
            --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \
            --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \
            --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c
          REQ
          /usr/bin/python3 -m venv /tmp/venv
          /tmp/venv/bin/pip install -q --disable-pip-version-check --only-binary :all: \
            --require-hashes -r "$RUNNER_TEMP/requirements.txt"
          /tmp/venv/bin/python -m bastion_agent.verify_release \
            --expect-version "$TAG" --out "$RUNNER_TEMP/release"
          ls -l "$RUNNER_TEMP/release"
      - name: the release notes, from that version's CHANGELOG section
        run: |
          /usr/bin/git show "$TAG:CHANGELOG.md" > "$RUNNER_TEMP/CHANGELOG.md"
          /usr/bin/python3 -I - "${TAG#v}" "$RUNNER_TEMP/CHANGELOG.md" > "$RUNNER_TEMP/notes.md" <<'PY'
          import re, sys
          ver, path = sys.argv[1], sys.argv[2]
          text = open(path, encoding="utf-8").read()
          m = re.search(r"^## %s[ \t]*\n(.*?)(?=^## |\Z)" % re.escape(ver), text, re.M | re.S)
          if not m:
              sys.exit("CHANGELOG.md at the tag has no section for %s" % ver)
          print(m.group(1).strip())
          print()
          print("---")
          print("The files below are this tag rebuilt by this repository's release workflow, "
                "attached only after their sha256 matched the signed manifest "
                "(`version.json` and its `.sig`). To check them yourself, run "
                "`python3 -m bastion_agent.verify_release` from a clone: "
                "see [Verify a release](https://github.com/%s#verify-a-release)."
                % "Privatae/bastion-agent")
          PY
      - name: attach the files to the release
        # The one step with the token. Its shell and Python are named by path,
        # and Python runs isolated (-I), so nothing an earlier step put on PATH
        # or in the environment chooses what runs here.
        shell: /usr/bin/bash -euo pipefail {0}
        env:
          GH_TOKEN: ${{ github.token }}
          GH_REPO: ${{ github.repository }}
        run: |
          /usr/bin/python3 -I - "$RUNNER_TEMP/release" "$RUNNER_TEMP/notes.md" <<'PY'
          import json, os, sys, urllib.error, urllib.parse, urllib.request
          out_dir, notes = sys.argv[1], sys.argv[2]
          api = os.environ.get("GITHUB_API_URL", "https://api.github.com").rstrip("/")
          repo, tag, token = os.environ["GH_REPO"], os.environ["TAG"], os.environ["GH_TOKEN"]
          def call(method, url, body=None, ctype="application/json"):
              data = body if isinstance(body, bytes) or body is None else json.dumps(body).encode()
              req = urllib.request.Request(url, data=data, method=method, headers={
                  "Authorization": "Bearer " + token, "Accept": "application/vnd.github+json",
                  "X-GitHub-Api-Version": "2022-11-28", "Content-Type": ctype,
                  "User-Agent": "bastion-agent-release"})
              try:
                  with urllib.request.urlopen(req, timeout=120) as r:
                      raw = r.read()
                      return r.status, json.loads(raw) if raw else None
              except urllib.error.HTTPError as e:
                  return e.code, None
          st, rel = call("GET", "%s/repos/%s/releases/tags/%s" % (api, repo, tag))
          if st == 404:
              body = open(notes, encoding="utf-8").read()
              st, rel = call("POST", "%s/repos/%s/releases" % (api, repo), {
                  "tag_name": tag, "name": "bastion agent " + tag[1:], "body": body,
                  "draft": False, "prerelease": False})
              if st != 201:
                  sys.exit("creating the release for %s failed: HTTP %d" % (tag, st))
              print("created the release for %s" % tag)
          elif st != 200:
              sys.exit("reading the release for %s failed: HTTP %d" % (tag, st))
          else:
              print("the release for %s exists; its text is left as it is" % tag)
          st, assets = call("GET", "%s/repos/%s/releases/%d/assets?per_page=100" % (api, repo, rel["id"]))
          if st != 200:
              sys.exit("listing the release's files failed: HTTP %d" % st)
          have = {a["name"]: a["id"] for a in assets}
          upload = rel["upload_url"].split("{", 1)[0]
          for name in sorted(os.listdir(out_dir)):
              if name in have:
                  st, _ = call("DELETE", "%s/repos/%s/releases/assets/%d" % (api, repo, have[name]))
                  if st != 204:
                      sys.exit("replacing %s failed: HTTP %d" % (name, st))
              with open(os.path.join(out_dir, name), "rb") as fh:
                  data = fh.read()
              st, a = call("POST", upload + "?" + urllib.parse.urlencode({"name": name}), data,
                           "application/octet-stream")
              if st != 201:
                  sys.exit("uploading %s failed: HTTP %d" % (name, st))
              print("attached %s (%d bytes)" % (a["name"], a["size"]))
          PY
@@ -14,6 +14,132 @@ Changes to the release tooling (`publish.sh`, `build.py`, `verify_release.py`)
 and to the documentation are listed under the version whose source first
 carried them, although they do not change what the `.pyz` does.
 ## 0.1.17
 - **Messages no longer claim what the code does not do.** A manifest with
  `sink.catalog: host` and no `seal` stage is refused with "the copy meant
  for the host is the sealed bundle", and one with a grant and no `seal`
  stage with "any grant requires the 'seal' stage"; neither says that
  nothing leaves or is served unsealed, since `serve`'s replies are never
  sealed. A malformed `doc` in `read_range` is refused as "the original
  file's sha256", which is what the id is. What the manifest accepts and
  refuses is unchanged.
 - **The documentation says what reaches a Host.** What the agent
  returns (catalog cards, search hits, excerpts by document id) comes from
  the sanitized text, and the agent sends no file or extracted text to a
  Host. A folder grant with `read`, the default, also lets the Host ask the
  bastion itself for the folder's file names, sizes and times and for raw
  ranges of its files, up to 512 KiB per read. From bastion 0.58.1 on, none
  of that reaches the agent's own `.agent/` folder. An older bastion serves
  `.agent/` and the plaintext extract store in it wherever it keeps the agent
  inside the folder, unless the grant's excluded paths list `.agent`.
  `seal` writes its bundle and sends it nowhere. A file card's `sha256` is
  the original file's hash.
 - **`flag` mode's limits are listed.** It does not look for bank account
  numbers other than IBANs and US routing numbers after a word such as
  "routing", crypto addresses other than Bitcoin addresses that start with
  1 or 3, or ID numbers such as passport numbers.
 - **Every model server is the one the environment names.** `EMBED_URL` and
  `SUMMARIZE_URL` default to the Ollama port on the same machine, and the
  bastion sets `EMBED_URL` to `127.0.0.1:11434`. `sterilize` mode sends the
  unsanitized text to the PII service at `PII_ADDR` when that is set;
  otherwise to the server at `LLM_URL`, unless `PII_RECALL=gliner` runs the
  model inside the agent.
 - **Smaller corrections.** The `doc` id is the original file's sha256
  (embed.md); CI runs on every push and pull request (packaging.md); signed
  manifests are checked from bastion 0.42.0, since 0.41.0 was never
  published; the bastion installs Privatae's signed build, which anyone can
  rebuild from this source.
 - **Tests look for the whole test card number.** Four tests checked that
  the four digits `4111` never appear in a sealed bundle, an index or a
  catalog. Those also hold sha256 hex and times, where the same four digits
  turn up by chance: CI failed on a seal manifest hash ending `...81934111`.
  They now look for the full card value, spaced and unspaced.
 ## 0.1.16
 - **Every `search` and `read_range` reply names its sanitize mode**
  (`sanitize_mode`: `flag` or `sterilize`, the mode the sanitize stage
  recorded for the text in the reply; `unknown` when that record does not
  vouch for it; a search with no hits names the manifest's mode, or `none`),
  so the bastion and the Host can say what was done to the text. Switching
  the manifest's mode does not relabel text an earlier sweep wrote. In `flag`
  mode, the mode the bastion deploys, names, street addresses and dates of
  birth remain. The field is new; nothing else in either reply changes.
 - **Each GitHub release carries its files.** Pushing a release tag starts
  `.github/workflows/release.yml`, which runs the release check against the
  published manifest, rebuilds the `.pyz` from the tag, and only when the
  sha256 matches attaches the `.pyz`, its `.sha256`, `version.json` and its
  signature to the release (making the release, with this file's section for
  it, when the tag has none). It can be run again by hand for the version
  the manifest names; older releases, whose manifest is no longer served,
  cannot be given their files afterwards. The manifest's `notes` now link to
  that GitHub release instead of the tag page on the forge.
 - **`verify_release` refuses a tag without its release record** (an annotated
  tag with the line `built from <built_from>`, as every release tag from
  0.1.14 on has), instead of building from any tag of that name. It gains
  `--expect-version`, which refuses a manifest naming another version, and
  `--out DIR`, which writes the rebuilt `.pyz` and `.sha256`, the manifest's
  exact bytes and its signature once everything matched.
 - **CI runs every test and pyflakes.** The test job installs `git` too, so
  the release verifier's seven end-to-end tests, which were skipped in CI, now
  run, and a skipped test fails the job. pyflakes 4.0.3 runs on both forges;
  the 31 findings it had (unused imports and variables) are fixed, and the
  packages' re-exports are listed in `__all__`.
 - **The README says what it means.** The agent is deployed from a grant and
  can also be run by hand; what crosses the tunnel is the catalog cards and
  sanitized excerpts; building and testing need PyYAML 6.0.3; the CI badge is
  the GitHub one.
 - **The rules pass finds more personal data on its own,** which matters
  because it is all that `flag` mode, the mode the bastion deploys, runs.
  Newly found: international phone numbers (`+44 7700 900123`,
  `+49 (0)30 1234567`, no separators, dots or dashes), national numbers with a
  trunk `0` (`07700 900123`), a seven-digit local number after a word such as
  call or phone, IBANs written in lower case, card numbers written with dots,
  SSNs written with spaces, and nine-digit SSNs with no separators when `SSN`
  or `social security` comes just before them. A bare nine-digit number, a
  seven-digit number with no phone word before it (`Order 555-0142`), a number
  that fails its checksum and a date are still left alone; a reference number
  that starts with `0` (`0123-456789`) is now taken for a national phone
  number. An expiry date or security code beside a card is still not found.
 - **A dotted number right after a version word is no longer taken for an IP
  address** (`version 1.2.3.4`), and SSNs that are never issued (group `00`,
  serial `0000`) are no longer masked.
 - **Every file is sanitized again once after this upgrade** (`SANITIZE_VERSION`
  3), so text kept by the older rules is redone.
 - **The docs say what the deployed mode leaves in.** The README, SECURITY.md
  and the sanitize, serve and PII docs say plainly that `flag` mode does not
  find names, street addresses or dates of birth, and that `sterilize` mode
  adds a model that looks for them.
 - **The PII service listens on 127.0.0.1 by default** instead of every
  interface; `PII_BIND` still chooses another address. Its container image runs
  as uid 10001 instead of root, so a volume mounted on `/data` must be writable
  by that user, and a container that should be reachable needs `PII_BIND` set.
 - **CI also runs on GitHub** (`.github/workflows/ci.yml`): the same jobs, the
  same pinned images.
 - **The README's license paragraph and SECURITY.md name each kind of Host**
  (docs only) instead of only Privatae's platform: a
  C-Series Host runs on your own machine and is open source (Privatae/cseries,
  AGPL-3.0-only), C-Series Cloud and a network membership are optional and
  paid, and a Host on Privatae's own platform is the case where the Host's
  software is not open source.
 - **Security reports go to security@privatae.ai,** the address
  privatae.ai/.well-known/security.txt names, and SECURITY.md links the
  bastion's policy on GitHub, where anyone can read it.
 - **How this repository is published.** From this release on, GitHub receives
  each change merged since the last release as its own commit, titled as the
  change was, instead of one commit per release, and the README says so.
 - **The personal-data check reads what it used to miss.** A listed term
  written right after a backslash escape (`\n`, `\t`, `\u00e9`) is found, and
  so is any value inside a run of base64 or base64url that decodes to text.
  The reserved entity ids are read from `/etc/privatae/reserved-entity-ids`
  (one per line, `RESERVED_ENTITY_IDS_FILE` to point elsewhere) instead of
  being written in the script; where that file is absent, as in public CI,
  the entity-id check is skipped and the run prints a NOTE, as the term check
  already does without its key. A file that holds no id, or a line that is
  not one, fails the run.
 ## 0.1.15
 - **No change to what the agent does.** The README says how this repository
@@ -11,6 +11,7 @@ From the repository root:
 ```sh
 python3 -m unittest discover .                    # every test, including the 3.8 floor
 python3 -m pyflakes bastion_agent pii scripts test_syntax_floor.py   # pyflakes==4.0.3, as CI runs it
 python3 -m bastion_agent.build --out /tmp/agent   # the .pyz and its .sha256
 python3 /tmp/agent/bastion-agent-*.pyz --version
 bash -n bastion_agent/publish.sh pii/verify-live.sh && sh -n bastion_agent/install.sh
@@ -19,7 +20,10 @@ bash -n bastion_agent/publish.sh pii/verify-live.sh && sh -n bastion_agent/insta
 - **All tests pass.** They need PyYAML (the manifest loader uses it), at the
  version `build.py` pins (`PYYAML_VERSION`) for the build tests. The
  round-trip seal tests are skipped where `age` and `age-keygen` are not
-  installed, so install them to run the whole suite.
+  installed, the TLS tests where `openssl` is not, and the release verifier's
  end-to-end tests where `git` is not, so install them to run the whole suite.
  CI installs them and fails on any skipped test.
 - **pyflakes finds nothing.** No unused import or variable, no undefined name.
 - **The Python 3.8 floor.** `test_syntax_floor.py` parses every module in
  `bastion_agent/` and `pii/` at 3.8 and denies the newer standard-library
  calls that parse on 3.8 and then fail at run time. The agent ships to
@@ -39,11 +43,19 @@ refuses adds a test that it refuses.
 ## What a change has to respect
- **Bytes never leave the machine.** The files, their extracted text and the
+- **The agent sends no file and no extracted text to a Host.** It keeps the
-  vault stay local. What leaves is the sealed catalog, to the recipients the
+  files, their extracted text and the vault on the machine. The one exception
-  manifest names, and what `serve` returns to a certificate the manifest
+  is `sterilize` mode with a server: the PII service at `PII_ADDR`, or the
-  allows. There is no plaintext sink: `sink.plaintext` is `never` and has no
+  recall model at `LLM_URL`, is sent the extracted text, and the PII service
-  other value.
+  keeps its own vault ([docs/sanitize.md](docs/sanitize.md#where-the-sterilizer-runs)).
  What the agent makes for others is the sealed catalog, for the recipients
  the manifest names, and what `serve` returns to a certificate the manifest
  allows. There is no plaintext sink:
  `sink.plaintext` is `never` and has no other value. (The bastion's own
  `read` op is separate: it returns ranges of any file in the granted folder
  that the grant does not exclude, as they are on disk. From bastion 0.58.1
  on it returns nothing in `.agent/`; an older bastion does unless the grant
  excludes `.agent`.)
 - **Sanitize before anything is written for later stages.** Chunks, vectors,
  keywords, titles, summaries, the catalog and everything `serve` returns come
  from the sanitized store, never the extract store. A test that searches an
@@ -1,16 +1,29 @@
 # bastion agent
-[![ci](https://git.privatae.ai/privatae/bastion-agent/actions/workflows/ci.yml/badge.svg?branch=main)](https://git.privatae.ai/privatae/bastion-agent/actions)
+[![ci](https://github.com/Privatae/bastion-agent/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/Privatae/bastion-agent/actions/workflows/ci.yml)
 The small program the bastion runs on your own machine, for one folder.
-It is deployed **from a grant, never installed by a person**: the bastion fetches
+It is deployed **from a grant**, and you do not install it yourself: the bastion fetches
 `privatae.ai/bastion-agent/version.json`, checks its ed25519 release signature
 (`version.json.<first 16 hex of its sha256>.sig`) against the keys compiled into the bastion, downloads the `.pyz`
 and checks it against the sha256 in that signed manifest,
 mints a P-256 identity per store, writes and signs a manifest from the grant you
-approved, and supervises `sweep` and `serve`. Your files never leave the machine;
+approved, and supervises `sweep` and `serve`. Nothing is uploaded or synced;
-what crosses the tunnel is the index and what the grant's mask allows.
+your Host asks, and gets what the grant's mask allows. What the agent returns
 is sanitized: catalog cards, search hits, and the excerpts a `read_range` by
 document id returns, each reply naming the sanitize mode that made it
 (`sanitize_mode`, [serve.md](docs/serve.md)). A grant with `read`, which a
 connected folder has by default, also lets the Host ask the bastion itself for
 the folder's file names, sizes and modification times, and for one range of
 one file per read, up to 512 KiB, as it is on disk and not sanitized. From
 bastion 0.58.1 on, none of this reaches the agent's own `.agent/` folder: the
 bastion leaves every folder named `.agent` out of a listing and refuses any
 request for a path in one. An older bastion serves `.agent/` like any other
 folder, including the unsanitized text the agent extracted from every file,
 unless `.agent` is among the grant's excluded paths
 ([runtime.md](docs/runtime.md#where-an-agent-lives-rootagent) says when the
 agent's folder is inside the shared folder).
 The published version is whatever `privatae.ai/bastion-agent/version.json`
 names. `bastion_agent/VERSION` on `main` is the next release, which may not be
@@ -18,7 +31,7 @@ published yet. Release manifests are published signed from 0.1.13 on, and the
 `.pyz` can be rebuilt from this source and compared
 ([Verify a release](#verify-a-release)).
-This repository is free software, licensed under the GNU General Public License version 3 or (at your option) any later version (see `LICENSE`). Copyright Privatae LLC. You may run it, read it, change it and redistribute it under that license. The Privatae platform the bastion connects to is separate, proprietary software and is not covered by this license. A member runs their own build on their own machine — nothing here depends on a Privatae-held credential or a Privatae-side build.
+This repository is free software, licensed under the GNU General Public License version 3 or (at your option) any later version (see `LICENSE`). Copyright Privatae LLC. You may run it, read it, change it and redistribute it under that license. The Host the bastion connects to is separate software and is not covered by this license: a C-Series Host is [Privatae/cseries](https://github.com/Privatae/cseries), under AGPL-3.0-only, and a Host on Privatae's own platform is proprietary software. Anyone can build and run the agent from this source with no Privatae account or key; the bastion installs Privatae's signed build, which this source reproduces.
 ## What it does
@@ -30,13 +43,13 @@ failure stops the run:
 |---|---|---|
 | walk | the map of the folder: files, sizes, types, what the grant excludes | [walk.md](docs/walk.md) |
 | extract | text out of each file, with built-in extractors or local tools | [extract.md](docs/extract.md) |
-| sanitize | finds personal data and replaces it with tokens; the originals stay in a vault on this machine | [sanitize.md](docs/sanitize.md), [pii-sterilization.md](docs/pii-sterilization.md) |
+| sanitize | replaces the personal data its mode finds with tokens; the originals stay in a vault on this machine, or in a PII service's own vault when `PII_ADDR` names one. In the mode the bastion deploys it does not find names, street addresses or dates of birth ([below](#how-the-bastion-runs-it)) | [sanitize.md](docs/sanitize.md), [pii-sterilization.md](docs/pii-sterilization.md) |
-| chunk, embed | splits the text and embeds it with a model on this machine | [embed.md](docs/embed.md) |
+| chunk, embed | splits the text and embeds it with the model server `EMBED_URL` names: by default, and always under the bastion, the Ollama port on the same machine | [embed.md](docs/embed.md) |
 | catalog | the curated dataset built from the stages above | [catalog.md](docs/catalog.md) |
-| seal | packs an allowlisted bundle of the catalog and encrypts it with age to the owner and the manifest's named recipients; the one way a dataset leaves the machine | [seal.md](docs/seal.md) |
+| seal | packs an allowlisted bundle of the catalog into one file and encrypts it with age to the owner and the manifest's named recipients; it writes the file into the output folder and sends it nowhere | [seal.md](docs/seal.md) |
-`serve` answers queries over the index with mutual TLS, returning sanitized
+`serve` answers queries over the index with mutual TLS, returning chunks and
-chunks and catalog cards ([serve.md](docs/serve.md)). The gate refuses an
+catalog cards from the sanitized text ([serve.md](docs/serve.md)). The gate refuses an
 unsigned, edited or foreign manifest, and the bastion never starts the agent
 without a valid one. (`--allow-unsigned` exists for development only; the
 bastion never passes it.) See [runtime.md](docs/runtime.md) and
@@ -45,14 +58,31 @@ bastion never passes it.) See [runtime.md](docs/runtime.md) and
 ## How the bastion runs it
 The bastion installs and runs the agent itself from a folder grant you
-approved; nobody installs it by hand. For each shared folder the bastion:
+approved; you do not install it by hand. (The `.pyz` can also be installed and
 run by hand, to try it outside the bastion: [packaging.md](docs/packaging.md).)
 For each shared folder the bastion:
 - downloads the `.pyz` only through the signed manifest, and never replaces an
  installed agent with an older version;
 - mints the agent its own identity and writes and signs its manifest from the
  grant, so the agent works only on what you granted;
- points the embed stage at the model server on this machine (a local ollama)
+- sets the sanitize stage to `flag` mode: rules only, no model. It replaces
-  and at nothing else;
+  e-mail addresses, phone numbers, US social security numbers, card numbers,
  IBANs, US routing numbers that follow a word such as "routing", Bitcoin
  addresses that start with 1 or 3, IP and MAC addresses, and keys and tokens
  of known formats ([the full list](docs/pii-sterilization.md#the-rules-pass-piidetectpy)).
  It does not look for other bank account numbers, other crypto addresses or
  ID numbers such as passport numbers, though digits in one can still be
  replaced when they look like a phone, card or social security number.
  **It does not find names, street addresses or dates of birth**, so text a
  Host receives from the agent still contains them. `sterilize` mode adds a
  model that looks for those. When `PII_ADDR` is set, the unsanitized text
  goes to the PII service it names. Otherwise the model is the server
  `LLM_URL` names, which is sent the unsanitized text, or, with
  `PII_RECALL=gliner`, one that runs inside the agent. The bastion does not
  set this mode ([sanitize.md](docs/sanitize.md#where-the-sterilizer-runs));
 - points the embed stage at `127.0.0.1:11434`, the Ollama port on the same
  machine, and at nothing else;
 - runs `serve` on the loopback address only, with mutual TLS, so the bastion is
  the only client;
 - keeps the personal-data vault in the bastion's own state directory rather
@@ -81,12 +111,16 @@ package, which is what the built artifact has always looked like inside.
 ## Build and test
 ```sh
 pip install pyyaml==6.0.3                # the PyYAML build.py vendors; the manifest loader imports it
 python3 -m unittest discover .          # every module's tests, plus the 3.8 floor
 python3 -m bastion_agent.build --out dist   # produces dist/bastion-agent-<ver>.pyz + .sha256
 python3 dist/bastion-agent-<ver>.pyz --version
 ```
-Every push runs the same checks in CI (`.gitea/workflows/ci.yml`).
+A test that needs `age`, `openssl` or `git` is skipped where that tool is not
 installed. Every push runs the same checks in CI (`.gitea/workflows/ci.yml` on
 git.privatae.ai, `.github/workflows/ci.yml` on GitHub), with the tools
 installed, so there no test is skipped, plus `pyflakes`.
 The build is reproducible: the same commit gives the same bytes on any machine
 with PyYAML at the pinned version (`PYYAML_VERSION` in `build.py`).
@@ -113,8 +147,9 @@ it); no key or account of ours. It:
   `bastion_agent/verify_release.py`, copied from `client-go/release_sign.go`
   in the bastion's repository;
 2. checks out the source of that version in a separate worktree: the tag
-   `v<version>`, or else the commit the manifest names as `built_from` if the
+   `v<version>`, which must be annotated with the line
-   clone has it;
+   `built from <built_from>`, or else the commit the manifest names as
   `built_from` if the clone has it;
 3. rebuilds the `.pyz` with that version's own `build.py`, and compares its
   sha256 with every `.pyz` the manifest lists.
@@ -122,21 +157,38 @@ A match means the file every bastion downloads is exactly what this source
 produces, and the signature means the release key holder published it. The
 exit code is 0 only when both hold.
-The public repository is a copy of each release's source, not the development
+From 0.1.16 on, each GitHub release carries the `.pyz`, its `.sha256`, the
-history, so `built_from` names a commit that exists only in the development
+signed manifest `version.json` and its signature `version.json.<h>.sig`. The
-repository. From this release on, each public snapshot carries the tag
+release workflow (`.github/workflows/release.yml`) attaches them when the tag
-`v<version>`, which marks the source the release was built from. A snapshot
+is pushed, after running this same check against the published manifest and
-published before that has no tag, and can be checked from `built_from` only
+rebuilding the `.pyz` from the tag; it attaches nothing if a digest differs.
-where that commit is available. A manifest from before signing has no
+To check downloaded files yourself, `sha256sum -c bastion-agent-<ver>.pyz.sha256`
 checks the `.pyz` against its `.sha256`, the manifest lists the sha256 of both,
 and `check_signature` in `bastion_agent/verify_release.py` checks the manifest
 against the pinned release keys:
 ```sh
 python3 -c 'import sys; from bastion_agent import verify_release as v; print(v.check_signature(open(sys.argv[1], "rb").read(), open(sys.argv[2]).read())[0]["files"])' version.json version.json.*.sig
 ```
 The public repository does not carry the development history, so
 `built_from` names a commit that exists only in the development repository.
 Each published release from 0.1.14 on carries the tag `v<version>`, which
 marks the source the release was built from. Earlier releases have no tag
 here, and can be checked from `built_from` only where that commit is
 available. A manifest from before signing has no
 signature file beside it, and the check refuses it.
 ## How this repo is published
-Development happens in a private repository. This repository receives the
+Development happens in a private repository, and its history is not
-source of each published release as one snapshot commit, tagged
+published. When a release is published, each change merged since the last one
-`v<version>`, whose tree is the source the signed manifest says the release
+arrives on GitHub, at
-was built from; that is why its history is short. The same snapshot is on
+[github.com/Privatae/bastion-agent](https://github.com/Privatae/bastion-agent),
-GitHub at [github.com/Privatae/bastion-agent](https://github.com/Privatae/bastion-agent).
+as a commit with the same files, titled as the change was and dated when it
 was merged; the copy at git.privatae.ai receives the release as one commit.
 In both, the release is tagged `v<version>` on a commit whose tree is the
 source the signed manifest says the release was built from.
 Every published `.pyz` can be rebuilt from its tag and compared
 ([Verify a release](#verify-a-release)).
@@ -4,7 +4,7 @@
 Please do not open a public issue for a security problem.
-Email **info@privatae.ai** with:
+Email **security@privatae.ai** with:
 - the agent's version (`python3 bastion-agent-<version>.pyz --version`) and
  platform,
@@ -50,13 +50,22 @@ example:
 - **The bastion**, which installs, updates and supervises the agent, signs its
  manifest and holds its keys. It is a separate program with its own policy:
-  [privatae-bastion's SECURITY.md](https://git.privatae.ai/privatae/privatae-bastion/src/branch/main/SECURITY.md).
+  [the bastion's SECURITY.md](https://github.com/Privatae/bastion/blob/main/SECURITY.md).
  Report there, or to the same address.
- **The Privatae platform** the bastion connects to. It is separate software
+- **The Host** the bastion connects to, which is separate software. A
-  with its own reporting path; reports about it sent to the address above are
+  C-Series Host runs on your own machine, in its own Linux virtual machine;
-  forwarded.
+  its source is [Privatae/cseries](https://github.com/Privatae/cseries), under
- **What the design already says it does not do.** The rules-only `flag` mode
+  AGPL-3.0-only, `npx cseries` installs the bastion beside it and pairs,
-  does not find names, and the recall pass is probabilistic
+  anchors and enrols it, and no Privatae account is needed. C-Series Cloud (a
  Host on a server of its own that Privatae sets up and keeps patched) and a
  Privatae network membership are optional and paid. A Host on Privatae's own
  platform is the case where the Host's software is not open source; the
  bastion keeps it to your grant and your approvals whatever the platform
  does. Reports about a Host sent to the address above are forwarded.
 - **What the design already says it does not do.** The rules-only `flag` mode,
  which is the mode the bastion deploys, does not find names, street addresses
  or dates of birth ([README](README.md#how-the-bastion-runs-it)), and the
  recall pass of `sterilize` mode is probabilistic
  ([docs/pii-sterilization.md](docs/pii-sterilization.md#limits)); the extract
  store in the output folder is plaintext by design
  ([docs/runtime.md](docs/runtime.md#where-an-agent-lives-rootagent)); the
@@ -1 +1 @@
-0.1.15
+0.1.17
@@ -10,3 +10,9 @@ from .manifest import (  # noqa: F401
    signing_payload,
    validate_document,
 )
 # The package's public names: the manifest's schema and its loader.
 __all__ = [
    "STAGES", "MASK_OPS", "SWEEPS", "GRANT_OPS", "Manifest", "ManifestError",
    "load_manifest", "signing_digest", "signing_payload", "validate_document",
 ]
@@ -1,6 +1,6 @@
 """bastion agent `catalog` stage — the curated dataset.
-The catalog is what a host reasons over and what `seal` ships. One card
+The catalog is what a host reasons over and what `seal` packs. One card
 per file, one per directory, small and structured, built from:
    walk.jsonl          metadata (path, size, mtime, mime, hash)
@@ -22,9 +22,10 @@ masked as [PII_TYPE] in the card. Contextual PII in a filename
 ("Marisol_medical.pdf") is not caught by rules — the honest limit of the
 sanitize stage applies here too, and the doc says so.
-The optional summarizer (manifest `summarize`) is a local chat model — a
+The optional summarizer (manifest `summarize`) is a chat model at the
-sensor: it fills the `summary` string and nothing else, is reached via
+server SUMMARIZE_URL names (never the manifest; by default the Ollama port
-SUMMARIZE_URL (never the manifest), sees only sanitized text, and is
+on this machine, 127.0.0.1:11434) — a sensor: it fills the `summary`
 string and nothing else, is sent only sanitized text, and is
 cached per (document, model, version). The first failure stops
 summarizing (cards still get written, `summary_error` set, exit 1).
 """
@@ -151,7 +152,7 @@ def mask_name(name, detect):
 def mask_path(path, detect):
    """Every component of a relative path through mask_name — a parent
    directory can carry an address as easily as a filename, and the file
-    card's path is what seal ships. Returns (masked
+    card's path is what serve returns and seal packs. Returns (masked
    path, number of components changed)."""
    if not path:
        return path, 0
@@ -340,7 +341,6 @@ def catalog(manifest, out_dir, env=None, summarizer=None, now=None):
            continue
        key = (w["root"], w["path"])
        counts["files"] += 1
        name = w["path"].rsplit("/", 1)[-1]
        shown_path, changed = mask_path(w["path"], detect)
        masked = shown_path.rsplit("/", 1)[-1]
        counts["names_masked"] += changed
@@ -7,11 +7,13 @@ tesseract, the gliner package) — their absence is a recorded gap in the
 stage output, not a failure here.
 """
 import importlib
 import json
 import os
 import platform
 import shutil
 import sys
 import urllib.error
 import urllib.request
 PY_FLOOR = (3, 8)
@@ -25,7 +27,6 @@ def _probe_http(url, path, timeout=5):
    """Reachable means the server answered — any HTTP status counts (a
    proxy that only permits the embed path answers 403 on '/'). Only a
    connection failure is 'not reachable'."""
    import urllib.error
    try:
        with urllib.request.urlopen(url.rstrip("/") + path, timeout=timeout) as r:
            return True, "HTTP %d" % r.status
@@ -65,7 +66,7 @@ def checks(env=None):
        add(tool, bool(p), p or ("not on PATH — " + why), required=req, stage=stage)
    try:
-        import gliner  # noqa: F401
+        importlib.import_module("gliner")
        add("gliner package", True, "importable (PII_RECALL=gliner available)", required=False, stage="sanitize")
    except ImportError:
        add("gliner package", False, "not installed — sanitize sterilize on a laptop needs "
@@ -41,7 +41,6 @@ import os
 import re
 import shutil
 import subprocess
 import sys
 import tempfile
 import time
 import unicodedata
@@ -539,9 +539,10 @@ def validate_document(doc, require_signed=False):
            errors.append("summarize: requires the 'catalog' stage (summaries live in the catalog)")
    if stages is not None:
        if sink is not None and sink.catalog == "host" and "seal" not in stages:
-            errors.append("sink: catalog 'host' requires the 'seal' stage — nothing leaves unsealed")
+            errors.append("sink: catalog 'host' requires the 'seal' stage — the copy meant for the host "
                          "is the sealed bundle")
        if grants and "seal" not in stages:
-            errors.append("grants: any grant requires the 'seal' stage — nothing is served unsealed")
+            errors.append("grants: any grant requires the 'seal' stage")
        if sink is not None and sink.seal_to and "seal" not in stages:
            errors.append("sink: seal_to given but 'seal' is not in stages")
@@ -192,7 +192,7 @@ installers = {n: {"url": "%s/%s" % (base_new, n), "sha256": h(os.path.join(pkg,
              for n in ("install.sh", "install.ps1")}
 json.dump({"version": ver, "built_from": sha, "python_floor": "3.8",
           "files": files, "installers": installers,
-           "notes": "https://git.privatae.ai/privatae/bastion-agent/releases/tag/v%s" % ver},
+           "notes": "https://github.com/Privatae/bastion-agent/releases/tag/v%s" % ver},
          sys.stdout, indent=2)
 PY
@@ -12,8 +12,15 @@ SECOND store:
    <out>/sanitize.summary.json
 From here on, later stages read ONLY the sanitized store. The extract
-store keeps the plaintext and stays on this machine; the vault is the
+store keeps the plaintext. This agent never serves it, but it sits in
-only way back (`/rehydrate` semantics, local).
+<out>, by default <root>/.agent/ inside the folder, where a sync provider
 can reach it. A bastion from 0.58.1 on never serves a folder named .agent
 to a host; an older bastion does to a host whose folder grant has `read`
 unless the grant excludes .agent, and any bastion serves an <out> inside
 the folder under another name.
 The vault is the only way back from a token to a value (`/rehydrate`
 semantics): a file on this machine in-process, the service's own with
 PII_ADDR.
 Two ways to reach the sterilizer, chosen from the environment, never the
 manifest (no addresses in manifests — docs/pii-sterilization.md):
@@ -52,7 +59,7 @@ from .manifest import signing_digest
 # Part of the reuse key: bump it whenever the pipeline would sterilize the
 # same text differently, or a stored result from older code is kept as is.
-SANITIZE_VERSION = 2
+SANITIZE_VERSION = 3
 SANITIZE_CHUNK_CHARS = 20000     # per sterilize call; under the service's 2 MiB body cap
 SERVICE_TIMEOUT = 600.0
@@ -1,9 +1,10 @@
 """bastion agent query endpoint — the container on the mesh.
-The other half of "access moves, data does not". Serves the owner's host
+Serves the owner's host and the manifest's grantees over MUTUAL TLS;
-and the manifest's grantees over MUTUAL TLS; nothing here ever reads the
+nothing here ever reads the extract store, the vault, or the vectors' raw
-extract store, the vault, or the vectors' raw table for a client — the
+table for a client. What it returns is sanitized chunks, catalog cards, the
-only bytes that leave are sanitized chunks and catalog cards.
+run state and health. (The bastion's own `read` op is not this server: it
 reads files in the granted folder as they are on disk.)
 Identity is the presented client certificate:
  - the TLS layer requires a client cert that chains to FREEAGENT_CA_ROOT
@@ -16,7 +17,13 @@ Ops (JSON over HTTPS):
  POST /search      {query, k?}           search  (owner, grantee w/ op)
  POST /read_range  {doc, start, end}     read_range (owner, grantee w/ op)
                                          from the SANITIZED store only,
-                                          <= READ_RANGE_MAX bytes
+                                          <= READ_RANGE_MAX characters
  Both answer with `sanitize_mode`, the mode the sanitize stage recorded for
  the text in the reply ("flag" or "sterilize"; the weakest across a
  search's hits; "unknown" when the record does not vouch for that text; a
  search with no hits names the manifest's mode, or "none"), so a caller can
  say what the text has had done to it: in flag mode names, street addresses
  and dates of birth remain.
  GET  /catalog[?prefix=&offset=&limit=]  owner only: the file cards (paged)
  GET  /catalog/dirs[?...]                owner only: the directory cards (paged)
  GET  /status                            owner only: run.state.json
@@ -65,6 +72,83 @@ class ServeError(Exception):
    pass
 def sanitize_mode(manifest):
    """The sanitize mode the manifest sets: "flag", "sterilize", or "none" when
    it names no sanitize stage. A reply with no text in it (a search with no
    hits) carries this; a reply with text carries served_mode."""
    s = getattr(manifest, "sanitize", None)
    return getattr(s, "mode", None) or "none"
 # Weakest first: a reply that mixes texts names the weakest mode among them.
 _MODE_RANK = {"unknown": 0, "flag": 1, "sterilize": 2}
 _index_lock = threading.Lock()
 _index_cache = {}  # sanitize.jsonl path -> ((ino, mtime_ns, size), {doc: mode})
 def _sanitize_index(out_dir):
    """({doc sha: mode} for every document the last sanitize run wrote with
    status ok, the index's mtime_ns), from <out>/sanitize.jsonl; ({}, None)
    when there is none. Re-read only when the file changes."""
    path = os.path.join(out_dir, "sanitize.jsonl")
    try:
        st = os.stat(path)
    except OSError:
        return {}, None
    sig = (st.st_ino, st.st_mtime_ns, st.st_size)
    with _index_lock:
        hit = _index_cache.get(path)
        if hit and hit[0] == sig:
            return hit[1], st.st_mtime_ns
        modes = {}
        try:
            with open(path, encoding="utf-8") as fh:
                for line in fh:
                    try:
                        rec = json.loads(line)
                    except ValueError:
                        continue
                    if isinstance(rec, dict) and rec.get("status") == "ok" and rec.get("sha256"):
                        modes[rec["sha256"]] = rec.get("mode")
        except OSError:
            return {}, None
        _index_cache[path] = (sig, modes)
        return modes, st.st_mtime_ns
 def served_mode(out_dir, doc, start=None, text=None):
    """The sanitize mode that wrote the text served for doc, as the sanitize
    stage recorded it, or "unknown" when that record does not vouch for the
    text. The manifest's mode says what the NEXT sweep will do; the store can
    still hold an earlier sweep's text (the manifest switched to sterilize and
    the sweep has not run, or stopped; a sweep is rewriting the store now; the
    chunks a search reads were cut before the latest sanitize run).
    Unknown when: no status-ok record for doc, the file is newer than the
    index (a run that has not finished rewrote it), or, for a search hit,
    `text` at `start` is not what the store holds now."""
    modes, index_mtime = _sanitize_index(out_dir)
    mode = modes.get(doc)
    if mode not in ("flag", "sterilize"):
        return "unknown"
    path = sanitized_store_path(out_dir, doc)
    try:
        if os.stat(path).st_mtime_ns > index_mtime:
            return "unknown"
        if text is not None:
            with open(path, encoding="utf-8") as fh:
                stored = fh.read()
            if stored[start:start + len(text)] != text:
                return "unknown"
    except (OSError, TypeError, ValueError):
        return "unknown"
    return mode
 def _weakest(modes):
    return min(modes, key=lambda m: _MODE_RANK.get(m, 0))
 def parse_bind(bind):
    bind = bind.strip()
    if bind.startswith("["):
@@ -312,7 +396,13 @@ class Handler(BaseHTTPRequestHandler):
            except (_embed.EmbedError, OSError) as e:
                self._send(502, {"error": "search unavailable: %s" % e}, cn, "search", {"q": qhash})
                return
-            self._send(200, {"hits": hits, "k": k}, cn, "search", {"q": qhash, "hits": len(hits)})
+            if hits:
                mode = _weakest([served_mode(self.out_dir, h.get("doc"), h.get("start"), h.get("text"))
                                 for h in hits])
            else:
                mode = sanitize_mode(self.policy.manifest)
            self._send(200, {"hits": hits, "k": k, "sanitize_mode": mode},
                       cn, "search", {"q": qhash, "hits": len(hits)})
            return
        if self.path == "/read_range":
            cn, denied = self._authorize("read_range")
@@ -325,7 +415,8 @@ class Handler(BaseHTTPRequestHandler):
                start = int(req.get("start", 0))
                end = int(req["end"])
                if not isinstance(doc, str) or len(doc) != 64 or not all(c in "0123456789abcdef" for c in doc):
-                    raise ValueError("doc must be a sanitized-document sha256")
+                    raise ValueError("doc must be a document id: the original file's sha256, "
                                     "64 lowercase hex")
                if start < 0 or end <= start:
                    raise ValueError("start/end out of order")
                end = min(end, start + READ_RANGE_MAX)
@@ -340,7 +431,8 @@ class Handler(BaseHTTPRequestHandler):
                text = fh.read()
            piece = text[start:end]
            self._send(200, {"doc": doc, "start": start, "end": start + len(piece),
-                             "total": len(text), "text": piece},
+                             "total": len(text), "text": piece,
                             "sanitize_mode": served_mode(self.out_dir, doc)},
                       cn, "read_range", {"doc": doc[:12], "chars": len(piece)})
            return
        self._send(404, {"error": "not found"}, self._cn(), None)
@@ -189,7 +189,8 @@ class StageTests(Case):
    def test_catalog_carries_no_values(self):
        _, files, dirs = self.run_stage(manifest(self.root))
        blob = json.dumps(files) + json.dumps(dirs)
-        for leak in ("m@example.com", "4111"):
+        # the whole card: four digits alone also turn up inside hashes and times
        for leak in ("m@example.com", "4111 1111 1111 1111", "4111111111111111"):
            self.assertNotIn(leak, blob)
    def test_summaries_file_and_directory_with_cache(self):
@@ -205,7 +206,8 @@ class StageTests(Case):
        # the model only ever saw sanitized text
        for _, _, prompt in _StubChat.calls:
            self.assertNotIn("m@example.com", prompt)
-            self.assertNotIn("4111", prompt)
+            for card in ("4111 1111 1111 1111", "4111111111111111"):
                self.assertNotIn(card, prompt)
        # second run: all summaries from cache, no calls
        _StubChat.calls = []
        s2, _, _ = self.run_stage(m)
@@ -123,12 +123,18 @@ class DoctrineTests(unittest.TestCase):
    def test_host_catalog_needs_seal(self):
        doc = with_(FULL, stages=["walk", "extract", "sanitize", "chunk", "embed", "catalog"],
                    grants=None, sink={"catalog": "host", "vectors": "local", "plaintext": "never"})
-        self.assertTrue(any("unsealed" in e for e in errors_of(doc)))
+        errs = errors_of(doc)
        self.assertTrue(any("sink: catalog 'host' requires the 'seal' stage" in e for e in errs))
        # serve's replies are never sealed, so the message must not claim that
        # nothing leaves, or is served, unsealed
        self.assertFalse(any("unsealed" in e for e in errs), errs)
    def test_grant_needs_seal(self):
        doc = with_(FULL, stages=["walk", "extract", "sanitize", "chunk", "embed", "catalog"],
                    sink={"catalog": "local", "vectors": "local", "plaintext": "never"})
-        self.assertTrue(any("grants: any grant requires the 'seal'" in e for e in errors_of(doc)))
+        errs = errors_of(doc)
        self.assertTrue(any("grants: any grant requires the 'seal'" in e for e in errs))
        self.assertFalse(any("unsealed" in e for e in errs), errs)
    def test_grant_search_needs_mask_search(self):
        doc = with_(FULL, mask=["read", "index", "summarize"])
@@ -12,7 +12,6 @@ import shutil
 import subprocess
 import tempfile
 import unittest
 from unittest import mock
 import yaml
@@ -104,7 +104,8 @@ class FlagModeTests(Case):
        s, idx = self.run_stage(manifest(self.root, "flag"))
        out = self.text_of(idx["note.txt"])
        self.assertNotIn("m@example.com", out)
-        self.assertNotIn("4111", out)
+        for card in ("4111 1111 1111 1111", "4111111111111111"):
            self.assertNotIn(card, out)
        self.assertIn("Marisol", out)  # contextual PII is NOT caught by flag mode
        self.assertIn("[PII_EMAIL_1]", out)
        self.assertIn("[PII_CREDIT_CARD_1]", out)
@@ -121,7 +122,9 @@ class FlagModeTests(Case):
        s, idx = self.run_stage(manifest(self.root, "flag"))
        blob = json.dumps(idx) + json.dumps(s)
        self.assertNotIn("m@example.com", blob)
-        self.assertNotIn("4111", blob)
+        # the whole card: four digits alone also turn up inside hashes and times
        for card in ("4111 1111 1111 1111", "4111111111111111"):
            self.assertNotIn(card, blob)
    @unittest.skipIf(IS_WINDOWS, "POSIX permission bits")
    def test_vault_is_private(self):
@@ -153,6 +156,19 @@ class FlagModeTests(Case):
        self.assertEqual(s2["counts"]["ok"], s1["counts"]["ok"])
        self.assertFalse(idx["note.txt"].get("reused", False))
    def test_text_kept_by_the_older_rules_is_redone(self):
        # version 2 kept an international number and a lowercase IBAN in the
        # clear; a store it wrote must not be reused by the wider rules
        put(self.root, "intl.txt", "Mobile +44 7700 900123, iban gb82west12345698765432\n")
        with mock.patch.object(sz, "SANITIZE_VERSION", 2):
            self.run_stage(manifest(self.root, "flag"))
        s, idx = self.run_stage(manifest(self.root, "flag"))
        self.assertEqual(s["counts"]["reused"], 0)
        out = self.text_of(idx["intl.txt"])
        self.assertNotIn("7700 900123", out)
        self.assertNotIn("gb82west", out)
        self.assertEqual(idx["intl.txt"]["types"], {"PHONE": 1, "BANK_ACCOUNT": 1})
    def test_tokens_stable_across_files_in_scope(self):
        put(self.root, "again.txt", "ping m@example.com again\n")
        _, idx = self.run_stage(manifest(self.root, "flag"))
@@ -112,9 +112,12 @@ class FailClosedTests(Case):
        for forbidden in ("walk.jsonl", "extract.jsonl", "sanitize.jsonl", "vectors.db", "vault.db"):
            self.assertFalse(any(n.endswith(forbidden) for n in names), forbidden)
        self.assertFalse(any(n.startswith(("extract/", "sanitize/")) for n in names))
-        # nothing in the bundle is a value
+        # nothing in the bundle is a value. The card is looked for whole, spaced
        # or not: four of its digits also turn up by chance inside the bundle's
        # sha256 hex (CI run 2483 failed on "...81934111").
        self.assertNotIn(b"m@example.com", plain)
-        self.assertNotIn(b"4111", plain)
+        self.assertNotIn(b"4111 1111 1111 1111", plain)
        self.assertNotIn(b"4111111111111111", plain)
        self.assertIn(b"[PII_EMAIL_1]", plain)  # sanitized head is there, as designed
        self.assertEqual(set(inv["members"]), set(names))
@@ -175,7 +178,7 @@ class RoundTripTests(Case):
    def test_tampered_member_refused(self):
        ident, pub = keygen(self.out)
        env = {"FREEAGENT_OWNER_RECIPIENT": pub}
-        s = sl.seal(manifest(self.root), self.out, env=env)
+        sl.seal(manifest(self.root), self.out, env=env)
        # rebuild a bundle with a modified catalog but the old inventory, seal it, and unseal
        plain, _ = sl.build_bundle(manifest(self.root), self.out, "x")
        buf = io.BytesIO()
@@ -22,7 +22,6 @@ from . import run as rn
 from . import serve as sv
 from .catalog import catalog
 from .extract import extract
 from .manifest import validate_document
 from .sanitize import sanitize
 from .test_embed import _Stub as EmbedStub  # deterministic bag-of-words embedder
 from .walk import walk
@@ -112,12 +111,12 @@ class ServeTests(unittest.TestCase):
        shutil.rmtree(cls.work, ignore_errors=True)
    @classmethod
-    def write_manifest(cls, grants):
+    def write_manifest(cls, grants, mode="flag"):
        import yaml
        doc = {"freeagent": 1, "id": "t", "owner": {"host": "owner-host"},
               "scope": {"roots": [cls.root]}, "mask": ["read", "search", "index"],
               "stages": ["walk", "extract", "sanitize", "chunk", "embed", "catalog", "seal"],
-               "sanitize": {"mode": "flag"},
+               "sanitize": {"mode": mode},
               "embed": {"model": "stub-model", "dim": 64, "chunk": {"size_tokens": 8, "overlap_tokens": 2}},
               "sweep": "manual",
               "sink": {"catalog": "host", "vectors": "local", "plaintext": "never"},
@@ -234,10 +233,100 @@ class ServeTests(unittest.TestCase):
            self.assertIn("m@example.com", fh.read())  # the plaintext exists on disk; it has no route
        st, _ = self.call("owner", "POST", "/read_range", {"doc": "0" * 64, "start": 0, "end": 10})
        self.assertEqual(st, 404)
-        self.assertEqual(self.call("owner", "POST", "/read_range", {"doc": "zz", "start": 0, "end": 1})[0], 400)
+        st, body = self.call("owner", "POST", "/read_range", {"doc": "zz", "start": 0, "end": 1})
        self.assertEqual(st, 400)
        # the id is the original file's hash (the extract and sanitized stores share it)
        self.assertIn("original file's sha256", body["error"])
        self.assertEqual(self.call("owner", "POST", "/search", {"query": ""})[0], 400)
        self.assertEqual(self.call("owner", "POST", "/nope", {})[0], 404)
    GRANTS = [{"host": "partner-host", "ops": ["search"], "until": "2026-12-31"},
              {"host": "old-host", "ops": ["search", "read_range"], "until": "2026-01-01"}]
    def modes(self):
        """(owner search, owner read_range, partner search) sanitize_mode for the apples doc."""
        st, body = self.call("owner", "POST", "/search", {"query": "apples", "k": 1})
        self.assertEqual(st, 200)
        doc = body["hits"][0]["doc"]
        st, rr = self.call("owner", "POST", "/read_range", {"doc": doc, "start": 0, "end": 10})
        self.assertEqual(st, 200)
        st, pb = self.call("partner", "POST", "/search", {"query": "apples", "k": 1})
        self.assertEqual(st, 200)
        return doc, (body["sanitize_mode"], rr["sanitize_mode"], pb["sanitize_mode"])
    def rewrite_index(self, fn):
        """Rewrite sanitize.jsonl through fn(record); returns the original bytes."""
        path = os.path.join(self.out, "sanitize.jsonl")
        with open(path, "rb") as fh:
            orig = fh.read()
        recs = [fn(json.loads(l)) for l in orig.decode("utf-8").splitlines() if l.strip()]
        with open(path, "w", encoding="utf-8") as fh:
            fh.write("".join(json.dumps(r, sort_keys=True) + "\n" for r in recs))
        return orig
    def restore_index(self, orig):
        with open(os.path.join(self.out, "sanitize.jsonl"), "wb") as fh:
            fh.write(orig)
    def test_a_reply_names_the_mode_that_wrote_its_text_not_the_manifests(self):
        # The store was swept in flag mode. Switching the manifest to
        # sterilize changes what the NEXT sweep does, not the text already
        # served: until a sweep rewrites it, every reply still says flag, so
        # the bastion never calls flag-mode text sanitized.
        try:
            for mode in ("flag", "sterilize", "flag"):
                self.write_manifest(grants=self.GRANTS, mode=mode)
                self.assertEqual(self.modes()[1], ("flag", "flag", "flag"), mode)
        finally:
            self.write_manifest(grants=self.GRANTS)
    def test_a_store_the_sanitize_stage_wrote_in_sterilize_mode_is_named_sterilize(self):
        orig = self.rewrite_index(lambda r: dict(r, mode="sterilize"))
        try:
            self.write_manifest(grants=self.GRANTS, mode="sterilize")
            self.assertEqual(self.modes()[1], ("sterilize", "sterilize", "sterilize"))
        finally:
            self.restore_index(orig)
            self.write_manifest(grants=self.GRANTS)
    def test_text_the_sanitize_record_does_not_vouch_for_is_unknown(self):
        doc, modes = self.modes()
        self.assertEqual(modes, ("flag", "flag", "flag"))
        spath = sv.sanitized_store_path(self.out, doc)
        st = os.stat(spath)
        index = os.path.join(self.out, "sanitize.jsonl")
        # a sweep still running has rewritten the file after the index
        later = os.stat(index).st_mtime_ns + 10 ** 9
        os.utime(spath, ns=(later, later))
        try:
            self.assertEqual(self.modes()[1], ("unknown", "unknown", "unknown"))
        finally:
            os.utime(spath, ns=(st.st_atime_ns, st.st_mtime_ns))
        # the chunks a search reads were cut from text the store no longer holds
        with open(spath, encoding="utf-8") as fh:
            text = fh.read()
        with open(spath, "w", encoding="utf-8") as fh:
            fh.write("X" * len(text))
        os.utime(spath, ns=(st.st_atime_ns, st.st_mtime_ns))
        try:
            got = self.modes()[1]
            self.assertEqual((got[0], got[2]), ("unknown", "unknown"))
        finally:
            with open(spath, "w", encoding="utf-8") as fh:
                fh.write(text)
            os.utime(spath, ns=(st.st_atime_ns, st.st_mtime_ns))
        # the last run did not write it (stopped: not_run), so the file is an older run's
        orig = self.rewrite_index(lambda r: dict(r, status="not_run") if r.get("sha256") == doc else r)
        try:
            self.assertEqual(self.modes()[1], ("unknown", "unknown", "unknown"))
        finally:
            self.restore_index(orig)
        self.assertEqual(self.modes()[1], ("flag", "flag", "flag"))
    def test_sanitize_mode_of_a_manifest_without_the_stage_is_none(self):
        self.assertEqual(sv.sanitize_mode(None), "none")
        self.assertEqual(sv.sanitize_mode(type("M", (), {"sanitize": None})()), "none")
    def test_silent_tcp_connection_does_not_stall_other_clients(self):
        # with the handshake on the accept loop one idle raw TCP
        # connection blocked every client; now the handshake runs in the
@@ -333,7 +422,7 @@ class UnitTests(unittest.TestCase):
    def test_parse_bind(self):
        self.assertEqual(sv.parse_bind("[::]:8470"), ("::", 8470))
        self.assertEqual(sv.parse_bind("10.0.0.5:9"), ("10.0.0.5", 9))
-        self.assertEqual(sv.parse_bind("[fd00:1::31]:8470"), ("fd00:1::31", 8470))
+        self.assertEqual(sv.parse_bind("[2001:db8::31]:8470"), ("2001:db8::31", 8470))
        with self.assertRaises(sv.ServeError):
            sv.parse_bind("nope")
@@ -169,16 +169,26 @@ class EndToEndTests(unittest.TestCase):
                           "files": {name: {"sha256": digest or self.digest},
                                     name + ".sha256": {"sha256": "0" * 64}}}).encode()
-    def verify(self, body, seed=None):
+    def verify(self, body, seed=None, args=()):
        msg = vr.statement("bastion-agent", self.version, hashlib.sha256(body).hexdigest())
-        sig = base64.b64encode(_sign(seed or self.SEED, msg))
+        sig = base64.b64encode(_sign(seed or self.SEED, msg)) + b"\n"
        served = {"/version.json": body, "/" + vr.sig_name(body): sig}
        self.served = served
        out, err = io.StringIO(), io.StringIO()
        with mock.patch.object(vr, "fetch", lambda url: served[url[len("https://test"):]]), \
                contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
-            code = vr.main(["--base", "https://test"], repo=self.repo, keys=(self.pub,))
+            code = vr.main(["--base", "https://test"] + list(args), repo=self.repo, keys=(self.pub,))
        return code, out.getvalue() + err.getvalue()
    def release_tag(self, built_from, annotated=True):
        tag = "v" + self.version
        if annotated:
            self.run_git("tag", "-a", tag, "-m", "bastion agent %s\n\nbuilt from %s" % (self.version, built_from))
        else:
            self.run_git("tag", tag)
        self.addCleanup(subprocess.run, ("git", "-C", self.repo, "tag", "-d", tag),
                        stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
    def test_tag_first_then_built_from(self):
        code, out = self.verify(self.manifest("0000000"))
        self.assertEqual(code, 1, out)
@@ -189,13 +199,57 @@ class EndToEndTests(unittest.TestCase):
        self.assertIn("source:    built_from", out)
        self.assertIn("matches %s" % self.digest, out)
-        self.run_git("tag", "v" + self.version)
+        self.release_tag("0000000")
        try:
        code, out = self.verify(self.manifest("0000000"))
        self.assertEqual(code, 0, out)
        self.assertIn("source:    tag v%s" % self.version, out)
-        finally:
+
    def test_a_tag_without_the_release_record_is_refused(self):
        # A lightweight tag, or one recording another build, says nothing
        # about which release it is: refused, not passed over for built_from.
        self.release_tag("0000000", annotated=False)
        code, out = self.verify(self.manifest(self.commit[:7]))
        self.assertEqual(code, 1, out)
        self.assertIn("does not carry the release record", out)
        self.assertNotIn("rebuild:", out)
        self.run_git("tag", "-d", "v" + self.version)
        self.release_tag("1111111")
        code, out = self.verify(self.manifest(self.commit[:7]))
        self.assertEqual(code, 1, out)
        self.assertIn("built from %s" % self.commit[:7], out)
    def test_another_version_than_expected_is_refused_before_any_checkout(self):
        code, out = self.verify(self.manifest(self.commit[:7]), args=["--expect-version", "v9.9.9"])
        self.assertEqual(code, 1, out)
        self.assertIn("names v%s, not v9.9.9" % self.version, out)
        self.assertNotIn("source:", out)
        code, out = self.verify(self.manifest(self.commit[:7]), args=["--expect-version", "v" + self.version])
        self.assertEqual(code, 0, out)
    def test_out_writes_what_a_release_carries(self):
        name = "bastion-agent-%s.pyz" % self.version
        sums = ("%s  %s\n" % (self.digest, name)).encode()
        out_dir = os.path.join(self.tmp, "release-out")
        self.addCleanup(shutil.rmtree, out_dir, True)
        body = json.dumps({"version": self.version, "built_from": self.commit[:7],
                           "files": {name: {"sha256": self.digest},
                                     name + ".sha256": {"sha256": hashlib.sha256(sums).hexdigest()}}}).encode()
        code, out = self.verify(body, args=["--out", out_dir])
        self.assertEqual(code, 0, out)
        sig = vr.sig_name(body)
        self.assertEqual(sorted(os.listdir(out_dir)), sorted([name, name + ".sha256", "version.json", sig]))
        with open(os.path.join(out_dir, "version.json"), "rb") as fh:
            self.assertEqual(fh.read(), body)
        with open(os.path.join(out_dir, sig), "rb") as fh:
            self.assertEqual(fh.read(), self.served["/" + sig])
        with open(os.path.join(out_dir, name), "rb") as fh:
            self.assertEqual(hashlib.sha256(fh.read()).hexdigest(), self.digest)
    def test_out_writes_nothing_when_the_rebuild_differs(self):
        out_dir = os.path.join(self.tmp, "release-out-bad")
        code, out = self.verify(self.manifest(self.commit[:7], digest="1" * 64), args=["--out", out_dir])
        self.assertEqual(code, 1, out)
        self.assertFalse(os.path.exists(out_dir))
    def test_a_different_build_is_reported(self):
        code, out = self.verify(self.manifest(self.commit, digest="1" * 64))
@@ -10,13 +10,12 @@ import hashlib
 import json
 import os
 import shutil
 import sys
 import tempfile
 import unittest
 from unittest import mock
 from .manifest import validate_document
-from .walk import Excluder, Walker, _glob_to_regex, load_hash_cache, walk
+from .walk import Excluder, _glob_to_regex, load_hash_cache, walk
 IS_WINDOWS = os.name == "nt"
@@ -15,9 +15,15 @@ and the signature says the release key holder published it.
 The source is the tag v<version>. The public repository is a snapshot of the
 development one, so the commit the manifest names (built_from) exists only
-there; the tag marks the same tree in the public history. When the tag is
+there; the tag marks the same tree in the public history, and it counts only
-absent and built_from is present (a clone of the development repository),
+as an annotated tag whose message has the line `built from <built_from>`. When
-built_from is used instead.
+the tag is absent and built_from is present (a clone of the development
 repository), built_from is used instead.
 `--expect-version` refuses a manifest that names another version, and `--out
 DIR` writes what a release carries once everything matched: the rebuilt .pyz
 and its .sha256, the manifest's exact bytes and its signature file. The
 GitHub release workflow (.github/workflows/release.yml) runs it that way.
 The signature check is ed25519 in pure Python, because the standard library
 has no ed25519 and this has to run where nothing else is installed. It only
@@ -228,15 +234,33 @@ def _commit(repo, ref):
    return r.stdout.strip() if r.returncode == 0 else ""
 def tag_records(repo, tag, built_from):
    """True when tag is an annotated tag whose message has the line
    `built from <built_from>`, the record publishing writes on every release
    tag. A tag without it says nothing about which release it is."""
    if not built_from:
        return False
    if _git(repo, "cat-file", "-t", "refs/tags/" + tag, check=False).stdout.strip() != "tag":
        return False
    msg = _git(repo, "tag", "-l", "--format=%(contents)", tag, check=False).stdout
    return ("built from " + built_from) in msg.splitlines()
 def pick_source(repo, version, built_from):
    """(label, commit) of the source to rebuild: the tag v<version>, else the
-    manifest's built_from, else VerifyError."""
+    manifest's built_from, else VerifyError. The tag counts only when it
    records the manifest's built_from (tag_records); one that does not is
    refused rather than passed over."""
    tag = "v" + version
    # Fetch the tag if the clone lacks it; a failure (offline, no such tag)
    # just leaves the local refs to decide.
    _git(repo, "fetch", "-q", "origin", "refs/tags/%s:refs/tags/%s" % (tag, tag), check=False)
    c = _commit(repo, "refs/tags/" + tag)
    if c:
        if not tag_records(repo, tag, built_from):
            raise VerifyError("the tag %s does not carry the release record (an annotated tag "
                              "with the line 'built from %s'); nothing was built"
                              % (tag, built_from or "(none named)"))
        return "tag " + tag, c
    if built_from:
        c = _commit(repo, built_from)
@@ -249,7 +273,8 @@ def pick_source(repo, version, built_from):
 def rebuild(repo, commit, version, work):
-    """sha256 of the .pyz that commit's own build.py produces."""
+    """sha256 of the .pyz that commit's own build.py produces. The build's
    output stays in <work>/dist."""
    src = os.path.join(work, "src")
    _git(repo, "worktree", "add", "-q", "--detach", src, commit)
    try:
@@ -269,20 +294,52 @@ def rebuild(repo, commit, version, work):
        _git(repo, "worktree", "remove", "--force", src, check=False)
 def write_release(out, dist, version, files, body, sig):
    """The files a release carries, into out: the rebuilt .pyz and its .sha256
    (each checked against the manifest's digest for that name), the manifest
    as it was served, byte for byte, and its signature under the name it is
    served by. Called only after the signature and the rebuild both passed."""
    os.makedirs(out, exist_ok=True)
    for name in ("bastion-agent-%s.pyz" % version, "bastion-agent-%s.pyz.sha256" % version):
        with open(os.path.join(dist, name), "rb") as fh:
            data = fh.read()
        want = (files.get(name) or {}).get("sha256")
        if hashlib.sha256(data).hexdigest() != want:
            raise VerifyError("the rebuilt %s does not match the manifest's sha256 for it" % name)
        with open(os.path.join(out, name), "wb") as fh:
            fh.write(data)
    with open(os.path.join(out, "version.json"), "wb") as fh:
        fh.write(body)
    with open(os.path.join(out, sig_name(body)), "wb") as fh:
        fh.write(sig)
    print("release:   wrote %s" % ", ".join(sorted(os.listdir(out))))
 def main(argv=None, repo=REPO, keys=None):
    ap = argparse.ArgumentParser(prog="bastion_agent.verify_release",
                                 description="Rebuild the published agent and compare it with the release manifest.")
    ap.add_argument("--base", default=os.environ.get("MANIFEST_BASE", DEFAULT_BASE),
                    help="where version.json is served (default %s)" % DEFAULT_BASE)
    ap.add_argument("--expect-version", metavar="VERSION",
                    help="refuse unless the published manifest names this version (v0.1.16 or 0.1.16)")
    ap.add_argument("--out", metavar="DIR",
                    help="when everything matches, write the rebuilt .pyz and its .sha256, the "
                         "manifest's exact bytes and its signature file into DIR")
    args = ap.parse_args(argv)
    base = args.base.rstrip("/")
    work = tempfile.mkdtemp(prefix="agent-verify-")
    try:
        body = fetch(base + "/version.json")
-        sig = fetch(base + "/" + sig_name(body)).decode("ascii", "replace")
+        sig_raw = fetch(base + "/" + sig_name(body))
        sig = sig_raw.decode("ascii", "replace")
        m, key = check_signature(body, sig, keys)
        ver, built_from = m["version"], m.get("built_from") or ""
        print("published: v%s built from %s" % (ver, built_from or "(not named)"))
        want = (args.expect_version or "").strip()
        if want.startswith("v"):
            want = want[1:]
        if want and want != ver:
            raise VerifyError("the published manifest names v%s, not v%s" % (ver, want))
        print("signature: ok (release key %d of %d, %s)"
              % (key + 1, len(RELEASE_KEYS if keys is None else keys), "primary" if key == 0 else "spare"))
        files = m.get("files") if isinstance(m.get("files"), dict) else {}
@@ -295,12 +352,16 @@ def main(argv=None, repo=REPO, keys=None):
        print("source:    %s (%s)" % (label, commit[:12]))
        got = rebuild(repo, commit, ver, work)
        bad = 0
-        for name, want in pyz.items():
+        for name, digest in pyz.items():
-            same = got == want
+            same = got == digest
            bad += not same
            print("rebuild:   %-28s %s" % (name, "matches %s" % got if same
-                                             else "DIFFERS (rebuilt %s, published %s)" % (got, want)))
+                                             else "DIFFERS (rebuilt %s, published %s)" % (got, digest)))
-        return 1 if bad else 0
+        if bad:
            return 1
        if args.out:
            write_release(args.out, os.path.join(work, "dist"), ver, files, body, sig_raw)
        return 0
    except VerifyError as exc:
        print("verify: %s" % exc, file=sys.stderr)
        return 1
@@ -25,7 +25,6 @@ import json
 import os
 import platform
 import re
 import sys
 import tempfile
 from .manifest import AGENT_DIR, signing_digest
@@ -57,7 +57,12 @@ MIME types), `keywords` (the top 12 over the merged term counts), `entities`
  hyphens, so `invoice-m@example.com.pdf` becomes `[PII_EMAIL]`; `mime` and
  `size` still say what it was. The summary counts masked segments as
  `names_masked`. **A name like `Marisol_medical.pdf` is not caught**: that is
-  contextual personal data, the same limit as the rules-only `flag` mode.
+  contextual personal data, the same limit as the rules-only `flag` mode. The
  mask is the catalog's: the bastion's own listing, for a grant with `read`,
  returns names as they are.
 - **Not hidden: content hashes.** Each file card's `sha256`, also the `doc` id in
  search hits and excerpts, is the hash of the original file, so whoever holds
  a copy of a file can confirm it is in the folder.
 ## The summarizer (a sensor)
@@ -1,7 +1,8 @@
 # bastion agent `chunk` and `embed` stages, and `search`
 `chunk` splits the sanitized text into overlapping windows, `embed` turns each
-window into a vector with a model on this machine, and `search` answers a
+window into a vector with the model server `EMBED_URL` names (by default the
 Ollama port on the same machine), and `search` answers a
 query against those vectors. They are `bastion_agent/embed.py`, standard
 library only.
@@ -9,9 +10,10 @@ library only.
 `python3 -m bastion_agent chunk <manifest> [--out DIR] [--require-signed]`
 reads the **sanitized** store (never the extract store) and writes
-`chunks.jsonl`: one row per window (`chunk_id`, `doc` (the sanitized text's
+`chunks.jsonl`: one row per window (`chunk_id`, `doc` (the original file's
-sha256), `idx`, `start` and `end` character offsets, `chars`, `tokens`, `v`),
+sha256, the id the extract and sanitized stores share), `idx`, `start` and
-sorted by `(doc, idx)`, plus `chunk.summary.json`.
+`end` character offsets, `chars`, `tokens`, `v`), sorted by `(doc, idx)`, plus
 `chunk.summary.json`.
 - Windows are `embed.chunk.size_tokens` long and advance by
  `size - overlap` tokens; the last window may be shorter. With any overlap,
@@ -27,8 +29,8 @@ sorted by `(doc, idx)`, plus `chunk.summary.json`.
  next. A changed document has a new sha256, so all of its windows get new ids
  and are embedded again. Identical windows inside one document (a repeated
  paragraph) share an id and are embedded once.
- Duplicate documents (the same sanitized content at several paths) are
+- Duplicate documents (the same file content at several paths) are chunked
-  chunked once.
+  once.
 ## embed
@@ -25,8 +25,12 @@ the stage refuses otherwise) and writes:
  recorded, which ties the walk map and the text store to one manifest.
 `DIR` defaults as for the walk (`<root>/.agent/` for one root). This text
-store is the plaintext of every file in scope, and it never leaves the machine;
+store is the plaintext of every file in scope. The agent never serves it, but
-see [runtime.md](runtime.md#where-an-agent-lives-rootagent).
+by default it sits in the folder, so if the folder is synced, the sync provider
 gets a copy. A bastion from 0.58.1 on never serves `.agent/` to a host. On an
 older bastion, a host whose folder grant has `read` can read the store like
 any other file unless the grant excludes `.agent`; see
 [runtime.md](runtime.md#where-an-agent-lives-rootagent).
 | `status` | Meaning |
 |---|---|
@@ -19,19 +19,21 @@ the control flow and every change, and a model proposes, never acts.
 The owner's host writes a **job manifest**, and it is signed (by the bastion,
 in the managed install). The agent executes exactly that manifest and nothing
 else. The output is a **catalog** (small, structured cards per file and per
-directory) plus local vectors. The files themselves never move; a host
+directory) plus local vectors. The agent never sends the files themselves; a
-reasons over them by querying the agent. Sharing is a grant that lets another
+host reasons over them by querying the agent. Sharing is a grant that lets
-host query this catalog: access moves, data does not.
+another host search the sanitized text or read ranges of it, until a set date
 ([`grants`](#grants)).
 ## Rules the schema enforces
-1. **Plaintext never leaves.** `sink.plaintext` is the constant `never`. Any
+1. **No plaintext sink.** `sink.plaintext` is the constant `never`. Any
   other value is a validation error, not a setting.
 2. **Nothing runs that was not granted.** `mask` is the ceiling; every stage
   must be covered by an op in the mask, or the manifest is refused before it
   runs.
-3. **Nothing leaves unsealed.** A catalog sent to the host, `seal_to`
+3. **Sharing needs `seal`.** `sink.catalog: host`, `seal_to` recipients, or
-   recipients, or any grant requires the `seal` stage.
+   any grant requires the `seal` stage. What `serve` returns, to the owner or
   to a grantee, is never sealed.
 4. **No addresses.** The sanitizer and the embedder are found from the
   machine's environment when the agent runs. The manifest never carries a
   host:port.
@@ -179,11 +181,14 @@ Present **if and only if** `embed` is in `stages`.
 | Field | Required | Rules |
 |---|---|---|
 | `model` | yes | Model name, 1–128 characters. |
-| `dim` | yes | Integer 1–8192. The stage refuses to run if the local endpoint's vectors have another dimension. Vectors are stored per model, so a later model change never discards existing vectors. |
+| `dim` | yes | Integer 1–8192. The stage refuses to write if the embedding server (`EMBED_URL`) returns vectors of another dimension. Vectors are stored per model, so a later model change never discards existing vectors. |
 | `chunk.size_tokens` | yes | Integer 1–8192. |
 | `chunk.overlap_tokens` | yes | Integer, 0 ≤ overlap < size. |
-There is no endpoint field. Embedding is local, always.
+There is no endpoint field: the embedder is the server `EMBED_URL` names in
 the agent's environment, by default the Ollama port on the same machine. The
 bastion sets it to `127.0.0.1:11434`; an agent run by hand can be pointed
 anywhere.
 ### `summarize`
@@ -191,7 +196,7 @@ Optional. Requires `summarize` in `mask` and `catalog` in `stages`.
 | Field | Required | Rules |
 |---|---|---|
-| `model` | yes | Local model name, 1–128 characters. |
+| `model` | yes | Model name, 1–128 characters, at the server `SUMMARIZE_URL` names. |
 | `per` | yes | Non-empty subset of `file`, `directory`. |
 ### `sweep`
@@ -202,7 +207,7 @@ One of `15m`, `hourly`, `daily`, `manual`.
 | Field | Required | Rules |
 |---|---|---|
-| `catalog` | yes | `local`: the sealed catalog stays here, and the host queries it. `host`: the sealed catalog is sent to the owner's host, so it can plan while this machine is asleep. Requires the `seal` stage. |
+| `catalog` | yes | `local`: the catalog stays here, and the host queries it through `serve`. `host`: the sealed catalog is meant for the owner's host. The agent writes it to `<out>/sealed/` and neither the agent nor the bastion sends it. In the default `<out>`, `<root>/.agent/`, a bastion from 0.58.1 on never serves it to a host; on an older bastion, a host whose folder grant has `read` can fetch it like any other file unless the grant excludes `.agent`. When `<out>` is inside the folder under any other name, any bastion serves it to such a host like any other file. Requires the `seal` stage. |
 | `vectors` | yes | `local`, the only value. |
 | `plaintext` | yes | `never`, the only value, ever. It is there so the rule is written down in every manifest and can be searched for, not remembered. |
 | `seal_to` | no | List of recipient keys, 1–128 characters each, no duplicates. Requires the `seal` stage. The owner's key is always added by the agent and need not be listed. |
@@ -111,7 +111,7 @@ run there, and the second, with `SIGNATURE=<the line it printed>` and
 `BASTION_TREE` pointing at a privatae-bastion checkout, checks the signature
 against the keys that checkout pins and the keys the published bastion pins,
 and only then publishes. What is signed is a short statement naming the kind
-of release, its version and the manifest's sha256. A bastion from 0.41.0 on
+of release, its version and the manifest's sha256. A bastion from 0.42.0 on
 fetches `version.json.<h>.sig`, refuses a manifest that does not verify, and
 never replaces an installed agent with an older version.
@@ -122,9 +122,12 @@ checks each edge's bytes against the manifest it serves. Where it publishes
 is not written into the script: the edges, the ssh key, the web root and the
 backup directory come from its environment, and it stops if one is missing.
-The manifest's `notes` link to the tag `v<version>` on the public repository,
+The manifest's `notes` link to the GitHub release of the tag `v<version>`,
-which marks the source of that release. `publish.sh` does not make the tag; it
+which marks the source of that release and carries its files. `publish.sh`
-is made when that release's source is published to the public repository.
+does not make the tag; it is made when that release's source is published to
 the public repository, and pushing it starts `.github/workflows/release.yml`,
 which rebuilds the `.pyz` from the tag and attaches it, with its `.sha256`, the
 signed manifest and its signature, once its sha256 matches the manifest.
 ### The legacy name
@@ -177,7 +180,10 @@ answering `--version` and `doctor --json` and validating the example manifest;
 heads; the `.agent` default output directory; and the doctor's checks against
 unreachable and missing inputs.
-This repository has no CI. [CONTRIBUTING.md](../CONTRIBUTING.md) lists the
+Every push to `main` and every pull request into it runs CI
 (`.gitea/workflows/ci.yml`, `.github/workflows/ci.yml`): the 3.8 floor,
 pyflakes, every test on Python 3.8 and a current Python, a personal-data scan
 and a reproducibility check. [CONTRIBUTING.md](../CONTRIBUTING.md) lists the
 checks to run before proposing a change, and `publish.sh` builds the release
 and runs it before anything is uploaded. The installers are not exercised by
 the test suite.
@@ -15,7 +15,9 @@ personal data is replaced with stable placeholder tokens, the real values stay
 in a local vault, and `/rehydrate` puts them back, locally, on demand. A cloud
 model can then work over the text without the values in it. Patterns alone
 cannot do this: "my daughter goes to Lincoln Elementary" contains no pattern,
-only meaning.
+only meaning. The agent the bastion deploys runs the rules pass only (`flag`
 mode), so names, street addresses and dates of birth stay in the text it
 returns ([sanitize.md](sanitize.md#modes-manifest-sanitizemode)).
 ## Pipeline (`pii/pipeline.py`)
@@ -116,7 +118,7 @@ Configuration, from the environment:
 | Variable | Default | Meaning |
 |---|---|---|
 | `PII_API_KEY` | (none) | Required. Without it the service refuses to start: `/rehydrate` returns real values, so there is no unauthenticated mode. (`PII_ALLOW_NOAUTH=1` exists for the unit tests only.) |
-| `PII_BIND` | `0.0.0.0:8091` | The address the service listens on; this is its exposure control. |
+| `PII_BIND` | `127.0.0.1:8091` | The address the service listens on; this is its exposure control. Another address only when set. |
 | `PII_DB` | `/data/vault.db` | The vault. |
 | `PII_RECALL` | `llm` | The recall backend (below). |
 | `LLM_URL` | `http://llm:8080` | The model server for `llm`. |
@@ -125,7 +127,9 @@ Configuration, from the environment:
 Two ways to run it, the same code:
 - **Container**: `pii/Dockerfile` (a pinned `python:*-slim` image, no pip
-  install; the vault on a mounted `/data`).
+  install; the vault on a mounted `/data`, writable by uid 10001, the
  unprivileged user it runs as). Inside a container the default loopback bind
  reaches nothing else, so set `PII_BIND` and publish the port deliberately.
 - **systemd**: `pii/pii.service`, with install steps in its header. It runs
  with `DynamicUser`, a private 0700 state directory for the vault
  (`/var/lib/privatae-pii/vault.db`), loopback bind (`127.0.0.1:8091`), and
@@ -167,9 +171,18 @@ the machine.
 Every detector is anchored by a checksum, a prefix or a fixed shape; none
 depends on a model.
- **E-mail, phone, SSN, IPv4/IPv6, MAC.**
+- **E-mail, IPv4/IPv6, MAC.** A dotted number right after a version word
- **Cards** → `CREDIT_CARD`, checked with Luhn.
+  (`version 1.2.3.4`, `v1.2.3.4`) is not an IP address.
- **IBAN** → `BANK_ACCOUNT`: ISO 13616 mod-97 plus a per-country length
+- **Phone**: North American and other grouped national forms; international
  numbers with `+` and a country code, any grouping, an optional `(0)` trunk
  prefix; national numbers with a trunk `0` (`07700 900123`); a seven-digit
  local number (`555-0142`) only after a word such as call, phone or tel.
 - **SSN**: dashed or spaced; nine bare digits only after `SSN` or `social
  security` within 40 characters. Numbers never issued (area 000, 666, 900+;
  group 00; serial 0000) are left alone.
 - **Cards** → `CREDIT_CARD`, with spaces, dots or dashes, checked with Luhn.
  An expiry date or security code beside the card is not detected.
 - **IBAN** → `BANK_ACCOUNT`, in either case: ISO 13616 mod-97 plus a per-country length
  table. It runs before the card detector, so an IBAN's digits cannot be
  claimed as a card number.
 - **US ABA routing number** → `BANK_ACCOUNT`: the weighted checksum and a
@@ -54,7 +54,25 @@ text with personal data tokenized (only the rules-pass anchors, unless the
 manifest uses `sterilize`). That is the same trust boundary as the folder
 itself, since the source files are already there, but a sync provider now
 receives a second, flattened, indexable copy. If that matters for a folder,
-exclude `.agent` from sync or keep the output elsewhere (`--out`).
+exclude `.agent` from sync or, for an agent run by hand, keep the output
 outside the folder (`--out`).
 Whether a host can reach `.agent/` depends on the bastion's version. From
 bastion 0.58.1 on, the bastion never serves it: a host's listings leave out
 every folder named `.agent`, at any depth and in any letter case, and the
 bastion refuses to stat, read or move anything in one, however the path is
 written and whatever link it goes through, and says why. It knows the folder
 by that name only, so an output folder inside the shared folder under any
 other name (`--out`) is served like any other folder. A bastion older than
 0.58.1 treats `.agent/` like any other folder: a host whose folder grant has
 `read` can list it and read the extract store as it is. On those bastions,
 add `.agent` to the paths the grant excludes; the bastion then leaves it out
 of a host's listings and refuses reads of it.
 The bastion keeps its agent in `<root>/.agent/` too, except on
 Windows when the bastion runs as the system service, or when it cannot create
 `.agent` in the folder (a read-only folder, for one); it then keeps it in its
 own state directory.
 The identity key is never in `.agent/`: it is wherever `FREEAGENT_TLS_KEY`
 points. The PII vault (plaintext values) is in `.agent/sanitize/vault.db` for
@@ -12,8 +12,11 @@ reads the extract index and store and writes:
 - **`sanitize/<aa>/<sha256>.txt`**: the sanitized text store. Personal data is
  replaced by stable `[PII_TYPE_n]` tokens. **From this stage on, later
-  stages read only this store.** The extract store keeps the plaintext and
+  stages read only this store.** The extract store keeps the plaintext. The
-  never leaves the machine.
+  agent never serves it, and while it is in `.agent/` (the default) a
  bastion from 0.58.1 on does not either. An older bastion lets a folder
  grant with `read` reach it there unless the grant excludes `.agent`
  ([runtime.md](runtime.md#where-an-agent-lives-rootagent)).
 - **`vault.db`**: the token-to-value map (in-process mode only), created 0600
  under umask 077. It is the only way back from a token to a value, and it
  stays on the machine. It is `<out>/sanitize/vault.db`, or
@@ -50,10 +53,17 @@ stages, there is no extract index, or the stage **stopped**; `2` completed with
  employers, medical facts) is found by the recall pass.
 - **`flag`**: the rules pass **only**: deterministic, no model, milliseconds.
  It replaces the anchors a pattern can find: e-mail addresses, phone
-  numbers, SSNs, cards, IBANs, routing numbers, crypto addresses, IP and MAC
+  numbers, SSNs, cards, IBANs, US routing numbers after a cue word, legacy and
-  addresses, API keys, private-key blocks, JWTs. **It does not find a name.**
+  P2SH Bitcoin addresses, IP and MAC addresses, API keys, private-key blocks,
-  A flag-mode catalog is not free of personal data, and the manifest doc says
+  JWTs ([the list](pii-sterilization.md#the-rules-pass-piidetectpy)). Other
-  so too.
+  bank account numbers, other crypto addresses (`bc1…`, Ethereum) and ID
  numbers such as passport numbers are not found. It can also replace an
  ordinary number that looks like a phone, card or social security number.
  **It does not find names, street addresses, dates of birth**, or anything
  else that takes reading for
  meaning (an employer, a school, a medical fact). It is the mode the bastion
  writes into every manifest it deploys. A flag-mode catalog is not free of
  personal data, and the manifest doc says so too.
 ## Where the sterilizer runs
@@ -1,8 +1,10 @@
-# bastion agent `seal` stage: the catalog leaves, nothing else does
+# bastion agent `seal` stage: the catalog in one encrypted bundle
-`seal` is the last stage of a run and the one way a dataset leaves the machine:
+`seal` is the last stage of a run: it packs an allowlisted bundle of the
-an allowlisted bundle of the catalog, encrypted with age to the owner and the
+catalog into one file, encrypted with age to the owner and the manifest's named
-manifest's named recipients. It is `bastion_agent/seal.py`.
+recipients. It writes the bundle into the output folder and sends it nowhere.
 `serve` answers queries with cards and excerpts, unsealed
 ([serve.md](serve.md)). It is `bastion_agent/seal.py`.
 ## What it does
@@ -58,8 +60,8 @@ plaintext.
 age (X25519 and ChaCha20-Poly1305) is a small, widely reviewed format with a
 single static binary for Mac, Windows and Linux. The envelope cannot be made
 with the standard library (it has no X25519 and no ChaCha20-Poly1305), so,
-unlike the extract stage's optional tools, **age is required here**: a catalog
+unlike the extract stage's optional tools, **age is required here**: without
-that cannot be sealed does not leave, and the stage says so and exits 1.
+it no bundle is written at all, and the stage says so and exits 1.
 ## Signature (optional)
@@ -1,8 +1,10 @@
 # bastion agent query endpoint: `serve`
-`serve` is how a host reads what the agent prepared without the data moving:
+`serve` is how a host reads what the agent prepared without being sent the
-a host (the owner's, or a grantee's) sends a query and gets **sanitized**
+files themselves: a host (the owner's, or a grantee's) sends a query and gets **sanitized**
-chunks and catalog cards back. Nothing else has a route: not the extract
+chunks and catalog cards back (sanitized by the manifest's mode: in `flag`
 mode, which the bastion deploys, names, street addresses and dates of birth
 remain; [sanitize.md](sanitize.md#modes-manifest-sanitizemode)). Nothing else has a route: not the extract
 store, not the vault, not the vectors table. It is `bastion_agent/serve.py`,
 mutual TLS with the standard library's `ssl`.
@@ -31,12 +33,26 @@ There is no other credential: no token, no password, no header.
 | Route | Body | Returns |
 |---|---|---|
-| `POST /search` | `{query, k?}` (k defaults to 5 and is held to 1–50) | `{hits, k}`: the top k sanitized windows with score, document id, chunk id, index and offsets. The same search as the CLI, through the same `EMBED_URL` model ([embed.md](embed.md)); 502 if the model cannot be reached. |
+| `POST /search` | `{query, k?}` (k defaults to 5 and is held to 1–50) | `{hits, k, sanitize_mode}`: the top k sanitized windows with score, document id, chunk id, index and offsets, and the sanitize mode that wrote them. The same search as the CLI, through the same `EMBED_URL` model ([embed.md](embed.md)); 502 if the model cannot be reached. |
-| `POST /read_range` | `{doc, start?, end}` | Up to 65,536 characters of the **sanitized** text of that document, with `total`. |
+| `POST /read_range` | `{doc, start?, end}` | Up to 65,536 characters of the **sanitized** text of that document, with `total` and `sanitize_mode`. |
 | `GET /catalog`, `GET /catalog/dirs` | query `prefix`, `offset`, `limit` (default 500, at most 2,000) | The file cards or the directory cards (owner only), a page at a time: `{cards, kind, total, offset, limit, next_offset, prefix}`, `next_offset` being `null` on the last page. 404 until the first sweep has written a catalog. |
 | `GET /status` | | `{state}`: `run.state.json` (owner only). |
 | `GET /health` | | `{ok, id, serving}` for any client that chained. |
 `sanitize_mode` says what was done to the text in that reply: the mode the
 sanitize stage recorded for that document in `sanitize.jsonl`, `flag` or
 `sterilize`. A search names the weakest mode among its hits. It is `unknown`
 when that record does not vouch for the text: the last sanitize run did not
 write the document (it stopped, or the file is an older run's), the file was
 rewritten after the record (a sweep is running), or a search hit's text is no
 longer what the store holds (the chunks predate the latest sanitize run). A
 search with no hits names the manifest's `sanitize.mode`, or `none` when the
 manifest has no sanitize stage. In `flag` mode, the one the bastion deploys,
 names, street addresses and dates of birth remain in the text; only
 `sterilize` looks for them. Changing the manifest's mode does not change the
 field: after a change of mode the next sweep sanitizes every file again, and
 until it has, the text served is the previous mode's and the reply says so.
 `doc` is the document's content hash (64 lowercase hex), the same id the
 extract and sanitized stores share. The route reads only the `sanitize/`
 directory, so the plaintext for that id, which exists on disk, is never what
@@ -9,9 +9,18 @@ WORKDIR /app
 # boot (recall.py was, once)
 COPY __init__.py detect.py llmclient.py pipeline.py recall.py service.py vault.py /app/pii/
-# Root inside the container is deliberate: the vault db and its directory
+# The service runs as its own unprivileged user. The vault and its directory
-# are created 0700/0600 (service sets umask 077) on the mounted /data,
+# are created 0700/0600 (the service sets umask 077), so a volume mounted on
-# and the single-purpose process is bind-scoped by PII_BIND. Revisit if
+# /data must be writable by uid 10001.
-# the service ever grows a second concern.
+RUN groupadd --system --gid 10001 pii \
    && useradd --system --uid 10001 --gid 10001 --no-create-home \
        --home-dir /nonexistent --shell /usr/sbin/nologin pii \
    && install -d -m 0700 -o pii -g pii /data
 USER 10001:10001
 # The service listens on 127.0.0.1 unless PII_BIND says otherwise, and inside
 # a container that is reachable from nothing else. To publish it, choose the
 # address and the host side explicitly, for example:
 #   docker run -e PII_BIND=0.0.0.0:8091 -p 127.0.0.1:8091:8091 ...
 ENV PYTHONUNBUFFERED=1
 CMD ["python", "-m", "pii.service"]
@@ -2,3 +2,9 @@ from . import detect, pipeline  # noqa: F401
 from .detect import PII_TYPES, Finding  # noqa: F401
 from .pipeline import TOKEN_RE, ground, rehydrate, replace_values, sterilize  # noqa: F401
 from .vault import Vault  # noqa: F401
 # The package's public names: the modules and what the agent calls in them.
 __all__ = [
    "detect", "pipeline", "PII_TYPES", "Finding", "TOKEN_RE", "ground", "rehydrate",
    "replace_values", "sterilize", "Vault",
 ]
@@ -54,23 +54,54 @@ IPV4_RE = re.compile(r"(?<![\d.])(?:\d{1,3}\.){3}\d{1,3}(?!\d)(?!\.[\w-])")
 IPV6_RE = re.compile(r"(?<![\w:.])(?:[0-9A-Fa-f]{0,4}:){2,7}[0-9A-Fa-f]{0,4}(?![\w:])")
 SSN_RE = re.compile(r"(?<![\d-])(\d{3})-(\d{2})-(\d{4})(?![\d-])")
 # Spaced: not when the groups are part of a longer run of spaced digits.
 SSN_SPACED_RE = re.compile(r"(?<![\d-])(?<!\d )(\d{3}) (\d{2}) (\d{4})(?![\d-])(?! \d)")
 # Nine bare digits are an SSN only after a keyword: an invoice or order
 # number has the same shape.
 SSN_BARE_RE = re.compile(r"(?<![\w-])(\d{3})(\d{2})(\d{4})(?![\w-])")
 SSN_CUE_RE = re.compile(r"(?i)\b(?:ssn|ss#|social\s+security|soc\.?\s*sec\b)")
-# 13–19 digits with optional single space/dash separators; Luhn decides.
+# 13–19 digits with optional single space/dot/dash separators; Luhn decides.
-CARD_RE = re.compile(r"(?<![\d-])\d(?:[ -]?\d){12,18}(?![\d-])")
+# A dot right after a digit before it, or before a digit after it, makes the
 # run part of a longer dotted number.
 CARD_RE = re.compile(r"(?<![\d-])(?<!\d\.)\d(?:[ .-]?\d){12,18}(?![\d-])(?!\.\d)")
 # Digit-count and shape checks happen in code; this only shapes candidates.
 PHONE_RE = re.compile(
    r"(?<![\w.])(?:\+\d{1,3}[\s.-]?)?(?:\(\d{1,4}\)[\s.-]?)?\d{2,4}(?:[\s.-]\d{2,4}){1,4}"
    r"(?![\w-])(?!\.[\w-])"   # a sentence-ending full stop is not part of the number
 )
 # International form: a leading + and country code, an optional (0) trunk
 # prefix or (area) code, then digit groups of any length (+44 7700 900123,
 # +49 (0)30 1234567, +442079460958). Groups after the first need a separator,
 # so a long digit run cannot be split many ways.
 INTL_PHONE_RE = re.compile(
    r"(?<![\w+])\+\d{1,3}(?:[ .-]?\(\d{1,4}\))?[ .-]?\d{1,12}(?:[ .-]\d{1,8}){0,5}"
    r"(?![\w-])(?!\.[\w-])"
 )
 # National form with a trunk 0 and a long subscriber group (07700 900123,
 # 030 1234567, (030) 1234567). The separator after the area code is required.
 NATIONAL_PHONE_RE = re.compile(
    r"(?<![\w.+])(?:\(0\d{1,4}\) ?|0\d{1,4}[ .-])\d{3,8}(?:[ .-]\d{2,4}){0,2}"
    r"(?![\w-])(?!\.[\w-])"
 )
 # A seven-digit local number (555-0142) is a phone only after a phone word;
 # order and reference numbers have the same shape.
 LOCAL_PHONE_RE = re.compile(r"(?<![\w.+-])\d{3}[ .-]\d{4}(?![\w-])(?!\.[\w-])")
 PHONE_CUE_RE = re.compile(
    r"(?i)\b(?:call|calling|ring|phone|tel|telephone|mobile|cell|fax|text|dial|reach)\b")
 # A dotted number right after a version word is a version, not an address.
 VERSION_CUE_RE = re.compile(r"(?i)(?:\bv|\bversion|\bver\.?|\brelease|\bbuild|\bfirmware)\s*:?\s*$")
 # ── checksum-validated account identifiers (borrowed from Presidio's
 # recognizer list; only the ones a checksum can anchor are here) ──────────
 # IBAN: country code + 2 check digits + 11-30 alphanumerics, optional
-# spaces every 4. mod-97 == 1 decides (ISO 13616).
+# spaces every 4, in either case. mod-97 == 1 and the country's length
-IBAN_RE = re.compile(r"(?<![A-Za-z0-9])[A-Z]{2}\d{2}(?:[ ]?[A-Z0-9]{4}){2,7}(?:[ ]?[A-Z0-9]{1,4})?(?![A-Za-z0-9])")
+# decide (ISO 13616), so random text of the same shape is not claimed.
 IBAN_RE = re.compile(r"(?<![A-Za-z0-9])[A-Za-z]{2}\d{2}(?:[ ]?[A-Za-z0-9]{4}){2,7}(?:[ ]?[A-Za-z0-9]{1,4})?(?![A-Za-z0-9])")
 IBAN_LENGTHS = {
    "AD": 24, "AE": 23, "AL": 28, "AT": 20, "AZ": 28, "BA": 20, "BE": 16, "BG": 22,
    "BH": 22, "BR": 29, "CH": 21, "CR": 22, "CY": 28, "CZ": 24, "DE": 22, "DK": 18,
@@ -147,8 +178,17 @@ def _find_secrets(text):
 def _find_ibans(text):
    for m in IBAN_RE.finditer(text):
-        if _iban_ok(m.group(0)):
+        # a spaced IBAN followed by a four-letter word ("... 7034 then")
-            yield Finding(m.group(0), "BANK_ACCOUNT", m.start(), m.end())
+        # matches with the word attached; drop trailing groups until it checks
        cand = m.group(0)
        while True:
            if _iban_ok(cand):
                yield Finding(cand, "BANK_ACCOUNT", m.start(), m.start() + len(cand))
                break
            cut = cand.rfind(" ")
            if cut < 0:
                break
            cand = cand[:cut]
 def _find_abas(text):
@@ -188,12 +228,18 @@ def _find_macs(text):
        yield Finding(m.group(0), "MAC_ADDRESS", m.start(), m.end())
 def _after_version_word(text, start):
    return VERSION_CUE_RE.search(text[max(0, start - 12):start]) is not None
 def _find_ips(text):
    for m in IPV4_RE.finditer(text):
        try:
            ipaddress.IPv4Address(m.group(0))
        except ValueError:
            continue
        if _after_version_word(text, m.start()):
            continue
        yield Finding(m.group(0), "IP_ADDRESS", m.start(), m.end())
    for m in IPV6_RE.finditer(text):
        cand = m.group(0)
@@ -204,10 +250,23 @@ def _find_ips(text):
        yield Finding(cand, "IP_ADDRESS", m.start(), m.end())
 def _ssn_ok(area, group, serial):
    # never issued: area 000, 666 or 900-999, group 00, serial 0000
    a = int(area)
    return a != 0 and a != 666 and a < 900 and group != "00" and serial != "0000"
 def _find_ssns(text):
    for m in SSN_RE.finditer(text):
-        area = int(m.group(1))
+        if _ssn_ok(m.group(1), m.group(2), m.group(3)):
-        if area == 0 or area == 666 or area >= 900:
+            yield Finding(m.group(0), "SSN", m.start(), m.end())
    for m in SSN_SPACED_RE.finditer(text):
        if _ssn_ok(m.group(1), m.group(2), m.group(3)):
            yield Finding(m.group(0), "SSN", m.start(), m.end())
    for m in SSN_BARE_RE.finditer(text):
        if not _ssn_ok(m.group(1), m.group(2), m.group(3)):
            continue
        if not SSN_CUE_RE.search(text[max(0, m.start() - 40):m.start()]):
            continue
        yield Finding(m.group(0), "SSN", m.start(), m.end())
@@ -225,6 +284,18 @@ def _find_phones(text):
        if not 10 <= len(digits) <= 15:
            continue
        yield Finding(m.group(0), "PHONE", m.start(), m.end())
    for m in INTL_PHONE_RE.finditer(text):
        # E.164 allows at most 15 digits; a (0) trunk prefix is not one of them
        digits = re.sub(r"\D", "", m.group(0).replace("(0)", "", 1))
        if 8 <= len(digits) <= 15:
            yield Finding(m.group(0), "PHONE", m.start(), m.end())
    for m in NATIONAL_PHONE_RE.finditer(text):
        digits = re.sub(r"\D", "", m.group(0))
        if 10 <= len(digits) <= 11:
            yield Finding(m.group(0), "PHONE", m.start(), m.end())
    for m in LOCAL_PHONE_RE.finditer(text):
        if PHONE_CUE_RE.search(text[max(0, m.start() - 30):m.start()]):
            yield Finding(m.group(0), "PHONE", m.start(), m.end())
 # Priority order: earlier detectors own their spans outright. Secrets go
@@ -22,7 +22,7 @@ downgrade the sensor. Stdlib-only here; the gliner package is the one
 optional third-party import in the module and it is loaded lazily.
 """
-from typing import Callable, List, Tuple
+from typing import List, Tuple
 from . import llmclient
 from .detect import PII_TYPES
@@ -19,7 +19,8 @@ exists for the unit-test harness only. /rehydrate returns REAL values —
 callers are trusted surfaces only; every call lands in the vault's audit
 table.
-Config (env): PII_BIND (host:port, default 0.0.0.0:8091), LLM_URL
+Config (env): PII_BIND (host:port, default 127.0.0.1:8091; another address
 only when set), LLM_URL
 (default http://llm:8080), PII_DB (default /data/vault.db),
 PII_LLM_TIMEOUT seconds (default 300).
 """
@@ -38,7 +39,7 @@ _MAX_BODY = 2 * 1024 * 1024
 class Config:
    def __init__(self, env):
-        self.bind = env.get("PII_BIND", "0.0.0.0:8091")
+        self.bind = env.get("PII_BIND", "127.0.0.1:8091")
        self.llm_url = env.get("LLM_URL", "http://llm:8080")
        self.db = env.get("PII_DB", "/data/vault.db")
        self.api_key = env.get("PII_API_KEY", "")
@@ -47,12 +47,12 @@ class DetectTests(unittest.TestCase):
        self.assertEqual(self.find("ref 4111 1111 1111 1112"), set())
    def test_ipv4_owns_dotted_span(self):
-        found = self.find("host at 192.168.68.54 responded")
+        found = self.find("host at 198.51.100.54 responded")
-        self.assertIn(("192.168.68.54", "IP_ADDRESS"), found)
+        self.assertIn(("198.51.100.54", "IP_ADDRESS"), found)
-        self.assertNotIn(("192.168.68.54", "PHONE"), found)
+        self.assertNotIn(("198.51.100.54", "PHONE"), found)
    def test_ipv4_invalid_octet(self):
-        self.assertNotIn(("999.168.68.541", "IP_ADDRESS"), self.find("x 999.168.68.541"))
+        self.assertNotIn(("999.51.100.541", "IP_ADDRESS"), self.find("x 999.51.100.541"))
    def test_ipv6_and_mac_disambiguation(self):
        found = self.find("fe80::1ff:fe23:4567:890a from aa:bb:cc:dd:ee:ff")
@@ -127,6 +127,87 @@ class DetectTests(unittest.TestCase):
        self.assertEqual(self.find("the skate park (AKA the bowl) opens at 9"), set())
 class RulesOnlyFormatTests(unittest.TestCase):
    """The rules pass is all that flag mode runs, so each format here must be
    found with no recall model at all, and each lookalike must be left alone."""
    FOUND = [
        ("UK mobile, international", "Mobile: +44 7700 900123", "+44 7700 900123", "PHONE"),
        ("German, (0) trunk prefix", "Tel. +49 (0)30 1234567", "+49 (0)30 1234567", "PHONE"),
        ("international, no separators", "ring +442079460958 today", "+442079460958", "PHONE"),
        ("international, dots", "fax +33.6.39.98.12.34 now", "+33.6.39.98.12.34", "PHONE"),
        ("international, dashes", "cell +44-20-7946-0958.", "+44-20-7946-0958", "PHONE"),
        ("UK national, trunk 0", "call 07700 900123 later", "07700 900123", "PHONE"),
        ("German national, trunk 0", "Tel 030 1234567", "030 1234567", "PHONE"),
        ("German national, (0xx)", "Tel (030) 1234567", "(030) 1234567", "PHONE"),
        ("7-digit local after a phone word", "Ring 555-0142 for the gate code.", "555-0142", "PHONE"),
        ("7-digit local, call", "call me on 555 0199", "555 0199", "PHONE"),
        ("IBAN lowercase", "iban: gb82west12345698765432", "gb82west12345698765432", "BANK_ACCOUNT"),
        ("IBAN lowercase, spaced", "pay de89 3704 0044 0532 0130 00 now",
         "de89 3704 0044 0532 0130 00", "BANK_ACCOUNT"),
        ("IBAN followed by a four-letter word", "to BE68 5390 0754 7034 then",
         "BE68 5390 0754 7034", "BANK_ACCOUNT"),
        ("card with dots", "Card 4111.1111.1111.1111", "4111.1111.1111.1111", "CREDIT_CARD"),
        ("card with dashes", "card 5555-5555-5555-4444 ok", "5555-5555-5555-4444", "CREDIT_CARD"),
        ("card after no.", "card no.4111 1111 1111 1111", "4111 1111 1111 1111", "CREDIT_CARD"),
        ("card, 4-6-5 grouping", "amex 3782 822463 10005", "3782 822463 10005", "CREDIT_CARD"),
        ("SSN with spaces", "SSN 219 09 9999", "219 09 9999", "SSN"),
        ("SSN, no separators, keyword", "Social security number 219099999.", "219099999", "SSN"),
        ("SSN, no separators, SSN:", "ssn: 219099999", "219099999", "SSN"),
    ]
    LEFT_ALONE = [
        ("order number", "Order 555-0142 shipped"),
        ("order number, digits only", "order #20431877 is packed"),
        ("9-digit invoice id", "invoice 219099999 is due"),
        ("16 digits failing Luhn, spaces", "ref 4111 1111 1111 1112"),
        ("16 digits failing Luhn, dots", "ref 4111.1111.1111.1112"),
        ("IBAN shape, bad checksum, lowercase", "iban gb82west12345698765433"),
        ("IBAN shape, bad checksum, uppercase", "iban GB82WEST12345698765433 ok"),
        ("version string", "version 1.2.3.4 shipped"),
        ("version string, v", "upgrade to v1.2.3.4 today"),
        ("ISO date", "released 2026-08-03 at noon"),
        ("US date", "Date of birth: 03/14/1987"),
        ("dotted date", "seen on 14.03.1987 in the hall"),
        ("SSN shape, group 00", "ssn 219-00-9999"),
        ("short number after +", "add +12 345 to it"),
        ("spaced digits inside a longer run", "lot 4 219 09 9999 7"),
    ]
    def find(self, text):
        return {(f.value, f.type) for f in detect.detect(text)}
    def test_each_format_is_found(self):
        for label, text, value, typ in self.FOUND:
            with self.subTest(label):
                self.assertIn((value, typ), self.find(text))
    def test_lookalikes_are_left_alone(self):
        for label, text in self.LEFT_ALONE:
            with self.subTest(label):
                self.assertEqual(self.find(text), set())
    def test_rules_only_sterilize_leaves_no_value(self):
        # what flag mode does: the pipeline with no recall model
        with tempfile.TemporaryDirectory() as tmp:
            vault = Vault(os.path.join(tmp, "v.db"))
            for label, text, value, _ in self.FOUND:
                with self.subTest(label):
                    out, _ = pipeline.sterilize(text, "s", vault, lambda t: [],
                                                escape_held_literals=True)
                    self.assertNotIn(value, out)
                    self.assertEqual(pipeline.rehydrate(out, "s", vault)[0], text)
            for label, text in self.LEFT_ALONE:
                with self.subTest(label):
                    out, _ = pipeline.sterilize(text, "s", vault, lambda t: [],
                                                escape_held_literals=True)
                    self.assertEqual(out, text)
    def test_ip_address_is_still_found_after_other_words(self):
        self.assertIn(("203.0.113.42", "IP_ADDRESS"), self.find("Server at 203.0.113.42 refused."))
        self.assertIn(("203.0.113.42", "IP_ADDRESS"), self.find("server 203.0.113.42 version 2"))
 class VaultTests(unittest.TestCase):
    def setUp(self):
        self.tmp = tempfile.TemporaryDirectory()
@@ -672,6 +753,14 @@ class ServiceTests(unittest.TestCase):
        with self.assertRaises(SystemExit):
            service.Config({"PII_DB": "x"})
    def test_default_bind_is_loopback(self):
        # /rehydrate returns real values over plain HTTP: only an explicit
        # setting may open the service beyond this machine
        self.assertEqual(service.Config({"PII_DB": "x", "PII_API_KEY": "k"}).bind,
                         "127.0.0.1:8091")
        self.assertEqual(service.Config({"PII_DB": "x", "PII_API_KEY": "k",
                                         "PII_BIND": "0.0.0.0:8091"}).bind, "0.0.0.0:8091")
    def test_fail_closed_on_unstartable_recall_backend(self):
        with self.assertRaises(SystemExit) as ctx:
            service.Config({"PII_DB": "x", "PII_API_KEY": "k", "PII_RECALL": "nope"})
@@ -17,31 +17,41 @@ lands here is published far past where it came from.
 This fails on:
-  term       a word or phrase from TERMS: the owner's and members' details that
+  term       a word or phrase from TERMS, a list of details about real people
-             were once copied into a repository (bank, employer, account
+             that must not appear in this repository. They are held as keyed
             digits, places, names, addresses, a device, a customer, a person's
             project, a Host's content, record ids). They are held as keyed
             digests, so this file does not publish what it keeps out, and a
             guess cannot be tested against the table without the key: every
             1-3 token n-gram of a line is normalised (lowercase letters and
             digits, joined by one space), HMAC-SHA256'd with the maintainers'
-             key and looked up. The key lives outside the repository
+             key and looked up. A backslash escape (\\n, \\t, \\x41, \\u00e9...)
             separates tokens as whitespace would, so a term written right
             after one is still found. The key lives outside the repository
             (NO_PERSONAL_DATA_KEY_FILE, default
             /etc/privatae/keys/no-personal-data.hmac), as do the plain list
             and the script that hashes it. Where the key is absent (a CI job,
             which holds no secrets) this class is skipped and the run says
-             so; the other classes always run. A key other than the one TERMS
+             so; the other classes still run. A key other than the one TERMS
             was made with fails the run (KEY_CHECK), since it would match
             nothing.
-  entity-id  a real entity id that labels a person's Host (RESERVED_ENTITY_IDS)
+  entity-id  a reserved entity id, one that labels a person's Host, in
-             in entity-N, entity N, entity_id=N, "entity_id": N, entity id N,
+             entity-N, entity N, entity_id=N, "entity_id": N, entity id N,
-             Host N, Host #N or host-N form; examples and tests use 9000 and up
+             Host N, Host #N or host-N form; examples and tests use 9000 and
             up. The ids live outside the repository, one per line
             (RESERVED_ENTITY_IDS_FILE, default
             /etc/privatae/reserved-entity-ids). Where that file is absent (a
             CI job) this class is skipped and the run says so; a file that
             holds no id, or a line that is not one, fails the run, since it
             would match nothing.
  provenance a line that says its data is real: "real rows" followed by
             "that" or "from", an all-caps "REAL" row, "verbatim" from a
             transcript or a run, a row "read off" an entity...
  email      a mailbox at a personal mail provider (gmail, yahoo, proton...)
  phone      a North American number outside the 555 fiction range
 A run of 24 or more base64 or base64url characters that decodes to mostly
 printable UTF-8 is decoded and its text checked for every class as well, so a
 value is not let through by being encoded.
 Every tracked text file is scanned, file names included, and every text member
 of each zip-format bundle (an app bundle is one), since a bundle is what
 installs. Lockfiles and other binary files are not.
@@ -63,6 +73,8 @@ Exit 0 = clean; exit 1 + FAIL lines otherwise.
 from __future__ import annotations
 import base64
 import binascii
 import hmac
 import os
 import re
@@ -118,13 +130,43 @@ TERMS: frozenset[str] = frozenset({
 })
 _NGRAM_MAX = 3
-# Entity ids of real Hosts that belong to people (the operator's own, members',
+# The reserved entity ids are read from a file outside the repository, one id
-# the platform's Security entity and the two security-test Hosts).
+# per line; blank lines and lines starting with # are ignored.
-RESERVED_ENTITY_IDS = (3, 102, 122, 123, 128, 163, 164)
+RESERVED_ENTITY_IDS_FILE = os.environ.get("RESERVED_ENTITY_IDS_FILE", "/etc/privatae/reserved-entity-ids")
-_IDS = "|".join(str(i) for i in RESERVED_ENTITY_IDS)
+
-ENTITY_ID = re.compile(
+
 def load_reserved_ids(path: str = RESERVED_ENTITY_IDS_FILE):
    """(ids, error). A missing file is (None, None): the class is skipped. A
    file that cannot be read, holds a line that is not an id, or holds no id
    at all is (None, error): a list that matches nothing must not pass."""
    try:
        with open(path, encoding="utf-8") as f:
            lines = f.read().splitlines()
    except FileNotFoundError:
        return None, None
    except (OSError, UnicodeDecodeError) as e:
        return None, "the reserved entity ids at %s cannot be read: %s" % (path, e)
    ids = []
    for n, raw in enumerate(lines, 1):
        s = raw.strip()
        if not s or s.startswith("#"):
            continue
        if not s.isdigit():
            return None, "%s:%d is not an entity id (one id per line)" % (path, n)
        ids.append(int(s))
    if not ids:
        return None, "%s holds no entity id, so the entity-id check would match nothing" % path
    return tuple(ids), None
 def entity_id_pattern(ids):
    """The entity-id class for these ids, or None (the class is off)."""
    if not ids:
        return None
    alts = "|".join(str(i) for i in sorted(set(ids), key=lambda i: -len(str(i))))
    return re.compile(
        r"\b(?:entit(?:y|ies)(?:[-_ ]|_id\s*==?\s*|_id[\"']?\s*:\s*|\s+id\s+|\s+#)"
-    r"|Host\s+#?|host-)(?:%s)\b" % _IDS, re.I)
+        r"|Host\s+#?|host-)(?:%s)\b" % alts, re.I)
 PROVENANCE = re.compile(
    r"\breal rows? (?:that|from|read|copied|taken|off)\b"
@@ -181,13 +223,25 @@ def key_matches(key: bytes) -> bool:
    return hmac.compare_digest(digest(KEY_CHECK_TEXT, key), KEY_CHECK)
 # A backslash escape as source and data files write it: a named control
 # character, a hex or unicode escape, or an octal one. "a\nb" is two words, but
 # the tokenizer alone reads "a" and "nb". The line is read once as written and
 # once with each escape turned into a space; a hit in either counts. Reading it
 # as written too keeps a term that follows a backslash which is not an escape
 # (a Windows path: C:\name) whole.
 _ESCAPE = re.compile(r"\\(?:x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8}|[0-7]{1,3}|[abefnrtv])")
 class TermScanner:
    """Finds TERMS in a line by hashing its 1-3 token n-grams. Digests are
-    cached, so a repeated n-gram is hashed once."""
+    cached, so a repeated n-gram is hashed once. It also carries the
    entity-id pattern for the reserved ids (None: that class is off)."""
-    def __init__(self, terms, key):
+    def __init__(self, terms, key, entity_ids=None):
        self.terms = terms
        self.key = key
        self.entity_ids = entity_ids
        self.entity_re = entity_id_pattern(entity_ids)
        self._cache = {}
    def _hit(self, gram: str) -> bool:
@@ -195,16 +249,64 @@ class TermScanner:
            self._cache[gram] = digest(gram, self.key) in self.terms
        return self._cache[gram]
    def _grams(self, text: str) -> dict:
        toks = [t.lower() for t in _TOKEN.findall(text)]
        hits = {}
        for n in range(1, _NGRAM_MAX + 1):
            for i in range(len(toks) - n + 1):
                gram = " ".join(toks[i:i + n])
                if self._hit(gram):
                    hits[gram] = hits.get(gram, 0) + 1
        return hits
    def scan(self, line: str) -> int:
        if self.key is None:
            return 0
-        toks = [t.lower() for t in _TOKEN.findall(line)]
+        hits = self._grams(line)
-        found = 0
+        if "\\" in line:
-        for n in range(1, _NGRAM_MAX + 1):
+            for gram, n in self._grams(_ESCAPE.sub(" ", line)).items():
-            for i in range(len(toks) - n + 1):
+                hits[gram] = max(hits.get(gram, 0), n)
-                if self._hit(" ".join(toks[i:i + n])):
+        return sum(hits.values())
-                    found += 1
+
-        return found
+
 # ── values inside base64 ─────────────────────────────────────────────────────
 # A run of base64 or base64url characters, with its padding. The run is decoded
 # at each of the four offsets, since it may be glued to the text before it or
 # cut by a window edge; an offset whose bytes are mostly printable UTF-8 is the
 # text. Random bytes almost never are, so identifiers, paths and digests that
 # happen to be base64-shaped decode to nothing worth reading.
 _B64_RUN = re.compile(r"[A-Za-z0-9+/_-]{24,}={0,2}")
 _B64_MIN = 24
 _B64_PRINTABLE = 0.85
 # A decoded text may itself hold base64; it is followed this many levels down.
 _B64_DEPTH = 2
 def _printable_share(text: str) -> float:
    if not text:
        return 0.0
    ok = sum(1 for ch in text if ch in "\t\n\r" or (ch.isprintable() and ch != "\ufffd"))
    return ok / len(text)
 def b64_texts(line: str):
    """The text of every base64 run in line that decodes to mostly printable
    UTF-8, one per run."""
    for m in _B64_RUN.finditer(line):
        run = m.group(0).rstrip("=").replace("-", "+").replace("_", "/")
        for off in range(4):
            body = run[off:]
            body = body[:len(body) - len(body) % 4] if len(body) % 4 == 1 else body
            if len(body) < _B64_MIN:
                break
            try:
                raw = base64.b64decode(body + "=" * (-len(body) % 4), validate=True)
            except (ValueError, binascii.Error):
                continue
            text = raw.decode("utf-8", "replace")
            if _printable_share(text) >= _B64_PRINTABLE:
                yield text
                break
 # A minified asset or a bundle member can hold one line of megabytes. It is read
@@ -241,15 +343,65 @@ def line_hits(line: str, scanner: TermScanner):
 # Classes whose line would carry the personal value itself: their FAIL line
 # names the file, the line and the class, never the text, so a CI log does not
-# publish what the gate keeps out.
+# publish what the gate keeps out. A line that holds one shows no text for any
 # finding, since another class's excerpt of the line would carry the value too.
 _QUIET = {"term", "email", "phone"}
-def line_findings(line: str, scanner: TermScanner):
+_PATH_SEP = re.compile(r"([/!])")
 def shown_path(label: str, scanner: TermScanner) -> str:
    """label as a FAIL line shows it. A path component (a directory, a file
    name, a bundle member's name) that holds a quiet value is replaced by a
    placeholder giving its depth, so the line says where without the text. A
    value that runs across components withholds them all."""
    def quiet(text):
        return any(cls in _QUIET for cls, _ in line_findings(text, scanner))
    if not quiet(label):
        return label
    parts = _PATH_SEP.split(label)
    def masked(every):
        out, depth = [], 0
        for p in parts:
            if p in ("/", "!"):
                out.append(p)
                continue
            depth += 1
            out.append("<withheld %d>" % depth if every or quiet(p) else p)
        return "".join(out)
    shown = masked(False)
    return masked(True) if quiet(shown) else shown
 def line_report(label: str, n: int, line: str, scanner: TermScanner, allow=(), used=None,
                skip_entity_ids: bool = False):
    """(class, FAIL text) for each finding on line n of label that no allow
    entry covers."""
    used = set() if used is None else used
    hits = list(line_hits(line, scanner))
    quiet = any(cls in _QUIET for _part, cls, _what in hits)
    out = []
    where = None
    for part, cls, what in hits:
        if (cls == "entity-id" and skip_entity_ids) or allowed(allow, label, cls, part, used):
            continue
        if where is None:
            where = shown_path(label, scanner)
        shown = "" if quiet else part.strip()[:140]
        out.append((cls, ("%s:%d  [%s: %s]  %s" % (where, n, cls, what, shown)).rstrip()))
    return out
 def line_findings(line: str, scanner: TermScanner, _depth: int = 0):
    out = []
    for _ in range(scanner.scan(line)):
        out.append(("term", "a listed term"))
-    for m in ENTITY_ID.finditer(line):
+    if scanner.entity_re is not None:
        for m in scanner.entity_re.finditer(line):
            out.append(("entity-id", m.group(0)))
    for m in PROVENANCE.finditer(line):
        out.append(("provenance", m.group(0)))
@@ -258,6 +410,10 @@ def line_findings(line: str, scanner: TermScanner):
    for m in PHONE.finditer(line):
        if (m.group(2) or m.group(5)) != "555":
            out.append(("phone", "a phone number"))
    if _depth < _B64_DEPTH:
        for text in b64_texts(line):
            for cls, what in line_findings(text, scanner, _depth + 1):
                out.append((cls, what if what.endswith("(in base64)") else what + " (in base64)"))
    return out
@@ -363,32 +519,61 @@ def self_test():
    fails = []
    probe_key = os.urandom(32)
    # Invented ids stand in for the reserved ones, which this file never holds.
    probe_ids = (9101, 9202, 9303)
    sc = TermScanner(frozenset(digest(g, probe_key) for g in ("zorblaxquent", "quilmar vesk", "tam orvel prax")),
-                     probe_key)
+                     probe_key, probe_ids)
    def classes(line):
        return {c for c, _ in line_findings(line, sc)}
    def b64(s: str, url: bool = False) -> str:
        enc = base64.urlsafe_b64encode if url else base64.b64encode
        return enc(live(s).encode("utf-8")).decode("ascii").rstrip("=" if url else "")
    positives = [(c, live(s)) for c, s in [
        ("term", "pays into the Zorblaxquent ledger"), ("term", "QUILMAR-vesk sent it"),
        ("term", "a tam_orvel.prax file"),
-        ("entity-id", "entity_id=|3,"), ("entity-id", '"entity_id": |128'), ("entity-id", "Entity |164's app"),
+        # A term right after a backslash escape, as a string literal or a JSON
-        ("entity-id", "kubectl -n entity-|3 exec"), ("entity-id", "pod entity-|3-0 restarted"), ("entity-id", "measured on Host #|163"),
+        # value writes it, and after a backslash that is not one.
-        ("entity-id", "gitea_user = 'host-|3'"), ("entity-id", "entity id |122 knocked"),
+        ("term", r'"line one\nZorblax|quent"'), ("term", r"col\tzorblax|quent\tcol"),
        ("term", r"end\r\nquilmar| vesk"), ("term", r"a\x41zorblax|quent"), ("term", r"x\u00e9zorblax|quent"),
        ("term", r"C:\Users\zorblax|quent\notes"),
        ("entity-id", "entity_id=|9101,"), ("entity-id", '"entity_id": |9202'), ("entity-id", "Entity |9303's app"),
        ("entity-id", "kubectl -n entity-|9101 exec"), ("entity-id", "pod entity-|9101-0 restarted"),
        ("entity-id", "measured on Host #|9303"), ("entity-id", "gitea_user = 'host-|9101'"),
        ("entity-id", "entity id |9202 knocked"),
        ("provenance", "the real| rows that fired on the Host"), ("provenance", "the REAL| row 9"),
        ("provenance", "verbatim| from the transcript"), ("provenance", "read| off entity 9001"),
        ("email", "mail ana.b@gm|ail.com today"), ("email", "x@proto|nmail.com"),
        ("phone", "call +1 (2|02) 555-|0100 or +1 30|1 444 1234"), ("phone", "+1|3014441234"),
    ]]
    # Every class is read inside base64 and base64url, padded or not, whole or
    # glued to the text before it.
    positives += [
        ("term", 'data: "%s"' % b64("a note that names Zorblax|quent outright")),
        ("term", "token=%s" % b64("pays QUILMAR| vesk ~~ été ok??", url=True)),
        ("term", "xyz" + b64("a note that names Zorblax|quent outright")),
        ("term", b64(b64("twice over: zorblax|quent, still found"))),
        ("entity-id", "blob %s end" % b64("the pod entity-|9101-0 restarted twice")),
        ("email", "%s" % b64("write to ana.b@gm|ail.com about it")),
    ]
    negatives = [live(s) for s in [
-        "zorblax quent", "quilmar", "a vesk quilmar", "orvel prax tam",
+        "zorblax quent", "quilmar", "a vesk quilmar", "orvel prax tam", r"zorblax\nquent",
-        "entity_id=9003", "entity_id=30", "entity 1023", "entity-3x", "host-31", "Host 7",
+        "entity_id=91011", "entity_id=910", "entity 92021", "entity-9101x", "host-91012", "Host 9104",
-        "entity-9310-0.svc", "the v0.3.128 tag", "entity_id = <id>",
+        "entity-9310-0.svc", "the v0.9.9101 tag", "entity_id = <id>",
        "real rows in a throwaway container", "the real row replaces the echo",
        "returns the text verbatim", "ana@example.com", "a•••@gmail.com shows the first letter",
        "+1 415 555 0199", "+1|5555550100", "ids 2026091812345", "port 8800, 4096 MiB",
        "MemoryHigh=15032385536 MemoryMax=17179869184", "2026-10-07 12:00",
        "widths 278 584 556 1015 667 722",
        # Base64-shaped text that is not base64 of text: identifiers, paths,
        # digests and random bytes are not decoded into findings.
        "TestPublishedSourceCarriesNoOldInternalNameAnywhere",
        "/usr/local/lib/python3/site-packages/zorbtool/__init__",
        "sha256:1ae0735f00daffa3aaf1363a5184c0d2dc55c78e3db4ec70241cdac97bf84b59",
        base64.b64encode(bytes(range(200, 256))).decode("ascii"),
        b64("a plain note with nothing listed in it at all"),
    ]]
    for cls, line in positives:
        if cls not in classes(line):
@@ -408,11 +593,41 @@ def self_test():
    # The real table is wired: every entry is a digest of the right shape.
    if not TERMS or not all(re.fullmatch(r"[0-9a-f]{32}", k) for k in TERMS):
        fails.append("self-test: TERMS is empty or malformed")
    # A line holding a quiet value shows no text for any finding on it: here a
    # provenance phrase beside a mailbox, and one base64 run holding both an
    # entity id and a term.
    blob = b64("the pod entity-|9101-0 belongs to zorblax|quent")
    for line, want, secret in ((live("the real| rows that came from ana.b@gm|ail.com"),
                                {"provenance", "email"}, live("ana.b@gm|ail")),
                               ("note %s end" % blob, {"entity-id", "term"}, blob[:24])):
        got = line_report("x.md", 1, line, sc)
        if {c for c, _ in got} != want or any(secret in t for _c, t in got):
            fails.append("self-test: a line with a quiet value is reported without its text, got %s" % got)
    # A path that holds a quiet value is shown with that component withheld,
    # by depth; one split across components withholds them all.
    for label, want in (("docs/zorblaxquent-notes.md", "docs/<withheld 2>"),
                        ("app.bundle!notes/" + live("ana.b@gm|ail.com") + ".txt", "app.bundle!notes/<withheld 3>"),
                        ("quilmar/vesk.md", "<withheld 1>/<withheld 2>"),
                        ("docs/readme.md", "docs/readme.md")):
        if shown_path(label, sc) != want:
            fails.append("self-test: path %r shown as %r, want %r" % (label, shown_path(label, sc), want))
    got = line_report("docs/" + live("ana.b@gm|ail.com") + ".md", 1, live("the real| rows that"), sc)
    if [t for _c, t in got] != [live("docs/<withheld 2>:1  [provenance: real| rows that]  the real| rows that")]:
        fails.append("self-test: a FAIL line's path is shown withheld, got %s" % got)
    # A term is counted once per place it stands, however many ways it is read.
    got = [c for c, _ in line_findings(live(r"a\nzorblax|quent b"), sc)]
    if got != ["term"]:
        fails.append("self-test: a term after an escape is one hit, got %s" % got)
    # Without a key the term class is off and nothing else changes.
-    keyless = TermScanner(sc.terms, None)
+    planted = live("the zorblax|quent ledger, entity_id=|9101, x@proto|nmail.com")
-    got = {c for c, _ in line_findings(live("the zorblax|quent ledger, entity_id=|3, x@proto|nmail.com"), keyless)}
+    keyless = TermScanner(sc.terms, None, probe_ids)
    got = {c for c, _ in line_findings(planted, keyless)}
    if got != {"entity-id", "email"}:
        fails.append("self-test: without a key only the term class is off, got %s" % sorted(got))
    # Without the ids the entity-id class is off and nothing else changes.
    got = {c for c, _ in line_findings(planted, TermScanner(sc.terms, probe_key))}
    if got != {"term", "email"}:
        fails.append("self-test: without the ids only the entity-id class is off, got %s" % sorted(got))
    # A key other than the one TERMS was made with is told apart from it.
    if key_matches(probe_key) or key_matches(b""):
        fails.append("self-test: a key TERMS was not made with passes the key check")
@@ -426,6 +641,18 @@ def self_test():
        if len(entries) != 1 or len(errors) != 2:
            fails.append("self-test: the allow file reader kept %d entries and %d errors, want 1 and 2"
                         % (len(entries), len(errors)))
        # The reserved ids are read one per line; a missing file turns the
        # class off, and a file that would match nothing is an error.
        idf = os.path.join(td, "ids")
        for body, want in (("# reserved\n9101\n\n 9202 \n", ((9101, 9202), False)),
                           ("9101\nentity-9202\n", (None, True)), ("# none\n\n", (None, True)), ("", (None, True))):
            with open(idf, "w", encoding="utf-8") as f:
                f.write(body)
            ids, err = load_reserved_ids(idf)
            if (ids, err is not None) != want:
                fails.append("self-test: reserved ids %r read as %r, %r" % (body, ids, err))
        if load_reserved_ids(os.path.join(td, "absent")) != (None, None):
            fails.append("self-test: a missing reserved-ids file is not (None, None)")
        # A zip-format bundle is opened by its signature, whatever it is named
        # (an app bundle, or a .zip such as a browser extension a site serves),
        # and its text members are read.
@@ -469,13 +696,20 @@ def main(argv) -> int:
    key = load_key()
    if key is None:
        print("  NOTE  term check skipped: no key at %s (set NO_PERSONAL_DATA_KEY_FILE); "
-              "the entity-id, provenance, email and phone checks still run" % KEY_FILE)
+              "the other checks still run" % KEY_FILE)
    elif not key_matches(key):
        fails.append("the key at %s is not the one TERMS was made with, so it would match "
                     "nothing: point NO_PERSONAL_DATA_KEY_FILE at that key, or remake TERMS and KEY_CHECK" % KEY_FILE)
        print("  FAIL  %s" % fails[-1])
        key = None
-    scanner = TermScanner(TERMS, key)
+    ids, ids_error = load_reserved_ids()
    if ids_error:
        fails.append(ids_error)
        print("  FAIL  %s" % ids_error)
    elif ids is None:
        print("  NOTE  entity-id check skipped: no reserved ids at %s (set RESERVED_ENTITY_IDS_FILE); "
              "the other checks still run" % RESERVED_ENTITY_IDS_FILE)
    scanner = TermScanner(TERMS, key, ids)
    used = set()
    findings = []
    counts = {}
@@ -488,21 +722,24 @@ def main(argv) -> int:
            continue
        for cls, what in line_findings(rel, scanner):
            if not allowed(allow, rel, cls, rel, used):
-                findings.append("%s  [%s: %s]  (file name)" % (rel, cls, what))
+                findings.append("%s  [%s: %s]  (file name)" % (shown_path(rel, scanner), cls, what))
                counts[cls] = counts.get(cls, 0) + 1
        for label, text in texts(root, rel):
            n_texts += 1
            if text is None:
                fails.append("%s has a zip signature but does not open as a zip, so its "
-                             "members cannot be read" % label)
+                             "members cannot be read" % shown_path(label, scanner))
                print("  FAIL  %s" % fails[-1])
                continue
            # A bundle member's name installs with it, like a file name.
            member = label[len(rel) + 1:]
            for cls, what in (line_findings(member, scanner) if member else ()):
                if not allowed(allow, label, cls, member, used):
                    findings.append("%s  [%s: %s]  (bundle member name)" % (shown_path(label, scanner), cls, what))
                    counts[cls] = counts.get(cls, 0) + 1
            for i, line in enumerate(text.splitlines(), 1):
-                for part, cls, what in line_hits(line, scanner):
+                for cls, finding in line_report(label, i, line, scanner, allow, used, label == allow_rel):
-                    if (cls == "entity-id" and label == allow_rel) or allowed(allow, label, cls, part, used):
+                    findings.append(finding)
                        continue
                    shown = "" if cls in _QUIET else part.strip()[:140]
                    findings.append(("%s:%d  [%s: %s]  %s" % (label, i, cls, what, shown)).rstrip())
                    counts[cls] = counts.get(cls, 0) + 1
    for f in findings:
        print("  FAIL  %s" % f)
Author	SHA1	Message	Date
Privatae	3ae3a2841c	bastion agent 0.1.17 ci / personal-data (push) Successful in 3s Details ci / test (push) Successful in 45s Details ci / reproducible (push) Successful in 3s Details	2026-10-29 21:42:06 +00:00
Privatae	e3074f50b0	bastion agent 0.1.16 ci / test (push) Successful in 1m1s Details ci / personal-data (push) Successful in 3s Details ci / reproducible (push) Successful in 4s Details	2026-10-25 23:16:50 +00:00