bastion agent 0.1.16

2026-10-25 23:16:50 +00:00
34 changed files with 1345 additions and 156 deletions
@@ -17,6 +17,8 @@ env:
  PY_CURRENT: python@sha256:7c61056e61ac89e852de05f3dc6fa51a6dd2181797bceed46aa725dd7cb2cd3b  # 3.13-slim
  # The PyYAML release bastion_agent/build.py vendors; the manifest loader imports it.
  PYYAML: pyyaml==6.0.3
  # The linter: every name a module imports is used, and none is undefined.
  PYFLAKES: pyflakes==4.0.3
  # The pull request's own head, or the pushed commit.
  WANT: ${{ gitea.event.pull_request.head.sha || gitea.sha }}
@@ -36,17 +38,27 @@ jobs:
        run: |
          docker run --rm -v "$PWD":/src:ro -w /src "$PY_CURRENT" \
            python -m unittest -v test_syntax_floor
      - name: pyflakes
        run: |
          docker run --rm -v "$PWD":/src:ro -w /src -e PYFLAKES "$PY_CURRENT" sh -euc '
            pip install -q --disable-pip-version-check --root-user-action=ignore "$PYFLAKES"
            python -m pyflakes bastion_agent pii scripts test_syntax_floor.py'
      - name: every test, on Python 3.8 and on a current Python
-        # age is installed so the seal round trips run instead of skipping.
+        # age is installed so the seal round trips run, and git so the release
        # verifier's end-to-end tests do. Every test runs here: a skipped one
        # fails the job (the only skips the suite has are for Windows, or for a
        # tool missing, and this job installs the tools).
        run: |
          set -euo pipefail
          for img in "$PY_FLOOR" "$PY_CURRENT"; do
            docker run --rm -v "$PWD":/src:ro -w /src -e PYYAML "$img" sh -euc '
              python --version
              apt-get -qq update >/dev/null
-              apt-get -qq install -y --no-install-recommends age >/dev/null
+              apt-get -qq install -y --no-install-recommends age git >/dev/null
              pip install -q --disable-pip-version-check --root-user-action=ignore "$PYYAML"
-              python -m unittest discover .'
+              python -m unittest discover -v . > /tmp/tests.log 2>&1 || { cat /tmp/tests.log; exit 1; }
              grep -E "^(Ran|OK|FAILED)" /tmp/tests.log
              if grep -E "\.\.\. skipped|^(OK|FAILED) \(.*skipped=" /tmp/tests.log; then echo "a test was skipped; CI runs every test"; exit 1; fi'
          done
  personal-data:
@@ -62,9 +74,14 @@ jobs:
          test "$(git rev-parse HEAD)" = "$WANT"
      - name: no person's data in the tree
        # Reads files, runs nothing from the tree. The term check needs the
-        # maintainers' key, which a CI job never holds, so here it is skipped and
+        # maintainers' key and the entity-id check the reserved ids, which a CI
-        # the run says so; the entity-id, provenance, email and phone checks run.
+        # job never holds, so the job names files that do not exist: both
-        run: python3 scripts/test_no_personal_data.py
+        # classes are skipped and the run prints a NOTE for each; the
        # provenance, email and phone checks run. Left at their defaults, a
        # path that exists but cannot be read fails the run instead.
        run: |
          none="$PWD/.ci-holds-none/none"
          NO_PERSONAL_DATA_KEY_FILE="$none" RESERVED_ENTITY_IDS_FILE="$none" python3 scripts/test_no_personal_data.py
  reproducible:
    needs: test
@@ -0,0 +1,116 @@
 # The checks of .gitea/workflows/ci.yml, for the copy of this repository on
 # GitHub. Gitea reads .gitea/workflows when it exists and then ignores this
 # directory, so each forge runs one of the two files, never both.
 #
 # The jobs, images and commands are the same as there. No action from the
 # marketplace is used: the commit is fetched with git from this repository's
 # public URL, so nothing outside this file and the pinned images runs. The job
 # token can only read.
 name: ci
 on:
  push:
    branches: [main]
  pull_request:
    branches: [main]
 permissions:
  contents: read
 env:
  PY_FLOOR: python@sha256:1d52838af602b4b5a831beb13a0e4d073280665ea7be7f69ce2382f29c5a613f    # 3.8-slim
  PY_CURRENT: python@sha256:7c61056e61ac89e852de05f3dc6fa51a6dd2181797bceed46aa725dd7cb2cd3b  # 3.13-slim
  # The PyYAML release bastion_agent/build.py vendors; the manifest loader imports it.
  PYYAML: pyyaml==6.0.3
  # The linter: every name a module imports is used, and none is undefined.
  PYFLAKES: pyflakes==4.0.3
  # The pull request's own head, or the pushed commit.
  WANT: ${{ github.event.pull_request.head.sha || github.sha }}
  REPO_URL: ${{ github.server_url }}/${{ github.repository }}.git
 jobs:
  test:
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    steps:
      - name: fetch the commit
        run: |
          set -euo pipefail
          git init -q .
          git fetch -q --depth=1 "$REPO_URL" "$WANT"
          git checkout -q --detach FETCH_HEAD
          test "$(git rev-parse HEAD)" = "$WANT"
      - name: the 3.8 syntax floor
        run: |
          docker run --rm -v "$PWD":/src:ro -w /src "$PY_CURRENT" \
            python -m unittest -v test_syntax_floor
      - name: pyflakes
        run: |
          docker run --rm -v "$PWD":/src:ro -w /src -e PYFLAKES "$PY_CURRENT" sh -euc '
            pip install -q --disable-pip-version-check --root-user-action=ignore "$PYFLAKES"
            python -m pyflakes bastion_agent pii scripts test_syntax_floor.py'
      - name: every test, on Python 3.8 and on a current Python
        # age is installed so the seal round trips run, and git so the release
        # verifier's end-to-end tests do. Every test runs here: a skipped one
        # fails the job (the only skips the suite has are for Windows, or for a
        # tool missing, and this job installs the tools).
        run: |
          set -euo pipefail
          for img in "$PY_FLOOR" "$PY_CURRENT"; do
            docker run --rm -v "$PWD":/src:ro -w /src -e PYYAML "$img" sh -euc '
              python --version
              apt-get -qq update >/dev/null
              apt-get -qq install -y --no-install-recommends age git >/dev/null
              pip install -q --disable-pip-version-check --root-user-action=ignore "$PYYAML"
              python -m unittest discover -v . > /tmp/tests.log 2>&1 || { cat /tmp/tests.log; exit 1; }
              grep -E "^(Ran|OK|FAILED)" /tmp/tests.log
              if grep -E "\.\.\. skipped|^(OK|FAILED) \(.*skipped=" /tmp/tests.log; then echo "a test was skipped; CI runs every test"; exit 1; fi'
          done
  personal-data:
    runs-on: ubuntu-24.04
    timeout-minutes: 10
    steps:
      - name: fetch the commit
        run: |
          set -euo pipefail
          git init -q .
          git fetch -q --depth=1 "$REPO_URL" "$WANT"
          git checkout -q --detach FETCH_HEAD
          test "$(git rev-parse HEAD)" = "$WANT"
      - name: no person's data in the tree
        # Reads files, runs nothing from the tree. The term check needs the
        # maintainers' key and the entity-id check the reserved ids, which
        # this job never holds, so both are skipped and the run prints a NOTE
        # for each; the provenance, email and phone checks run.
        run: |
          set -euo pipefail
          docker run --rm -v "$PWD":/src:ro -w /src "$PY_CURRENT" \
            python -I scripts/test_no_personal_data.py --root /src
  reproducible:
    needs: test
    runs-on: ubuntu-24.04
    timeout-minutes: 20
    steps:
      - name: fetch the commit
        run: |
          set -euo pipefail
          git init -q .
          git fetch -q --depth=1 "$REPO_URL" "$WANT"
          git checkout -q --detach FETCH_HEAD
          test "$(git rev-parse HEAD)" = "$WANT"
      - name: two builds on each Python produce one sha256
        # Anyone must be able to rebuild a release and get the published bytes,
        # so the .pyz may not depend on the build or the Python that made it.
        run: |
          set -euo pipefail
          for img in "$PY_FLOOR" "$PY_CURRENT"; do
            docker run --rm -v "$PWD":/src:ro -w /src -e PYYAML "$img" sh -euc '
              pip install -q --disable-pip-version-check --root-user-action=ignore "$PYYAML"
              python -m bastion_agent.build --out /tmp/a | awk "{print \$NF}"
              python -m bastion_agent.build --out /tmp/b | awk "{print \$NF}"'
          done | tee digests
          test "$(wc -l < digests)" -eq 4
          test "$(sort -u digests | wc -l)" -eq 1
          echo "every build is $(head -1 digests)"
@@ -0,0 +1,174 @@
 # Attach the release files to the GitHub release of a tag v<version>.
 #
 # GitHub only: Gitea reads .gitea/workflows when it exists and then ignores
 # this directory, so the forge's copy of this repository never runs this file.
 #
 # The files are not uploaded from anywhere else. This job rebuilds them from
 # the tag and attaches them only when they are the published release:
 #
 #   1. bastion_agent/verify_release.py fetches the live manifest
 #      (privatae.ai/bastion-agent/version.json) and its signature, checks the
 #      signature against the release keys the bastion pins, refuses unless the
 #      manifest names this tag's version and the tag is annotated with the line
 #      `built from <built_from>`, rebuilds the .pyz from the tag with that
 #      tag's own build.py, and compares the sha256 with the manifest;
 #   2. only then are the rebuilt .pyz, its .sha256, the manifest (its exact
 #      bytes) and the signature file attached to the release, which is created
 #      when the tag has none, with that version's CHANGELOG section.
 #
 # The live manifest names only the newest release, so a release can be given
 # its files only while it is the published one. Older releases, whose manifest
 # is no longer served, cannot be filled in afterwards. A run that failed (the
 # tag pushed before the release went live, say) is run again from the Actions
 # tab with the tag as input.
 #
 # No action from the marketplace is used, and nothing is fetched with a
 # credential: the repository is read from its public URL. The job may write
 # to the repository's releases; the token is handed to the last step only.
 name: release
 on:
  push:
    tags: ["v*.*.*"]
  workflow_dispatch:
    inputs:
      tag:
        description: "the release tag, vX.Y.Z"
        required: true
 permissions:
  contents: read
 env:
  REPO_URL: ${{ github.server_url }}/${{ github.repository }}.git
  TAG: ${{ github.event.inputs.tag || github.ref_name }}
 defaults:
  run:
    shell: bash -euo pipefail {0}
 jobs:
  release:
    runs-on: ubuntu-24.04
    timeout-minutes: 20
    permissions:
      contents: write
    steps:
      - name: the tag is a release tag
        run: |
          printf '%s\n' "$TAG" | grep -Exq 'v[0-9]+\.[0-9]+\.[0-9]+' \
            || { echo "not a release tag: $TAG"; exit 1; }
      - name: fetch the repository and its tags
        # verify_release.py is run from the commit this workflow runs from: the
        # tag itself on a tag push, the default branch on a manual run. It
        # builds the tag's own source in a separate worktree.
        run: |
          git init -q .
          git remote add origin "$REPO_URL"
          git fetch -q origin "+refs/tags/*:refs/tags/*" "$GITHUB_SHA"
          git checkout -q --detach "$GITHUB_SHA"
          test "$(git rev-parse HEAD)" = "$GITHUB_SHA"
          git rev-parse -q --verify "refs/tags/$TAG^{commit}" >/dev/null \
            || { echo "no tag $TAG in this repository"; exit 1; }
      - name: check the signed manifest and rebuild the .pyz from the tag
        run: |
          # The PyYAML release bastion_agent/build.py vendors, as a wheel only
          # and only with one of these sha256s (PyPI's, for Linux on x86_64 and
          # arm64, Python 3.12 and 3.13): the one package this job takes from
          # outside the repository, and this step runs before the upload step
          # that holds the token. A different PyYAML would only make the
          # rebuild differ and the step refuse.
          cat > "$RUNNER_TEMP/requirements.txt" <<'REQ'
          pyyaml==6.0.3 \
            --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \
            --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \
            --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \
            --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c
          REQ
          /usr/bin/python3 -m venv /tmp/venv
          /tmp/venv/bin/pip install -q --disable-pip-version-check --only-binary :all: \
            --require-hashes -r "$RUNNER_TEMP/requirements.txt"
          /tmp/venv/bin/python -m bastion_agent.verify_release \
            --expect-version "$TAG" --out "$RUNNER_TEMP/release"
          ls -l "$RUNNER_TEMP/release"
      - name: the release notes, from that version's CHANGELOG section
        run: |
          /usr/bin/git show "$TAG:CHANGELOG.md" > "$RUNNER_TEMP/CHANGELOG.md"
          /usr/bin/python3 -I - "${TAG#v}" "$RUNNER_TEMP/CHANGELOG.md" > "$RUNNER_TEMP/notes.md" <<'PY'
          import re, sys
          ver, path = sys.argv[1], sys.argv[2]
          text = open(path, encoding="utf-8").read()
          m = re.search(r"^## %s[ \t]*\n(.*?)(?=^## |\Z)" % re.escape(ver), text, re.M | re.S)
          if not m:
              sys.exit("CHANGELOG.md at the tag has no section for %s" % ver)
          print(m.group(1).strip())
          print()
          print("---")
          print("The files below are this tag rebuilt by this repository's release workflow, "
                "attached only after their sha256 matched the signed manifest "
                "(`version.json` and its `.sig`). To check them yourself, run "
                "`python3 -m bastion_agent.verify_release` from a clone: "
                "see [Verify a release](https://github.com/%s#verify-a-release)."
                % "Privatae/bastion-agent")
          PY
      - name: attach the files to the release
        # The one step with the token. Its shell and Python are named by path,
        # and Python runs isolated (-I), so nothing an earlier step put on PATH
        # or in the environment chooses what runs here.
        shell: /usr/bin/bash -euo pipefail {0}
        env:
          GH_TOKEN: ${{ github.token }}
          GH_REPO: ${{ github.repository }}
        run: |
          /usr/bin/python3 -I - "$RUNNER_TEMP/release" "$RUNNER_TEMP/notes.md" <<'PY'
          import json, os, sys, urllib.error, urllib.parse, urllib.request
          out_dir, notes = sys.argv[1], sys.argv[2]
          api = os.environ.get("GITHUB_API_URL", "https://api.github.com").rstrip("/")
          repo, tag, token = os.environ["GH_REPO"], os.environ["TAG"], os.environ["GH_TOKEN"]
          def call(method, url, body=None, ctype="application/json"):
              data = body if isinstance(body, bytes) or body is None else json.dumps(body).encode()
              req = urllib.request.Request(url, data=data, method=method, headers={
                  "Authorization": "Bearer " + token, "Accept": "application/vnd.github+json",
                  "X-GitHub-Api-Version": "2022-11-28", "Content-Type": ctype,
                  "User-Agent": "bastion-agent-release"})
              try:
                  with urllib.request.urlopen(req, timeout=120) as r:
                      raw = r.read()
                      return r.status, json.loads(raw) if raw else None
              except urllib.error.HTTPError as e:
                  return e.code, None
          st, rel = call("GET", "%s/repos/%s/releases/tags/%s" % (api, repo, tag))
          if st == 404:
              body = open(notes, encoding="utf-8").read()
              st, rel = call("POST", "%s/repos/%s/releases" % (api, repo), {
                  "tag_name": tag, "name": "bastion agent " + tag[1:], "body": body,
                  "draft": False, "prerelease": False})
              if st != 201:
                  sys.exit("creating the release for %s failed: HTTP %d" % (tag, st))
              print("created the release for %s" % tag)
          elif st != 200:
              sys.exit("reading the release for %s failed: HTTP %d" % (tag, st))
          else:
              print("the release for %s exists; its text is left as it is" % tag)
          st, assets = call("GET", "%s/repos/%s/releases/%d/assets?per_page=100" % (api, repo, rel["id"]))
          if st != 200:
              sys.exit("listing the release's files failed: HTTP %d" % st)
          have = {a["name"]: a["id"] for a in assets}
          upload = rel["upload_url"].split("{", 1)[0]
          for name in sorted(os.listdir(out_dir)):
              if name in have:
                  st, _ = call("DELETE", "%s/repos/%s/releases/assets/%d" % (api, repo, have[name]))
                  if st != 204:
                      sys.exit("replacing %s failed: HTTP %d" % (name, st))
              with open(os.path.join(out_dir, name), "rb") as fh:
                  data = fh.read()
              st, a = call("POST", upload + "?" + urllib.parse.urlencode({"name": name}), data,
                           "application/octet-stream")
              if st != 201:
                  sys.exit("uploading %s failed: HTTP %d" % (name, st))
              print("attached %s (%d bytes)" % (a["name"], a["size"]))
          PY
@@ -14,6 +14,89 @@ Changes to the release tooling (`publish.sh`, `build.py`, `verify_release.py`)
 and to the documentation are listed under the version whose source first
 carried them, although they do not change what the `.pyz` does.
 ## 0.1.16
 - **Every `search` and `read_range` reply names its sanitize mode**
  (`sanitize_mode`: `flag` or `sterilize`, the mode the sanitize stage
  recorded for the text in the reply; `unknown` when that record does not
  vouch for it; a search with no hits names the manifest's mode, or `none`),
  so the bastion and the Host can say what was done to the text. Switching
  the manifest's mode does not relabel text an earlier sweep wrote. In `flag`
  mode, the mode the bastion deploys, names, street addresses and dates of
  birth remain. The field is new; nothing else in either reply changes.
 - **Each GitHub release carries its files.** Pushing a release tag starts
  `.github/workflows/release.yml`, which runs the release check against the
  published manifest, rebuilds the `.pyz` from the tag, and only when the
  sha256 matches attaches the `.pyz`, its `.sha256`, `version.json` and its
  signature to the release (making the release, with this file's section for
  it, when the tag has none). It can be run again by hand for the version
  the manifest names; older releases, whose manifest is no longer served,
  cannot be given their files afterwards. The manifest's `notes` now link to
  that GitHub release instead of the tag page on the forge.
 - **`verify_release` refuses a tag without its release record** (an annotated
  tag with the line `built from <built_from>`, as every release tag from
  0.1.14 on has), instead of building from any tag of that name. It gains
  `--expect-version`, which refuses a manifest naming another version, and
  `--out DIR`, which writes the rebuilt `.pyz` and `.sha256`, the manifest's
  exact bytes and its signature once everything matched.
 - **CI runs every test and pyflakes.** The test job installs `git` too, so
  the release verifier's seven end-to-end tests, which were skipped in CI, now
  run, and a skipped test fails the job. pyflakes 4.0.3 runs on both forges;
  the 31 findings it had (unused imports and variables) are fixed, and the
  packages' re-exports are listed in `__all__`.
 - **The README says what it means.** The agent is deployed from a grant and
  can also be run by hand; what crosses the tunnel is the catalog cards and
  sanitized excerpts; building and testing need PyYAML 6.0.3; the CI badge is
  the GitHub one.
 - **The rules pass finds more personal data on its own,** which matters
  because it is all that `flag` mode, the mode the bastion deploys, runs.
  Newly found: international phone numbers (`+44 7700 900123`,
  `+49 (0)30 1234567`, no separators, dots or dashes), national numbers with a
  trunk `0` (`07700 900123`), a seven-digit local number after a word such as
  call or phone, IBANs written in lower case, card numbers written with dots,
  SSNs written with spaces, and nine-digit SSNs with no separators when `SSN`
  or `social security` comes just before them. A bare nine-digit number, a
  seven-digit number with no phone word before it (`Order 555-0142`), a number
  that fails its checksum and a date are still left alone; a reference number
  that starts with `0` (`0123-456789`) is now taken for a national phone
  number. An expiry date or security code beside a card is still not found.
 - **A dotted number right after a version word is no longer taken for an IP
  address** (`version 1.2.3.4`), and SSNs that are never issued (group `00`,
  serial `0000`) are no longer masked.
 - **Every file is sanitized again once after this upgrade** (`SANITIZE_VERSION`
  3), so text kept by the older rules is redone.
 - **The docs say what the deployed mode leaves in.** The README, SECURITY.md
  and the sanitize, serve and PII docs say plainly that `flag` mode does not
  find names, street addresses or dates of birth, and that `sterilize` mode
  adds a model that looks for them.
 - **The PII service listens on 127.0.0.1 by default** instead of every
  interface; `PII_BIND` still chooses another address. Its container image runs
  as uid 10001 instead of root, so a volume mounted on `/data` must be writable
  by that user, and a container that should be reachable needs `PII_BIND` set.
 - **CI also runs on GitHub** (`.github/workflows/ci.yml`): the same jobs, the
  same pinned images.
 - **The README's license paragraph and SECURITY.md name each kind of Host**
  (docs only) instead of only Privatae's platform: a
  C-Series Host runs on your own machine and is open source (Privatae/cseries,
  AGPL-3.0-only), C-Series Cloud and a network membership are optional and
  paid, and a Host on Privatae's own platform is the case where the Host's
  software is not open source.
 - **Security reports go to security@privatae.ai,** the address
  privatae.ai/.well-known/security.txt names, and SECURITY.md links the
  bastion's policy on GitHub, where anyone can read it.
 - **How this repository is published.** From this release on, GitHub receives
  each change merged since the last release as its own commit, titled as the
  change was, instead of one commit per release, and the README says so.
 - **The personal-data check reads what it used to miss.** A listed term
  written right after a backslash escape (`\n`, `\t`, `\u00e9`) is found, and
  so is any value inside a run of base64 or base64url that decodes to text.
  The reserved entity ids are read from `/etc/privatae/reserved-entity-ids`
  (one per line, `RESERVED_ENTITY_IDS_FILE` to point elsewhere) instead of
  being written in the script; where that file is absent, as in public CI,
  the entity-id check is skipped and the run prints a NOTE, as the term check
  already does without its key. A file that holds no id, or a line that is
  not one, fails the run.
 ## 0.1.15
 - **No change to what the agent does.** The README says how this repository
@@ -11,6 +11,7 @@ From the repository root:
 ```sh
 python3 -m unittest discover .                    # every test, including the 3.8 floor
 python3 -m pyflakes bastion_agent pii scripts test_syntax_floor.py   # pyflakes==4.0.3, as CI runs it
 python3 -m bastion_agent.build --out /tmp/agent   # the .pyz and its .sha256
 python3 /tmp/agent/bastion-agent-*.pyz --version
 bash -n bastion_agent/publish.sh pii/verify-live.sh && sh -n bastion_agent/install.sh
@@ -19,7 +20,10 @@ bash -n bastion_agent/publish.sh pii/verify-live.sh && sh -n bastion_agent/insta
 - **All tests pass.** They need PyYAML (the manifest loader uses it), at the
  version `build.py` pins (`PYYAML_VERSION`) for the build tests. The
  round-trip seal tests are skipped where `age` and `age-keygen` are not
-  installed, so install them to run the whole suite.
+  installed, the TLS tests where `openssl` is not, and the release verifier's
  end-to-end tests where `git` is not, so install them to run the whole suite.
  CI installs them and fails on any skipped test.
 - **pyflakes finds nothing.** No unused import or variable, no undefined name.
 - **The Python 3.8 floor.** `test_syntax_floor.py` parses every module in
  `bastion_agent/` and `pii/` at 3.8 and denies the newer standard-library
  calls that parse on 3.8 and then fail at run time. The agent ships to
@@ -1,16 +1,18 @@
 # bastion agent
-[![ci](https://git.privatae.ai/privatae/bastion-agent/actions/workflows/ci.yml/badge.svg?branch=main)](https://git.privatae.ai/privatae/bastion-agent/actions)
+[![ci](https://github.com/Privatae/bastion-agent/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/Privatae/bastion-agent/actions/workflows/ci.yml)
 The small program the bastion runs on your own machine, for one folder.
-It is deployed **from a grant, never installed by a person**: the bastion fetches
+It is deployed **from a grant**, and you do not install it yourself: the bastion fetches
 `privatae.ai/bastion-agent/version.json`, checks its ed25519 release signature
 (`version.json.<first 16 hex of its sha256>.sig`) against the keys compiled into the bastion, downloads the `.pyz`
 and checks it against the sha256 in that signed manifest,
 mints a P-256 identity per store, writes and signs a manifest from the grant you
-approved, and supervises `sweep` and `serve`. Your files never leave the machine;
+approved, and supervises `sweep` and `serve`. Your files stay on the machine;
-what crosses the tunnel is the index and what the grant's mask allows.
+what crosses the tunnel is what the grant's mask allows: catalog cards, and
 the sanitized excerpts that `search` and `read_range` return, each reply naming
 the sanitize mode that made it (`sanitize_mode`, [serve.md](docs/serve.md)).
 The published version is whatever `privatae.ai/bastion-agent/version.json`
 names. `bastion_agent/VERSION` on `main` is the next release, which may not be
@@ -18,7 +20,7 @@ published yet. Release manifests are published signed from 0.1.13 on, and the
 `.pyz` can be rebuilt from this source and compared
 ([Verify a release](#verify-a-release)).
-This repository is free software, licensed under the GNU General Public License version 3 or (at your option) any later version (see `LICENSE`). Copyright Privatae LLC. You may run it, read it, change it and redistribute it under that license. The Privatae platform the bastion connects to is separate, proprietary software and is not covered by this license. A member runs their own build on their own machine — nothing here depends on a Privatae-held credential or a Privatae-side build.
+This repository is free software, licensed under the GNU General Public License version 3 or (at your option) any later version (see `LICENSE`). Copyright Privatae LLC. You may run it, read it, change it and redistribute it under that license. The Host the bastion connects to is separate software and is not covered by this license: a C-Series Host is [Privatae/cseries](https://github.com/Privatae/cseries), under AGPL-3.0-only, and a Host on Privatae's own platform is proprietary software. A member runs their own build on their own machine — nothing here depends on a Privatae-held credential or a Privatae-side build.
 ## What it does
@@ -30,13 +32,13 @@ failure stops the run:
 |---|---|---|
 | walk | the map of the folder: files, sizes, types, what the grant excludes | [walk.md](docs/walk.md) |
 | extract | text out of each file, with built-in extractors or local tools | [extract.md](docs/extract.md) |
-| sanitize | finds personal data and replaces it with tokens; the originals stay in a vault on this machine | [sanitize.md](docs/sanitize.md), [pii-sterilization.md](docs/pii-sterilization.md) |
+| sanitize | replaces the personal data its mode finds with tokens; the originals stay in a vault on this machine. In the mode the bastion deploys it does not find names, street addresses or dates of birth ([below](#how-the-bastion-runs-it)) | [sanitize.md](docs/sanitize.md), [pii-sterilization.md](docs/pii-sterilization.md) |
 | chunk, embed | splits the text and embeds it with a model on this machine | [embed.md](docs/embed.md) |
 | catalog | the curated dataset built from the stages above | [catalog.md](docs/catalog.md) |
 | seal | packs an allowlisted bundle of the catalog and encrypts it with age to the owner and the manifest's named recipients; the one way a dataset leaves the machine | [seal.md](docs/seal.md) |
-`serve` answers queries over the index with mutual TLS, returning sanitized
+`serve` answers queries over the index with mutual TLS, returning chunks and
-chunks and catalog cards ([serve.md](docs/serve.md)). The gate refuses an
+catalog cards from the sanitized text ([serve.md](docs/serve.md)). The gate refuses an
 unsigned, edited or foreign manifest, and the bastion never starts the agent
 without a valid one. (`--allow-unsigned` exists for development only; the
 bastion never passes it.) See [runtime.md](docs/runtime.md) and
@@ -45,12 +47,20 @@ bastion never passes it.) See [runtime.md](docs/runtime.md) and
 ## How the bastion runs it
 The bastion installs and runs the agent itself from a folder grant you
-approved; nobody installs it by hand. For each shared folder the bastion:
+approved; you do not install it by hand. (The `.pyz` can also be installed and
 run by hand, to try it outside the bastion: [packaging.md](docs/packaging.md).)
 For each shared folder the bastion:
 - downloads the `.pyz` only through the signed manifest, and never replaces an
  installed agent with an older version;
 - mints the agent its own identity and writes and signs its manifest from the
  grant, so the agent works only on what you granted;
 - sets the sanitize stage to `flag` mode: rules only, no model. It replaces
  e-mail addresses, phone numbers, card, bank and social security numbers, IP
  and MAC addresses and keys. **It does not find names, street addresses or
  dates of birth**, so text a Host receives from the agent still contains them.
  `sterilize` mode adds a model on this machine that looks for those, and the
  bastion does not set it ([sanitize.md](docs/sanitize.md#modes-manifest-sanitizemode));
 - points the embed stage at the model server on this machine (a local ollama)
  and at nothing else;
 - runs `serve` on the loopback address only, with mutual TLS, so the bastion is
@@ -81,12 +91,16 @@ package, which is what the built artifact has always looked like inside.
 ## Build and test
 ```sh
 pip install pyyaml==6.0.3                # the PyYAML build.py vendors; the manifest loader imports it
 python3 -m unittest discover .          # every module's tests, plus the 3.8 floor
 python3 -m bastion_agent.build --out dist   # produces dist/bastion-agent-<ver>.pyz + .sha256
 python3 dist/bastion-agent-<ver>.pyz --version
 ```
-Every push runs the same checks in CI (`.gitea/workflows/ci.yml`).
+A test that needs `age`, `openssl` or `git` is skipped where that tool is not
 installed. Every push runs the same checks in CI (`.gitea/workflows/ci.yml` on
 git.privatae.ai, `.github/workflows/ci.yml` on GitHub), with the tools
 installed, so there no test is skipped, plus `pyflakes`.
 The build is reproducible: the same commit gives the same bytes on any machine
 with PyYAML at the pinned version (`PYYAML_VERSION` in `build.py`).
@@ -113,8 +127,9 @@ it); no key or account of ours. It:
   `bastion_agent/verify_release.py`, copied from `client-go/release_sign.go`
   in the bastion's repository;
 2. checks out the source of that version in a separate worktree: the tag
-   `v<version>`, or else the commit the manifest names as `built_from` if the
+   `v<version>`, which must be annotated with the line
-   clone has it;
+   `built from <built_from>`, or else the commit the manifest names as
   `built_from` if the clone has it;
 3. rebuilds the `.pyz` with that version's own `build.py`, and compares its
   sha256 with every `.pyz` the manifest lists.
@@ -122,21 +137,38 @@ A match means the file every bastion downloads is exactly what this source
 produces, and the signature means the release key holder published it. The
 exit code is 0 only when both hold.
-The public repository is a copy of each release's source, not the development
+From 0.1.16 on, each GitHub release carries the `.pyz`, its `.sha256`, the
-history, so `built_from` names a commit that exists only in the development
+signed manifest `version.json` and its signature `version.json.<h>.sig`. The
-repository. From this release on, each public snapshot carries the tag
+release workflow (`.github/workflows/release.yml`) attaches them when the tag
-`v<version>`, which marks the source the release was built from. A snapshot
+is pushed, after running this same check against the published manifest and
-published before that has no tag, and can be checked from `built_from` only
+rebuilding the `.pyz` from the tag; it attaches nothing if a digest differs.
-where that commit is available. A manifest from before signing has no
+To check downloaded files yourself, `sha256sum -c bastion-agent-<ver>.pyz.sha256`
 checks the `.pyz` against its `.sha256`, the manifest lists the sha256 of both,
 and `check_signature` in `bastion_agent/verify_release.py` checks the manifest
 against the pinned release keys:
 ```sh
 python3 -c 'import sys; from bastion_agent import verify_release as v; print(v.check_signature(open(sys.argv[1], "rb").read(), open(sys.argv[2]).read())[0]["files"])' version.json version.json.*.sig
 ```
 The public repository does not carry the development history, so
 `built_from` names a commit that exists only in the development repository.
 Each published release from 0.1.14 on carries the tag `v<version>`, which
 marks the source the release was built from. Earlier releases have no tag
 here, and can be checked from `built_from` only where that commit is
 available. A manifest from before signing has no
 signature file beside it, and the check refuses it.
 ## How this repo is published
-Development happens in a private repository. This repository receives the
+Development happens in a private repository, and its history is not
-source of each published release as one snapshot commit, tagged
+published. When a release is published, each change merged since the last one
-`v<version>`, whose tree is the source the signed manifest says the release
+arrives on GitHub, at
-was built from; that is why its history is short. The same snapshot is on
+[github.com/Privatae/bastion-agent](https://github.com/Privatae/bastion-agent),
-GitHub at [github.com/Privatae/bastion-agent](https://github.com/Privatae/bastion-agent).
+as a commit with the same files, titled as the change was and dated when it
 was merged; the copy at git.privatae.ai receives the release as one commit.
 In both, the release is tagged `v<version>` on a commit whose tree is the
 source the signed manifest says the release was built from.
 Every published `.pyz` can be rebuilt from its tag and compared
 ([Verify a release](#verify-a-release)).
@@ -4,7 +4,7 @@
 Please do not open a public issue for a security problem.
-Email **info@privatae.ai** with:
+Email **security@privatae.ai** with:
 - the agent's version (`python3 bastion-agent-<version>.pyz --version`) and
  platform,
@@ -50,13 +50,22 @@ example:
 - **The bastion**, which installs, updates and supervises the agent, signs its
  manifest and holds its keys. It is a separate program with its own policy:
-  [privatae-bastion's SECURITY.md](https://git.privatae.ai/privatae/privatae-bastion/src/branch/main/SECURITY.md).
+  [the bastion's SECURITY.md](https://github.com/Privatae/bastion/blob/main/SECURITY.md).
  Report there, or to the same address.
- **The Privatae platform** the bastion connects to. It is separate software
+- **The Host** the bastion connects to, which is separate software. A
-  with its own reporting path; reports about it sent to the address above are
+  C-Series Host runs on your own machine, in its own Linux virtual machine;
-  forwarded.
+  its source is [Privatae/cseries](https://github.com/Privatae/cseries), under
- **What the design already says it does not do.** The rules-only `flag` mode
+  AGPL-3.0-only, `npx cseries` installs the bastion beside it and pairs,
-  does not find names, and the recall pass is probabilistic
+  anchors and enrols it, and no Privatae account is needed. C-Series Cloud (a
  Host on a server of its own that Privatae sets up and keeps patched) and a
  Privatae network membership are optional and paid. A Host on Privatae's own
  platform is the case where the Host's software is not open source; the
  bastion keeps it to your grant and your approvals whatever the platform
  does. Reports about a Host sent to the address above are forwarded.
 - **What the design already says it does not do.** The rules-only `flag` mode,
  which is the mode the bastion deploys, does not find names, street addresses
  or dates of birth ([README](README.md#how-the-bastion-runs-it)), and the
  recall pass of `sterilize` mode is probabilistic
  ([docs/pii-sterilization.md](docs/pii-sterilization.md#limits)); the extract
  store in the output folder is plaintext by design
  ([docs/runtime.md](docs/runtime.md#where-an-agent-lives-rootagent)); the
@@ -1 +1 @@
-0.1.15
+0.1.16
@@ -10,3 +10,9 @@ from .manifest import (  # noqa: F401
    signing_payload,
    validate_document,
 )
 # The package's public names: the manifest's schema and its loader.
 __all__ = [
    "STAGES", "MASK_OPS", "SWEEPS", "GRANT_OPS", "Manifest", "ManifestError",
    "load_manifest", "signing_digest", "signing_payload", "validate_document",
 ]
@@ -340,7 +340,6 @@ def catalog(manifest, out_dir, env=None, summarizer=None, now=None):
            continue
        key = (w["root"], w["path"])
        counts["files"] += 1
        name = w["path"].rsplit("/", 1)[-1]
        shown_path, changed = mask_path(w["path"], detect)
        masked = shown_path.rsplit("/", 1)[-1]
        counts["names_masked"] += changed
@@ -7,11 +7,13 @@ tesseract, the gliner package) — their absence is a recorded gap in the
 stage output, not a failure here.
 """
 import importlib
 import json
 import os
 import platform
 import shutil
 import sys
 import urllib.error
 import urllib.request
 PY_FLOOR = (3, 8)
@@ -25,7 +27,6 @@ def _probe_http(url, path, timeout=5):
    """Reachable means the server answered — any HTTP status counts (a
    proxy that only permits the embed path answers 403 on '/'). Only a
    connection failure is 'not reachable'."""
    import urllib.error
    try:
        with urllib.request.urlopen(url.rstrip("/") + path, timeout=timeout) as r:
            return True, "HTTP %d" % r.status
@@ -65,7 +66,7 @@ def checks(env=None):
        add(tool, bool(p), p or ("not on PATH — " + why), required=req, stage=stage)
    try:
-        import gliner  # noqa: F401
+        importlib.import_module("gliner")
        add("gliner package", True, "importable (PII_RECALL=gliner available)", required=False, stage="sanitize")
    except ImportError:
        add("gliner package", False, "not installed — sanitize sterilize on a laptop needs "
@@ -41,7 +41,6 @@ import os
 import re
 import shutil
 import subprocess
 import sys
 import tempfile
 import time
 import unicodedata
@@ -192,7 +192,7 @@ installers = {n: {"url": "%s/%s" % (base_new, n), "sha256": h(os.path.join(pkg,
              for n in ("install.sh", "install.ps1")}
 json.dump({"version": ver, "built_from": sha, "python_floor": "3.8",
           "files": files, "installers": installers,
-           "notes": "https://git.privatae.ai/privatae/bastion-agent/releases/tag/v%s" % ver},
+           "notes": "https://github.com/Privatae/bastion-agent/releases/tag/v%s" % ver},
          sys.stdout, indent=2)
 PY
@@ -52,7 +52,7 @@ from .manifest import signing_digest
 # Part of the reuse key: bump it whenever the pipeline would sterilize the
 # same text differently, or a stored result from older code is kept as is.
-SANITIZE_VERSION = 2
+SANITIZE_VERSION = 3
 SANITIZE_CHUNK_CHARS = 20000     # per sterilize call; under the service's 2 MiB body cap
 SERVICE_TIMEOUT = 600.0
@@ -17,6 +17,12 @@ Ops (JSON over HTTPS):
  POST /read_range  {doc, start, end}     read_range (owner, grantee w/ op)
                                          from the SANITIZED store only,
                                          <= READ_RANGE_MAX bytes
  Both answer with `sanitize_mode`, the mode the sanitize stage recorded for
  the text in the reply ("flag" or "sterilize"; the weakest across a
  search's hits; "unknown" when the record does not vouch for that text; a
  search with no hits names the manifest's mode, or "none"), so a caller can
  say what the text has had done to it: in flag mode names, street addresses
  and dates of birth remain.
  GET  /catalog[?prefix=&offset=&limit=]  owner only: the file cards (paged)
  GET  /catalog/dirs[?...]                owner only: the directory cards (paged)
  GET  /status                            owner only: run.state.json
@@ -65,6 +71,83 @@ class ServeError(Exception):
    pass
 def sanitize_mode(manifest):
    """The sanitize mode the manifest sets: "flag", "sterilize", or "none" when
    it names no sanitize stage. A reply with no text in it (a search with no
    hits) carries this; a reply with text carries served_mode."""
    s = getattr(manifest, "sanitize", None)
    return getattr(s, "mode", None) or "none"
 # Weakest first: a reply that mixes texts names the weakest mode among them.
 _MODE_RANK = {"unknown": 0, "flag": 1, "sterilize": 2}
 _index_lock = threading.Lock()
 _index_cache = {}  # sanitize.jsonl path -> ((ino, mtime_ns, size), {doc: mode})
 def _sanitize_index(out_dir):
    """({doc sha: mode} for every document the last sanitize run wrote with
    status ok, the index's mtime_ns), from <out>/sanitize.jsonl; ({}, None)
    when there is none. Re-read only when the file changes."""
    path = os.path.join(out_dir, "sanitize.jsonl")
    try:
        st = os.stat(path)
    except OSError:
        return {}, None
    sig = (st.st_ino, st.st_mtime_ns, st.st_size)
    with _index_lock:
        hit = _index_cache.get(path)
        if hit and hit[0] == sig:
            return hit[1], st.st_mtime_ns
        modes = {}
        try:
            with open(path, encoding="utf-8") as fh:
                for line in fh:
                    try:
                        rec = json.loads(line)
                    except ValueError:
                        continue
                    if isinstance(rec, dict) and rec.get("status") == "ok" and rec.get("sha256"):
                        modes[rec["sha256"]] = rec.get("mode")
        except OSError:
            return {}, None
        _index_cache[path] = (sig, modes)
        return modes, st.st_mtime_ns
 def served_mode(out_dir, doc, start=None, text=None):
    """The sanitize mode that wrote the text served for doc, as the sanitize
    stage recorded it, or "unknown" when that record does not vouch for the
    text. The manifest's mode says what the NEXT sweep will do; the store can
    still hold an earlier sweep's text (the manifest switched to sterilize and
    the sweep has not run, or stopped; a sweep is rewriting the store now; the
    chunks a search reads were cut before the latest sanitize run).
    Unknown when: no status-ok record for doc, the file is newer than the
    index (a run that has not finished rewrote it), or, for a search hit,
    `text` at `start` is not what the store holds now."""
    modes, index_mtime = _sanitize_index(out_dir)
    mode = modes.get(doc)
    if mode not in ("flag", "sterilize"):
        return "unknown"
    path = sanitized_store_path(out_dir, doc)
    try:
        if os.stat(path).st_mtime_ns > index_mtime:
            return "unknown"
        if text is not None:
            with open(path, encoding="utf-8") as fh:
                stored = fh.read()
            if stored[start:start + len(text)] != text:
                return "unknown"
    except (OSError, TypeError, ValueError):
        return "unknown"
    return mode
 def _weakest(modes):
    return min(modes, key=lambda m: _MODE_RANK.get(m, 0))
 def parse_bind(bind):
    bind = bind.strip()
    if bind.startswith("["):
@@ -312,7 +395,13 @@ class Handler(BaseHTTPRequestHandler):
            except (_embed.EmbedError, OSError) as e:
                self._send(502, {"error": "search unavailable: %s" % e}, cn, "search", {"q": qhash})
                return
-            self._send(200, {"hits": hits, "k": k}, cn, "search", {"q": qhash, "hits": len(hits)})
+            if hits:
                mode = _weakest([served_mode(self.out_dir, h.get("doc"), h.get("start"), h.get("text"))
                                 for h in hits])
            else:
                mode = sanitize_mode(self.policy.manifest)
            self._send(200, {"hits": hits, "k": k, "sanitize_mode": mode},
                       cn, "search", {"q": qhash, "hits": len(hits)})
            return
        if self.path == "/read_range":
            cn, denied = self._authorize("read_range")
@@ -340,7 +429,8 @@ class Handler(BaseHTTPRequestHandler):
                text = fh.read()
            piece = text[start:end]
            self._send(200, {"doc": doc, "start": start, "end": start + len(piece),
-                             "total": len(text), "text": piece},
+                             "total": len(text), "text": piece,
                             "sanitize_mode": served_mode(self.out_dir, doc)},
                       cn, "read_range", {"doc": doc[:12], "chars": len(piece)})
            return
        self._send(404, {"error": "not found"}, self._cn(), None)
@@ -12,7 +12,6 @@ import shutil
 import subprocess
 import tempfile
 import unittest
 from unittest import mock
 import yaml
@@ -153,6 +153,19 @@ class FlagModeTests(Case):
        self.assertEqual(s2["counts"]["ok"], s1["counts"]["ok"])
        self.assertFalse(idx["note.txt"].get("reused", False))
    def test_text_kept_by_the_older_rules_is_redone(self):
        # version 2 kept an international number and a lowercase IBAN in the
        # clear; a store it wrote must not be reused by the wider rules
        put(self.root, "intl.txt", "Mobile +44 7700 900123, iban gb82west12345698765432\n")
        with mock.patch.object(sz, "SANITIZE_VERSION", 2):
            self.run_stage(manifest(self.root, "flag"))
        s, idx = self.run_stage(manifest(self.root, "flag"))
        self.assertEqual(s["counts"]["reused"], 0)
        out = self.text_of(idx["intl.txt"])
        self.assertNotIn("7700 900123", out)
        self.assertNotIn("gb82west", out)
        self.assertEqual(idx["intl.txt"]["types"], {"PHONE": 1, "BANK_ACCOUNT": 1})
    def test_tokens_stable_across_files_in_scope(self):
        put(self.root, "again.txt", "ping m@example.com again\n")
        _, idx = self.run_stage(manifest(self.root, "flag"))
@@ -175,7 +175,7 @@ class RoundTripTests(Case):
    def test_tampered_member_refused(self):
        ident, pub = keygen(self.out)
        env = {"FREEAGENT_OWNER_RECIPIENT": pub}
-        s = sl.seal(manifest(self.root), self.out, env=env)
+        sl.seal(manifest(self.root), self.out, env=env)
        # rebuild a bundle with a modified catalog but the old inventory, seal it, and unseal
        plain, _ = sl.build_bundle(manifest(self.root), self.out, "x")
        buf = io.BytesIO()
@@ -22,7 +22,6 @@ from . import run as rn
 from . import serve as sv
 from .catalog import catalog
 from .extract import extract
 from .manifest import validate_document
 from .sanitize import sanitize
 from .test_embed import _Stub as EmbedStub  # deterministic bag-of-words embedder
 from .walk import walk
@@ -112,12 +111,12 @@ class ServeTests(unittest.TestCase):
        shutil.rmtree(cls.work, ignore_errors=True)
    @classmethod
-    def write_manifest(cls, grants):
+    def write_manifest(cls, grants, mode="flag"):
        import yaml
        doc = {"freeagent": 1, "id": "t", "owner": {"host": "owner-host"},
               "scope": {"roots": [cls.root]}, "mask": ["read", "search", "index"],
               "stages": ["walk", "extract", "sanitize", "chunk", "embed", "catalog", "seal"],
-               "sanitize": {"mode": "flag"},
+               "sanitize": {"mode": mode},
               "embed": {"model": "stub-model", "dim": 64, "chunk": {"size_tokens": 8, "overlap_tokens": 2}},
               "sweep": "manual",
               "sink": {"catalog": "host", "vectors": "local", "plaintext": "never"},
@@ -238,6 +237,93 @@ class ServeTests(unittest.TestCase):
        self.assertEqual(self.call("owner", "POST", "/search", {"query": ""})[0], 400)
        self.assertEqual(self.call("owner", "POST", "/nope", {})[0], 404)
    GRANTS = [{"host": "partner-host", "ops": ["search"], "until": "2026-12-31"},
              {"host": "old-host", "ops": ["search", "read_range"], "until": "2026-01-01"}]
    def modes(self):
        """(owner search, owner read_range, partner search) sanitize_mode for the apples doc."""
        st, body = self.call("owner", "POST", "/search", {"query": "apples", "k": 1})
        self.assertEqual(st, 200)
        doc = body["hits"][0]["doc"]
        st, rr = self.call("owner", "POST", "/read_range", {"doc": doc, "start": 0, "end": 10})
        self.assertEqual(st, 200)
        st, pb = self.call("partner", "POST", "/search", {"query": "apples", "k": 1})
        self.assertEqual(st, 200)
        return doc, (body["sanitize_mode"], rr["sanitize_mode"], pb["sanitize_mode"])
    def rewrite_index(self, fn):
        """Rewrite sanitize.jsonl through fn(record); returns the original bytes."""
        path = os.path.join(self.out, "sanitize.jsonl")
        with open(path, "rb") as fh:
            orig = fh.read()
        recs = [fn(json.loads(l)) for l in orig.decode("utf-8").splitlines() if l.strip()]
        with open(path, "w", encoding="utf-8") as fh:
            fh.write("".join(json.dumps(r, sort_keys=True) + "\n" for r in recs))
        return orig
    def restore_index(self, orig):
        with open(os.path.join(self.out, "sanitize.jsonl"), "wb") as fh:
            fh.write(orig)
    def test_a_reply_names_the_mode_that_wrote_its_text_not_the_manifests(self):
        # The store was swept in flag mode. Switching the manifest to
        # sterilize changes what the NEXT sweep does, not the text already
        # served: until a sweep rewrites it, every reply still says flag, so
        # the bastion never calls flag-mode text sanitized.
        try:
            for mode in ("flag", "sterilize", "flag"):
                self.write_manifest(grants=self.GRANTS, mode=mode)
                self.assertEqual(self.modes()[1], ("flag", "flag", "flag"), mode)
        finally:
            self.write_manifest(grants=self.GRANTS)
    def test_a_store_the_sanitize_stage_wrote_in_sterilize_mode_is_named_sterilize(self):
        orig = self.rewrite_index(lambda r: dict(r, mode="sterilize"))
        try:
            self.write_manifest(grants=self.GRANTS, mode="sterilize")
            self.assertEqual(self.modes()[1], ("sterilize", "sterilize", "sterilize"))
        finally:
            self.restore_index(orig)
            self.write_manifest(grants=self.GRANTS)
    def test_text_the_sanitize_record_does_not_vouch_for_is_unknown(self):
        doc, modes = self.modes()
        self.assertEqual(modes, ("flag", "flag", "flag"))
        spath = sv.sanitized_store_path(self.out, doc)
        st = os.stat(spath)
        index = os.path.join(self.out, "sanitize.jsonl")
        # a sweep still running has rewritten the file after the index
        later = os.stat(index).st_mtime_ns + 10 ** 9
        os.utime(spath, ns=(later, later))
        try:
            self.assertEqual(self.modes()[1], ("unknown", "unknown", "unknown"))
        finally:
            os.utime(spath, ns=(st.st_atime_ns, st.st_mtime_ns))
        # the chunks a search reads were cut from text the store no longer holds
        with open(spath, encoding="utf-8") as fh:
            text = fh.read()
        with open(spath, "w", encoding="utf-8") as fh:
            fh.write("X" * len(text))
        os.utime(spath, ns=(st.st_atime_ns, st.st_mtime_ns))
        try:
            got = self.modes()[1]
            self.assertEqual((got[0], got[2]), ("unknown", "unknown"))
        finally:
            with open(spath, "w", encoding="utf-8") as fh:
                fh.write(text)
            os.utime(spath, ns=(st.st_atime_ns, st.st_mtime_ns))
        # the last run did not write it (stopped: not_run), so the file is an older run's
        orig = self.rewrite_index(lambda r: dict(r, status="not_run") if r.get("sha256") == doc else r)
        try:
            self.assertEqual(self.modes()[1], ("unknown", "unknown", "unknown"))
        finally:
            self.restore_index(orig)
        self.assertEqual(self.modes()[1], ("flag", "flag", "flag"))
    def test_sanitize_mode_of_a_manifest_without_the_stage_is_none(self):
        self.assertEqual(sv.sanitize_mode(None), "none")
        self.assertEqual(sv.sanitize_mode(type("M", (), {"sanitize": None})()), "none")
    def test_silent_tcp_connection_does_not_stall_other_clients(self):
        # with the handshake on the accept loop one idle raw TCP
        # connection blocked every client; now the handshake runs in the
@@ -333,7 +419,7 @@ class UnitTests(unittest.TestCase):
    def test_parse_bind(self):
        self.assertEqual(sv.parse_bind("[::]:8470"), ("::", 8470))
        self.assertEqual(sv.parse_bind("10.0.0.5:9"), ("10.0.0.5", 9))
-        self.assertEqual(sv.parse_bind("[fd00:1::31]:8470"), ("fd00:1::31", 8470))
+        self.assertEqual(sv.parse_bind("[2001:db8::31]:8470"), ("2001:db8::31", 8470))
        with self.assertRaises(sv.ServeError):
            sv.parse_bind("nope")
@@ -169,16 +169,26 @@ class EndToEndTests(unittest.TestCase):
                           "files": {name: {"sha256": digest or self.digest},
                                     name + ".sha256": {"sha256": "0" * 64}}}).encode()
-    def verify(self, body, seed=None):
+    def verify(self, body, seed=None, args=()):
        msg = vr.statement("bastion-agent", self.version, hashlib.sha256(body).hexdigest())
-        sig = base64.b64encode(_sign(seed or self.SEED, msg))
+        sig = base64.b64encode(_sign(seed or self.SEED, msg)) + b"\n"
        served = {"/version.json": body, "/" + vr.sig_name(body): sig}
        self.served = served
        out, err = io.StringIO(), io.StringIO()
        with mock.patch.object(vr, "fetch", lambda url: served[url[len("https://test"):]]), \
                contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
-            code = vr.main(["--base", "https://test"], repo=self.repo, keys=(self.pub,))
+            code = vr.main(["--base", "https://test"] + list(args), repo=self.repo, keys=(self.pub,))
        return code, out.getvalue() + err.getvalue()
    def release_tag(self, built_from, annotated=True):
        tag = "v" + self.version
        if annotated:
            self.run_git("tag", "-a", tag, "-m", "bastion agent %s\n\nbuilt from %s" % (self.version, built_from))
        else:
            self.run_git("tag", tag)
        self.addCleanup(subprocess.run, ("git", "-C", self.repo, "tag", "-d", tag),
                        stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
    def test_tag_first_then_built_from(self):
        code, out = self.verify(self.manifest("0000000"))
        self.assertEqual(code, 1, out)
@@ -189,13 +199,57 @@ class EndToEndTests(unittest.TestCase):
        self.assertIn("source:    built_from", out)
        self.assertIn("matches %s" % self.digest, out)
-        self.run_git("tag", "v" + self.version)
+        self.release_tag("0000000")
        try:
        code, out = self.verify(self.manifest("0000000"))
        self.assertEqual(code, 0, out)
        self.assertIn("source:    tag v%s" % self.version, out)
-        finally:
+
    def test_a_tag_without_the_release_record_is_refused(self):
        # A lightweight tag, or one recording another build, says nothing
        # about which release it is: refused, not passed over for built_from.
        self.release_tag("0000000", annotated=False)
        code, out = self.verify(self.manifest(self.commit[:7]))
        self.assertEqual(code, 1, out)
        self.assertIn("does not carry the release record", out)
        self.assertNotIn("rebuild:", out)
        self.run_git("tag", "-d", "v" + self.version)
        self.release_tag("1111111")
        code, out = self.verify(self.manifest(self.commit[:7]))
        self.assertEqual(code, 1, out)
        self.assertIn("built from %s" % self.commit[:7], out)
    def test_another_version_than_expected_is_refused_before_any_checkout(self):
        code, out = self.verify(self.manifest(self.commit[:7]), args=["--expect-version", "v9.9.9"])
        self.assertEqual(code, 1, out)
        self.assertIn("names v%s, not v9.9.9" % self.version, out)
        self.assertNotIn("source:", out)
        code, out = self.verify(self.manifest(self.commit[:7]), args=["--expect-version", "v" + self.version])
        self.assertEqual(code, 0, out)
    def test_out_writes_what_a_release_carries(self):
        name = "bastion-agent-%s.pyz" % self.version
        sums = ("%s  %s\n" % (self.digest, name)).encode()
        out_dir = os.path.join(self.tmp, "release-out")
        self.addCleanup(shutil.rmtree, out_dir, True)
        body = json.dumps({"version": self.version, "built_from": self.commit[:7],
                           "files": {name: {"sha256": self.digest},
                                     name + ".sha256": {"sha256": hashlib.sha256(sums).hexdigest()}}}).encode()
        code, out = self.verify(body, args=["--out", out_dir])
        self.assertEqual(code, 0, out)
        sig = vr.sig_name(body)
        self.assertEqual(sorted(os.listdir(out_dir)), sorted([name, name + ".sha256", "version.json", sig]))
        with open(os.path.join(out_dir, "version.json"), "rb") as fh:
            self.assertEqual(fh.read(), body)
        with open(os.path.join(out_dir, sig), "rb") as fh:
            self.assertEqual(fh.read(), self.served["/" + sig])
        with open(os.path.join(out_dir, name), "rb") as fh:
            self.assertEqual(hashlib.sha256(fh.read()).hexdigest(), self.digest)
    def test_out_writes_nothing_when_the_rebuild_differs(self):
        out_dir = os.path.join(self.tmp, "release-out-bad")
        code, out = self.verify(self.manifest(self.commit[:7], digest="1" * 64), args=["--out", out_dir])
        self.assertEqual(code, 1, out)
        self.assertFalse(os.path.exists(out_dir))
    def test_a_different_build_is_reported(self):
        code, out = self.verify(self.manifest(self.commit, digest="1" * 64))
@@ -10,13 +10,12 @@ import hashlib
 import json
 import os
 import shutil
 import sys
 import tempfile
 import unittest
 from unittest import mock
 from .manifest import validate_document
-from .walk import Excluder, Walker, _glob_to_regex, load_hash_cache, walk
+from .walk import Excluder, _glob_to_regex, load_hash_cache, walk
 IS_WINDOWS = os.name == "nt"
@@ -15,9 +15,15 @@ and the signature says the release key holder published it.
 The source is the tag v<version>. The public repository is a snapshot of the
 development one, so the commit the manifest names (built_from) exists only
-there; the tag marks the same tree in the public history. When the tag is
+there; the tag marks the same tree in the public history, and it counts only
-absent and built_from is present (a clone of the development repository),
+as an annotated tag whose message has the line `built from <built_from>`. When
-built_from is used instead.
+the tag is absent and built_from is present (a clone of the development
 repository), built_from is used instead.
 `--expect-version` refuses a manifest that names another version, and `--out
 DIR` writes what a release carries once everything matched: the rebuilt .pyz
 and its .sha256, the manifest's exact bytes and its signature file. The
 GitHub release workflow (.github/workflows/release.yml) runs it that way.
 The signature check is ed25519 in pure Python, because the standard library
 has no ed25519 and this has to run where nothing else is installed. It only
@@ -228,15 +234,33 @@ def _commit(repo, ref):
    return r.stdout.strip() if r.returncode == 0 else ""
 def tag_records(repo, tag, built_from):
    """True when tag is an annotated tag whose message has the line
    `built from <built_from>`, the record publishing writes on every release
    tag. A tag without it says nothing about which release it is."""
    if not built_from:
        return False
    if _git(repo, "cat-file", "-t", "refs/tags/" + tag, check=False).stdout.strip() != "tag":
        return False
    msg = _git(repo, "tag", "-l", "--format=%(contents)", tag, check=False).stdout
    return ("built from " + built_from) in msg.splitlines()
 def pick_source(repo, version, built_from):
    """(label, commit) of the source to rebuild: the tag v<version>, else the
-    manifest's built_from, else VerifyError."""
+    manifest's built_from, else VerifyError. The tag counts only when it
    records the manifest's built_from (tag_records); one that does not is
    refused rather than passed over."""
    tag = "v" + version
    # Fetch the tag if the clone lacks it; a failure (offline, no such tag)
    # just leaves the local refs to decide.
    _git(repo, "fetch", "-q", "origin", "refs/tags/%s:refs/tags/%s" % (tag, tag), check=False)
    c = _commit(repo, "refs/tags/" + tag)
    if c:
        if not tag_records(repo, tag, built_from):
            raise VerifyError("the tag %s does not carry the release record (an annotated tag "
                              "with the line 'built from %s'); nothing was built"
                              % (tag, built_from or "(none named)"))
        return "tag " + tag, c
    if built_from:
        c = _commit(repo, built_from)
@@ -249,7 +273,8 @@ def pick_source(repo, version, built_from):
 def rebuild(repo, commit, version, work):
-    """sha256 of the .pyz that commit's own build.py produces."""
+    """sha256 of the .pyz that commit's own build.py produces. The build's
    output stays in <work>/dist."""
    src = os.path.join(work, "src")
    _git(repo, "worktree", "add", "-q", "--detach", src, commit)
    try:
@@ -269,20 +294,52 @@ def rebuild(repo, commit, version, work):
        _git(repo, "worktree", "remove", "--force", src, check=False)
 def write_release(out, dist, version, files, body, sig):
    """The files a release carries, into out: the rebuilt .pyz and its .sha256
    (each checked against the manifest's digest for that name), the manifest
    as it was served, byte for byte, and its signature under the name it is
    served by. Called only after the signature and the rebuild both passed."""
    os.makedirs(out, exist_ok=True)
    for name in ("bastion-agent-%s.pyz" % version, "bastion-agent-%s.pyz.sha256" % version):
        with open(os.path.join(dist, name), "rb") as fh:
            data = fh.read()
        want = (files.get(name) or {}).get("sha256")
        if hashlib.sha256(data).hexdigest() != want:
            raise VerifyError("the rebuilt %s does not match the manifest's sha256 for it" % name)
        with open(os.path.join(out, name), "wb") as fh:
            fh.write(data)
    with open(os.path.join(out, "version.json"), "wb") as fh:
        fh.write(body)
    with open(os.path.join(out, sig_name(body)), "wb") as fh:
        fh.write(sig)
    print("release:   wrote %s" % ", ".join(sorted(os.listdir(out))))
 def main(argv=None, repo=REPO, keys=None):
    ap = argparse.ArgumentParser(prog="bastion_agent.verify_release",
                                 description="Rebuild the published agent and compare it with the release manifest.")
    ap.add_argument("--base", default=os.environ.get("MANIFEST_BASE", DEFAULT_BASE),
                    help="where version.json is served (default %s)" % DEFAULT_BASE)
    ap.add_argument("--expect-version", metavar="VERSION",
                    help="refuse unless the published manifest names this version (v0.1.16 or 0.1.16)")
    ap.add_argument("--out", metavar="DIR",
                    help="when everything matches, write the rebuilt .pyz and its .sha256, the "
                         "manifest's exact bytes and its signature file into DIR")
    args = ap.parse_args(argv)
    base = args.base.rstrip("/")
    work = tempfile.mkdtemp(prefix="agent-verify-")
    try:
        body = fetch(base + "/version.json")
-        sig = fetch(base + "/" + sig_name(body)).decode("ascii", "replace")
+        sig_raw = fetch(base + "/" + sig_name(body))
        sig = sig_raw.decode("ascii", "replace")
        m, key = check_signature(body, sig, keys)
        ver, built_from = m["version"], m.get("built_from") or ""
        print("published: v%s built from %s" % (ver, built_from or "(not named)"))
        want = (args.expect_version or "").strip()
        if want.startswith("v"):
            want = want[1:]
        if want and want != ver:
            raise VerifyError("the published manifest names v%s, not v%s" % (ver, want))
        print("signature: ok (release key %d of %d, %s)"
              % (key + 1, len(RELEASE_KEYS if keys is None else keys), "primary" if key == 0 else "spare"))
        files = m.get("files") if isinstance(m.get("files"), dict) else {}
@@ -295,12 +352,16 @@ def main(argv=None, repo=REPO, keys=None):
        print("source:    %s (%s)" % (label, commit[:12]))
        got = rebuild(repo, commit, ver, work)
        bad = 0
-        for name, want in pyz.items():
+        for name, digest in pyz.items():
-            same = got == want
+            same = got == digest
            bad += not same
            print("rebuild:   %-28s %s" % (name, "matches %s" % got if same
-                                             else "DIFFERS (rebuilt %s, published %s)" % (got, want)))
+                                             else "DIFFERS (rebuilt %s, published %s)" % (got, digest)))
-        return 1 if bad else 0
+        if bad:
            return 1
        if args.out:
            write_release(args.out, os.path.join(work, "dist"), ver, files, body, sig_raw)
        return 0
    except VerifyError as exc:
        print("verify: %s" % exc, file=sys.stderr)
        return 1
@@ -25,7 +25,6 @@ import json
 import os
 import platform
 import re
 import sys
 import tempfile
 from .manifest import AGENT_DIR, signing_digest
@@ -122,9 +122,12 @@ checks each edge's bytes against the manifest it serves. Where it publishes
 is not written into the script: the edges, the ssh key, the web root and the
 backup directory come from its environment, and it stops if one is missing.
-The manifest's `notes` link to the tag `v<version>` on the public repository,
+The manifest's `notes` link to the GitHub release of the tag `v<version>`,
-which marks the source of that release. `publish.sh` does not make the tag; it
+which marks the source of that release and carries its files. `publish.sh`
-is made when that release's source is published to the public repository.
+does not make the tag; it is made when that release's source is published to
 the public repository, and pushing it starts `.github/workflows/release.yml`,
 which rebuilds the `.pyz` from the tag and attaches it, with its `.sha256`, the
 signed manifest and its signature, once its sha256 matches the manifest.
 ### The legacy name
@@ -15,7 +15,9 @@ personal data is replaced with stable placeholder tokens, the real values stay
 in a local vault, and `/rehydrate` puts them back, locally, on demand. A cloud
 model can then work over the text without the values in it. Patterns alone
 cannot do this: "my daughter goes to Lincoln Elementary" contains no pattern,
-only meaning.
+only meaning. The agent the bastion deploys runs the rules pass only (`flag`
 mode), so names, street addresses and dates of birth stay in the text it
 returns ([sanitize.md](sanitize.md#modes-manifest-sanitizemode)).
 ## Pipeline (`pii/pipeline.py`)
@@ -116,7 +118,7 @@ Configuration, from the environment:
 | Variable | Default | Meaning |
 |---|---|---|
 | `PII_API_KEY` | (none) | Required. Without it the service refuses to start: `/rehydrate` returns real values, so there is no unauthenticated mode. (`PII_ALLOW_NOAUTH=1` exists for the unit tests only.) |
-| `PII_BIND` | `0.0.0.0:8091` | The address the service listens on; this is its exposure control. |
+| `PII_BIND` | `127.0.0.1:8091` | The address the service listens on; this is its exposure control. Another address only when set. |
 | `PII_DB` | `/data/vault.db` | The vault. |
 | `PII_RECALL` | `llm` | The recall backend (below). |
 | `LLM_URL` | `http://llm:8080` | The model server for `llm`. |
@@ -125,7 +127,9 @@ Configuration, from the environment:
 Two ways to run it, the same code:
 - **Container**: `pii/Dockerfile` (a pinned `python:*-slim` image, no pip
-  install; the vault on a mounted `/data`).
+  install; the vault on a mounted `/data`, writable by uid 10001, the
  unprivileged user it runs as). Inside a container the default loopback bind
  reaches nothing else, so set `PII_BIND` and publish the port deliberately.
 - **systemd**: `pii/pii.service`, with install steps in its header. It runs
  with `DynamicUser`, a private 0700 state directory for the vault
  (`/var/lib/privatae-pii/vault.db`), loopback bind (`127.0.0.1:8091`), and
@@ -167,9 +171,18 @@ the machine.
 Every detector is anchored by a checksum, a prefix or a fixed shape; none
 depends on a model.
- **E-mail, phone, SSN, IPv4/IPv6, MAC.**
+- **E-mail, IPv4/IPv6, MAC.** A dotted number right after a version word
- **Cards** → `CREDIT_CARD`, checked with Luhn.
+  (`version 1.2.3.4`, `v1.2.3.4`) is not an IP address.
- **IBAN** → `BANK_ACCOUNT`: ISO 13616 mod-97 plus a per-country length
+- **Phone**: North American and other grouped national forms; international
  numbers with `+` and a country code, any grouping, an optional `(0)` trunk
  prefix; national numbers with a trunk `0` (`07700 900123`); a seven-digit
  local number (`555-0142`) only after a word such as call, phone or tel.
 - **SSN**: dashed or spaced; nine bare digits only after `SSN` or `social
  security` within 40 characters. Numbers never issued (area 000, 666, 900+;
  group 00; serial 0000) are left alone.
 - **Cards** → `CREDIT_CARD`, with spaces, dots or dashes, checked with Luhn.
  An expiry date or security code beside the card is not detected.
 - **IBAN** → `BANK_ACCOUNT`, in either case: ISO 13616 mod-97 plus a per-country length
  table. It runs before the card detector, so an IBAN's digits cannot be
  claimed as a card number.
 - **US ABA routing number** → `BANK_ACCOUNT`: the weighted checksum and a
@@ -51,9 +51,12 @@ stages, there is no extract index, or the stage **stopped**; `2` completed with
 - **`flag`**: the rules pass **only**: deterministic, no model, milliseconds.
  It replaces the anchors a pattern can find: e-mail addresses, phone
  numbers, SSNs, cards, IBANs, routing numbers, crypto addresses, IP and MAC
-  addresses, API keys, private-key blocks, JWTs. **It does not find a name.**
+  addresses, API keys, private-key blocks, JWTs. It can also replace an
-  A flag-mode catalog is not free of personal data, and the manifest doc says
+  ordinary number that looks like a phone or ID number. **It does not find names,
-  so too.
+  street addresses, dates of birth**, or anything else that takes reading for
  meaning (an employer, a school, a medical fact). It is the mode the bastion
  writes into every manifest it deploys. A flag-mode catalog is not free of
  personal data, and the manifest doc says so too.
 ## Where the sterilizer runs
@@ -2,7 +2,9 @@
 `serve` is how a host reads what the agent prepared without the data moving:
 a host (the owner's, or a grantee's) sends a query and gets **sanitized**
-chunks and catalog cards back. Nothing else has a route: not the extract
+chunks and catalog cards back (sanitized by the manifest's mode: in `flag`
 mode, which the bastion deploys, names, street addresses and dates of birth
 remain; [sanitize.md](sanitize.md#modes-manifest-sanitizemode)). Nothing else has a route: not the extract
 store, not the vault, not the vectors table. It is `bastion_agent/serve.py`,
 mutual TLS with the standard library's `ssl`.
@@ -31,12 +33,26 @@ There is no other credential: no token, no password, no header.
 | Route | Body | Returns |
 |---|---|---|
-| `POST /search` | `{query, k?}` (k defaults to 5 and is held to 1–50) | `{hits, k}`: the top k sanitized windows with score, document id, chunk id, index and offsets. The same search as the CLI, through the same `EMBED_URL` model ([embed.md](embed.md)); 502 if the model cannot be reached. |
+| `POST /search` | `{query, k?}` (k defaults to 5 and is held to 1–50) | `{hits, k, sanitize_mode}`: the top k sanitized windows with score, document id, chunk id, index and offsets, and the sanitize mode that wrote them. The same search as the CLI, through the same `EMBED_URL` model ([embed.md](embed.md)); 502 if the model cannot be reached. |
-| `POST /read_range` | `{doc, start?, end}` | Up to 65,536 characters of the **sanitized** text of that document, with `total`. |
+| `POST /read_range` | `{doc, start?, end}` | Up to 65,536 characters of the **sanitized** text of that document, with `total` and `sanitize_mode`. |
 | `GET /catalog`, `GET /catalog/dirs` | query `prefix`, `offset`, `limit` (default 500, at most 2,000) | The file cards or the directory cards (owner only), a page at a time: `{cards, kind, total, offset, limit, next_offset, prefix}`, `next_offset` being `null` on the last page. 404 until the first sweep has written a catalog. |
 | `GET /status` | | `{state}`: `run.state.json` (owner only). |
 | `GET /health` | | `{ok, id, serving}` for any client that chained. |
 `sanitize_mode` says what was done to the text in that reply: the mode the
 sanitize stage recorded for that document in `sanitize.jsonl`, `flag` or
 `sterilize`. A search names the weakest mode among its hits. It is `unknown`
 when that record does not vouch for the text: the last sanitize run did not
 write the document (it stopped, or the file is an older run's), the file was
 rewritten after the record (a sweep is running), or a search hit's text is no
 longer what the store holds (the chunks predate the latest sanitize run). A
 search with no hits names the manifest's `sanitize.mode`, or `none` when the
 manifest has no sanitize stage. In `flag` mode, the one the bastion deploys,
 names, street addresses and dates of birth remain in the text; only
 `sterilize` looks for them. Changing the manifest's mode does not change the
 field: after a change of mode the next sweep sanitizes every file again, and
 until it has, the text served is the previous mode's and the reply says so.
 `doc` is the document's content hash (64 lowercase hex), the same id the
 extract and sanitized stores share. The route reads only the `sanitize/`
 directory, so the plaintext for that id, which exists on disk, is never what
@@ -9,9 +9,18 @@ WORKDIR /app
 # boot (recall.py was, once)
 COPY __init__.py detect.py llmclient.py pipeline.py recall.py service.py vault.py /app/pii/
-# Root inside the container is deliberate: the vault db and its directory
+# The service runs as its own unprivileged user. The vault and its directory
-# are created 0700/0600 (service sets umask 077) on the mounted /data,
+# are created 0700/0600 (the service sets umask 077), so a volume mounted on
-# and the single-purpose process is bind-scoped by PII_BIND. Revisit if
+# /data must be writable by uid 10001.
-# the service ever grows a second concern.
+RUN groupadd --system --gid 10001 pii \
    && useradd --system --uid 10001 --gid 10001 --no-create-home \
        --home-dir /nonexistent --shell /usr/sbin/nologin pii \
    && install -d -m 0700 -o pii -g pii /data
 USER 10001:10001
 # The service listens on 127.0.0.1 unless PII_BIND says otherwise, and inside
 # a container that is reachable from nothing else. To publish it, choose the
 # address and the host side explicitly, for example:
 #   docker run -e PII_BIND=0.0.0.0:8091 -p 127.0.0.1:8091:8091 ...
 ENV PYTHONUNBUFFERED=1
 CMD ["python", "-m", "pii.service"]
@@ -2,3 +2,9 @@ from . import detect, pipeline  # noqa: F401
 from .detect import PII_TYPES, Finding  # noqa: F401
 from .pipeline import TOKEN_RE, ground, rehydrate, replace_values, sterilize  # noqa: F401
 from .vault import Vault  # noqa: F401
 # The package's public names: the modules and what the agent calls in them.
 __all__ = [
    "detect", "pipeline", "PII_TYPES", "Finding", "TOKEN_RE", "ground", "rehydrate",
    "replace_values", "sterilize", "Vault",
 ]
@@ -54,23 +54,54 @@ IPV4_RE = re.compile(r"(?<![\d.])(?:\d{1,3}\.){3}\d{1,3}(?!\d)(?!\.[\w-])")
 IPV6_RE = re.compile(r"(?<![\w:.])(?:[0-9A-Fa-f]{0,4}:){2,7}[0-9A-Fa-f]{0,4}(?![\w:])")
 SSN_RE = re.compile(r"(?<![\d-])(\d{3})-(\d{2})-(\d{4})(?![\d-])")
 # Spaced: not when the groups are part of a longer run of spaced digits.
 SSN_SPACED_RE = re.compile(r"(?<![\d-])(?<!\d )(\d{3}) (\d{2}) (\d{4})(?![\d-])(?! \d)")
 # Nine bare digits are an SSN only after a keyword: an invoice or order
 # number has the same shape.
 SSN_BARE_RE = re.compile(r"(?<![\w-])(\d{3})(\d{2})(\d{4})(?![\w-])")
 SSN_CUE_RE = re.compile(r"(?i)\b(?:ssn|ss#|social\s+security|soc\.?\s*sec\b)")
-# 13–19 digits with optional single space/dash separators; Luhn decides.
+# 13–19 digits with optional single space/dot/dash separators; Luhn decides.
-CARD_RE = re.compile(r"(?<![\d-])\d(?:[ -]?\d){12,18}(?![\d-])")
+# A dot right after a digit before it, or before a digit after it, makes the
 # run part of a longer dotted number.
 CARD_RE = re.compile(r"(?<![\d-])(?<!\d\.)\d(?:[ .-]?\d){12,18}(?![\d-])(?!\.\d)")
 # Digit-count and shape checks happen in code; this only shapes candidates.
 PHONE_RE = re.compile(
    r"(?<![\w.])(?:\+\d{1,3}[\s.-]?)?(?:\(\d{1,4}\)[\s.-]?)?\d{2,4}(?:[\s.-]\d{2,4}){1,4}"
    r"(?![\w-])(?!\.[\w-])"   # a sentence-ending full stop is not part of the number
 )
 # International form: a leading + and country code, an optional (0) trunk
 # prefix or (area) code, then digit groups of any length (+44 7700 900123,
 # +49 (0)30 1234567, +442079460958). Groups after the first need a separator,
 # so a long digit run cannot be split many ways.
 INTL_PHONE_RE = re.compile(
    r"(?<![\w+])\+\d{1,3}(?:[ .-]?\(\d{1,4}\))?[ .-]?\d{1,12}(?:[ .-]\d{1,8}){0,5}"
    r"(?![\w-])(?!\.[\w-])"
 )
 # National form with a trunk 0 and a long subscriber group (07700 900123,
 # 030 1234567, (030) 1234567). The separator after the area code is required.
 NATIONAL_PHONE_RE = re.compile(
    r"(?<![\w.+])(?:\(0\d{1,4}\) ?|0\d{1,4}[ .-])\d{3,8}(?:[ .-]\d{2,4}){0,2}"
    r"(?![\w-])(?!\.[\w-])"
 )
 # A seven-digit local number (555-0142) is a phone only after a phone word;
 # order and reference numbers have the same shape.
 LOCAL_PHONE_RE = re.compile(r"(?<![\w.+-])\d{3}[ .-]\d{4}(?![\w-])(?!\.[\w-])")
 PHONE_CUE_RE = re.compile(
    r"(?i)\b(?:call|calling|ring|phone|tel|telephone|mobile|cell|fax|text|dial|reach)\b")
 # A dotted number right after a version word is a version, not an address.
 VERSION_CUE_RE = re.compile(r"(?i)(?:\bv|\bversion|\bver\.?|\brelease|\bbuild|\bfirmware)\s*:?\s*$")
 # ── checksum-validated account identifiers (borrowed from Presidio's
 # recognizer list; only the ones a checksum can anchor are here) ──────────
 # IBAN: country code + 2 check digits + 11-30 alphanumerics, optional
-# spaces every 4. mod-97 == 1 decides (ISO 13616).
+# spaces every 4, in either case. mod-97 == 1 and the country's length
-IBAN_RE = re.compile(r"(?<![A-Za-z0-9])[A-Z]{2}\d{2}(?:[ ]?[A-Z0-9]{4}){2,7}(?:[ ]?[A-Z0-9]{1,4})?(?![A-Za-z0-9])")
+# decide (ISO 13616), so random text of the same shape is not claimed.
 IBAN_RE = re.compile(r"(?<![A-Za-z0-9])[A-Za-z]{2}\d{2}(?:[ ]?[A-Za-z0-9]{4}){2,7}(?:[ ]?[A-Za-z0-9]{1,4})?(?![A-Za-z0-9])")
 IBAN_LENGTHS = {
    "AD": 24, "AE": 23, "AL": 28, "AT": 20, "AZ": 28, "BA": 20, "BE": 16, "BG": 22,
    "BH": 22, "BR": 29, "CH": 21, "CR": 22, "CY": 28, "CZ": 24, "DE": 22, "DK": 18,
@@ -147,8 +178,17 @@ def _find_secrets(text):
 def _find_ibans(text):
    for m in IBAN_RE.finditer(text):
-        if _iban_ok(m.group(0)):
+        # a spaced IBAN followed by a four-letter word ("... 7034 then")
-            yield Finding(m.group(0), "BANK_ACCOUNT", m.start(), m.end())
+        # matches with the word attached; drop trailing groups until it checks
        cand = m.group(0)
        while True:
            if _iban_ok(cand):
                yield Finding(cand, "BANK_ACCOUNT", m.start(), m.start() + len(cand))
                break
            cut = cand.rfind(" ")
            if cut < 0:
                break
            cand = cand[:cut]
 def _find_abas(text):
@@ -188,12 +228,18 @@ def _find_macs(text):
        yield Finding(m.group(0), "MAC_ADDRESS", m.start(), m.end())
 def _after_version_word(text, start):
    return VERSION_CUE_RE.search(text[max(0, start - 12):start]) is not None
 def _find_ips(text):
    for m in IPV4_RE.finditer(text):
        try:
            ipaddress.IPv4Address(m.group(0))
        except ValueError:
            continue
        if _after_version_word(text, m.start()):
            continue
        yield Finding(m.group(0), "IP_ADDRESS", m.start(), m.end())
    for m in IPV6_RE.finditer(text):
        cand = m.group(0)
@@ -204,10 +250,23 @@ def _find_ips(text):
        yield Finding(cand, "IP_ADDRESS", m.start(), m.end())
 def _ssn_ok(area, group, serial):
    # never issued: area 000, 666 or 900-999, group 00, serial 0000
    a = int(area)
    return a != 0 and a != 666 and a < 900 and group != "00" and serial != "0000"
 def _find_ssns(text):
    for m in SSN_RE.finditer(text):
-        area = int(m.group(1))
+        if _ssn_ok(m.group(1), m.group(2), m.group(3)):
-        if area == 0 or area == 666 or area >= 900:
+            yield Finding(m.group(0), "SSN", m.start(), m.end())
    for m in SSN_SPACED_RE.finditer(text):
        if _ssn_ok(m.group(1), m.group(2), m.group(3)):
            yield Finding(m.group(0), "SSN", m.start(), m.end())
    for m in SSN_BARE_RE.finditer(text):
        if not _ssn_ok(m.group(1), m.group(2), m.group(3)):
            continue
        if not SSN_CUE_RE.search(text[max(0, m.start() - 40):m.start()]):
            continue
        yield Finding(m.group(0), "SSN", m.start(), m.end())
@@ -225,6 +284,18 @@ def _find_phones(text):
        if not 10 <= len(digits) <= 15:
            continue
        yield Finding(m.group(0), "PHONE", m.start(), m.end())
    for m in INTL_PHONE_RE.finditer(text):
        # E.164 allows at most 15 digits; a (0) trunk prefix is not one of them
        digits = re.sub(r"\D", "", m.group(0).replace("(0)", "", 1))
        if 8 <= len(digits) <= 15:
            yield Finding(m.group(0), "PHONE", m.start(), m.end())
    for m in NATIONAL_PHONE_RE.finditer(text):
        digits = re.sub(r"\D", "", m.group(0))
        if 10 <= len(digits) <= 11:
            yield Finding(m.group(0), "PHONE", m.start(), m.end())
    for m in LOCAL_PHONE_RE.finditer(text):
        if PHONE_CUE_RE.search(text[max(0, m.start() - 30):m.start()]):
            yield Finding(m.group(0), "PHONE", m.start(), m.end())
 # Priority order: earlier detectors own their spans outright. Secrets go
@@ -22,7 +22,7 @@ downgrade the sensor. Stdlib-only here; the gliner package is the one
 optional third-party import in the module and it is loaded lazily.
 """
-from typing import Callable, List, Tuple
+from typing import List, Tuple
 from . import llmclient
 from .detect import PII_TYPES
@@ -19,7 +19,8 @@ exists for the unit-test harness only. /rehydrate returns REAL values —
 callers are trusted surfaces only; every call lands in the vault's audit
 table.
-Config (env): PII_BIND (host:port, default 0.0.0.0:8091), LLM_URL
+Config (env): PII_BIND (host:port, default 127.0.0.1:8091; another address
 only when set), LLM_URL
 (default http://llm:8080), PII_DB (default /data/vault.db),
 PII_LLM_TIMEOUT seconds (default 300).
 """
@@ -38,7 +39,7 @@ _MAX_BODY = 2 * 1024 * 1024
 class Config:
    def __init__(self, env):
-        self.bind = env.get("PII_BIND", "0.0.0.0:8091")
+        self.bind = env.get("PII_BIND", "127.0.0.1:8091")
        self.llm_url = env.get("LLM_URL", "http://llm:8080")
        self.db = env.get("PII_DB", "/data/vault.db")
        self.api_key = env.get("PII_API_KEY", "")
@@ -47,12 +47,12 @@ class DetectTests(unittest.TestCase):
        self.assertEqual(self.find("ref 4111 1111 1111 1112"), set())
    def test_ipv4_owns_dotted_span(self):
-        found = self.find("host at 192.168.68.54 responded")
+        found = self.find("host at 198.51.100.54 responded")
-        self.assertIn(("192.168.68.54", "IP_ADDRESS"), found)
+        self.assertIn(("198.51.100.54", "IP_ADDRESS"), found)
-        self.assertNotIn(("192.168.68.54", "PHONE"), found)
+        self.assertNotIn(("198.51.100.54", "PHONE"), found)
    def test_ipv4_invalid_octet(self):
-        self.assertNotIn(("999.168.68.541", "IP_ADDRESS"), self.find("x 999.168.68.541"))
+        self.assertNotIn(("999.51.100.541", "IP_ADDRESS"), self.find("x 999.51.100.541"))
    def test_ipv6_and_mac_disambiguation(self):
        found = self.find("fe80::1ff:fe23:4567:890a from aa:bb:cc:dd:ee:ff")
@@ -127,6 +127,87 @@ class DetectTests(unittest.TestCase):
        self.assertEqual(self.find("the skate park (AKA the bowl) opens at 9"), set())
 class RulesOnlyFormatTests(unittest.TestCase):
    """The rules pass is all that flag mode runs, so each format here must be
    found with no recall model at all, and each lookalike must be left alone."""
    FOUND = [
        ("UK mobile, international", "Mobile: +44 7700 900123", "+44 7700 900123", "PHONE"),
        ("German, (0) trunk prefix", "Tel. +49 (0)30 1234567", "+49 (0)30 1234567", "PHONE"),
        ("international, no separators", "ring +442079460958 today", "+442079460958", "PHONE"),
        ("international, dots", "fax +33.6.39.98.12.34 now", "+33.6.39.98.12.34", "PHONE"),
        ("international, dashes", "cell +44-20-7946-0958.", "+44-20-7946-0958", "PHONE"),
        ("UK national, trunk 0", "call 07700 900123 later", "07700 900123", "PHONE"),
        ("German national, trunk 0", "Tel 030 1234567", "030 1234567", "PHONE"),
        ("German national, (0xx)", "Tel (030) 1234567", "(030) 1234567", "PHONE"),
        ("7-digit local after a phone word", "Ring 555-0142 for the gate code.", "555-0142", "PHONE"),
        ("7-digit local, call", "call me on 555 0199", "555 0199", "PHONE"),
        ("IBAN lowercase", "iban: gb82west12345698765432", "gb82west12345698765432", "BANK_ACCOUNT"),
        ("IBAN lowercase, spaced", "pay de89 3704 0044 0532 0130 00 now",
         "de89 3704 0044 0532 0130 00", "BANK_ACCOUNT"),
        ("IBAN followed by a four-letter word", "to BE68 5390 0754 7034 then",
         "BE68 5390 0754 7034", "BANK_ACCOUNT"),
        ("card with dots", "Card 4111.1111.1111.1111", "4111.1111.1111.1111", "CREDIT_CARD"),
        ("card with dashes", "card 5555-5555-5555-4444 ok", "5555-5555-5555-4444", "CREDIT_CARD"),
        ("card after no.", "card no.4111 1111 1111 1111", "4111 1111 1111 1111", "CREDIT_CARD"),
        ("card, 4-6-5 grouping", "amex 3782 822463 10005", "3782 822463 10005", "CREDIT_CARD"),
        ("SSN with spaces", "SSN 219 09 9999", "219 09 9999", "SSN"),
        ("SSN, no separators, keyword", "Social security number 219099999.", "219099999", "SSN"),
        ("SSN, no separators, SSN:", "ssn: 219099999", "219099999", "SSN"),
    ]
    LEFT_ALONE = [
        ("order number", "Order 555-0142 shipped"),
        ("order number, digits only", "order #20431877 is packed"),
        ("9-digit invoice id", "invoice 219099999 is due"),
        ("16 digits failing Luhn, spaces", "ref 4111 1111 1111 1112"),
        ("16 digits failing Luhn, dots", "ref 4111.1111.1111.1112"),
        ("IBAN shape, bad checksum, lowercase", "iban gb82west12345698765433"),
        ("IBAN shape, bad checksum, uppercase", "iban GB82WEST12345698765433 ok"),
        ("version string", "version 1.2.3.4 shipped"),
        ("version string, v", "upgrade to v1.2.3.4 today"),
        ("ISO date", "released 2026-08-03 at noon"),
        ("US date", "Date of birth: 03/14/1987"),
        ("dotted date", "seen on 14.03.1987 in the hall"),
        ("SSN shape, group 00", "ssn 219-00-9999"),
        ("short number after +", "add +12 345 to it"),
        ("spaced digits inside a longer run", "lot 4 219 09 9999 7"),
    ]
    def find(self, text):
        return {(f.value, f.type) for f in detect.detect(text)}
    def test_each_format_is_found(self):
        for label, text, value, typ in self.FOUND:
            with self.subTest(label):
                self.assertIn((value, typ), self.find(text))
    def test_lookalikes_are_left_alone(self):
        for label, text in self.LEFT_ALONE:
            with self.subTest(label):
                self.assertEqual(self.find(text), set())
    def test_rules_only_sterilize_leaves_no_value(self):
        # what flag mode does: the pipeline with no recall model
        with tempfile.TemporaryDirectory() as tmp:
            vault = Vault(os.path.join(tmp, "v.db"))
            for label, text, value, _ in self.FOUND:
                with self.subTest(label):
                    out, _ = pipeline.sterilize(text, "s", vault, lambda t: [],
                                                escape_held_literals=True)
                    self.assertNotIn(value, out)
                    self.assertEqual(pipeline.rehydrate(out, "s", vault)[0], text)
            for label, text in self.LEFT_ALONE:
                with self.subTest(label):
                    out, _ = pipeline.sterilize(text, "s", vault, lambda t: [],
                                                escape_held_literals=True)
                    self.assertEqual(out, text)
    def test_ip_address_is_still_found_after_other_words(self):
        self.assertIn(("203.0.113.42", "IP_ADDRESS"), self.find("Server at 203.0.113.42 refused."))
        self.assertIn(("203.0.113.42", "IP_ADDRESS"), self.find("server 203.0.113.42 version 2"))
 class VaultTests(unittest.TestCase):
    def setUp(self):
        self.tmp = tempfile.TemporaryDirectory()
@@ -672,6 +753,14 @@ class ServiceTests(unittest.TestCase):
        with self.assertRaises(SystemExit):
            service.Config({"PII_DB": "x"})
    def test_default_bind_is_loopback(self):
        # /rehydrate returns real values over plain HTTP: only an explicit
        # setting may open the service beyond this machine
        self.assertEqual(service.Config({"PII_DB": "x", "PII_API_KEY": "k"}).bind,
                         "127.0.0.1:8091")
        self.assertEqual(service.Config({"PII_DB": "x", "PII_API_KEY": "k",
                                         "PII_BIND": "0.0.0.0:8091"}).bind, "0.0.0.0:8091")
    def test_fail_closed_on_unstartable_recall_backend(self):
        with self.assertRaises(SystemExit) as ctx:
            service.Config({"PII_DB": "x", "PII_API_KEY": "k", "PII_RECALL": "nope"})
@@ -17,31 +17,41 @@ lands here is published far past where it came from.
 This fails on:
-  term       a word or phrase from TERMS: the owner's and members' details that
+  term       a word or phrase from TERMS, a list of details about real people
-             were once copied into a repository (bank, employer, account
+             that must not appear in this repository. They are held as keyed
             digits, places, names, addresses, a device, a customer, a person's
             project, a Host's content, record ids). They are held as keyed
             digests, so this file does not publish what it keeps out, and a
             guess cannot be tested against the table without the key: every
             1-3 token n-gram of a line is normalised (lowercase letters and
             digits, joined by one space), HMAC-SHA256'd with the maintainers'
-             key and looked up. The key lives outside the repository
+             key and looked up. A backslash escape (\\n, \\t, \\x41, \\u00e9...)
             separates tokens as whitespace would, so a term written right
             after one is still found. The key lives outside the repository
             (NO_PERSONAL_DATA_KEY_FILE, default
             /etc/privatae/keys/no-personal-data.hmac), as do the plain list
             and the script that hashes it. Where the key is absent (a CI job,
             which holds no secrets) this class is skipped and the run says
-             so; the other classes always run. A key other than the one TERMS
+             so; the other classes still run. A key other than the one TERMS
             was made with fails the run (KEY_CHECK), since it would match
             nothing.
-  entity-id  a real entity id that labels a person's Host (RESERVED_ENTITY_IDS)
+  entity-id  a reserved entity id, one that labels a person's Host, in
-             in entity-N, entity N, entity_id=N, "entity_id": N, entity id N,
+             entity-N, entity N, entity_id=N, "entity_id": N, entity id N,
-             Host N, Host #N or host-N form; examples and tests use 9000 and up
+             Host N, Host #N or host-N form; examples and tests use 9000 and
             up. The ids live outside the repository, one per line
             (RESERVED_ENTITY_IDS_FILE, default
             /etc/privatae/reserved-entity-ids). Where that file is absent (a
             CI job) this class is skipped and the run says so; a file that
             holds no id, or a line that is not one, fails the run, since it
             would match nothing.
  provenance a line that says its data is real: "real rows" followed by
             "that" or "from", an all-caps "REAL" row, "verbatim" from a
             transcript or a run, a row "read off" an entity...
  email      a mailbox at a personal mail provider (gmail, yahoo, proton...)
  phone      a North American number outside the 555 fiction range
 A run of 24 or more base64 or base64url characters that decodes to mostly
 printable UTF-8 is decoded and its text checked for every class as well, so a
 value is not let through by being encoded.
 Every tracked text file is scanned, file names included, and every text member
 of each zip-format bundle (an app bundle is one), since a bundle is what
 installs. Lockfiles and other binary files are not.
@@ -63,6 +73,8 @@ Exit 0 = clean; exit 1 + FAIL lines otherwise.
 from __future__ import annotations
 import base64
 import binascii
 import hmac
 import os
 import re
@@ -118,13 +130,43 @@ TERMS: frozenset[str] = frozenset({
 })
 _NGRAM_MAX = 3
-# Entity ids of real Hosts that belong to people (the operator's own, members',
+# The reserved entity ids are read from a file outside the repository, one id
-# the platform's Security entity and the two security-test Hosts).
+# per line; blank lines and lines starting with # are ignored.
-RESERVED_ENTITY_IDS = (3, 102, 122, 123, 128, 163, 164)
+RESERVED_ENTITY_IDS_FILE = os.environ.get("RESERVED_ENTITY_IDS_FILE", "/etc/privatae/reserved-entity-ids")
-_IDS = "|".join(str(i) for i in RESERVED_ENTITY_IDS)
+
-ENTITY_ID = re.compile(
+
 def load_reserved_ids(path: str = RESERVED_ENTITY_IDS_FILE):
    """(ids, error). A missing file is (None, None): the class is skipped. A
    file that cannot be read, holds a line that is not an id, or holds no id
    at all is (None, error): a list that matches nothing must not pass."""
    try:
        with open(path, encoding="utf-8") as f:
            lines = f.read().splitlines()
    except FileNotFoundError:
        return None, None
    except (OSError, UnicodeDecodeError) as e:
        return None, "the reserved entity ids at %s cannot be read: %s" % (path, e)
    ids = []
    for n, raw in enumerate(lines, 1):
        s = raw.strip()
        if not s or s.startswith("#"):
            continue
        if not s.isdigit():
            return None, "%s:%d is not an entity id (one id per line)" % (path, n)
        ids.append(int(s))
    if not ids:
        return None, "%s holds no entity id, so the entity-id check would match nothing" % path
    return tuple(ids), None
 def entity_id_pattern(ids):
    """The entity-id class for these ids, or None (the class is off)."""
    if not ids:
        return None
    alts = "|".join(str(i) for i in sorted(set(ids), key=lambda i: -len(str(i))))
    return re.compile(
        r"\b(?:entit(?:y|ies)(?:[-_ ]|_id\s*==?\s*|_id[\"']?\s*:\s*|\s+id\s+|\s+#)"
-    r"|Host\s+#?|host-)(?:%s)\b" % _IDS, re.I)
+        r"|Host\s+#?|host-)(?:%s)\b" % alts, re.I)
 PROVENANCE = re.compile(
    r"\breal rows? (?:that|from|read|copied|taken|off)\b"
@@ -181,13 +223,25 @@ def key_matches(key: bytes) -> bool:
    return hmac.compare_digest(digest(KEY_CHECK_TEXT, key), KEY_CHECK)
 # A backslash escape as source and data files write it: a named control
 # character, a hex or unicode escape, or an octal one. "a\nb" is two words, but
 # the tokenizer alone reads "a" and "nb". The line is read once as written and
 # once with each escape turned into a space; a hit in either counts. Reading it
 # as written too keeps a term that follows a backslash which is not an escape
 # (a Windows path: C:\name) whole.
 _ESCAPE = re.compile(r"\\(?:x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8}|[0-7]{1,3}|[abefnrtv])")
 class TermScanner:
    """Finds TERMS in a line by hashing its 1-3 token n-grams. Digests are
-    cached, so a repeated n-gram is hashed once."""
+    cached, so a repeated n-gram is hashed once. It also carries the
    entity-id pattern for the reserved ids (None: that class is off)."""
-    def __init__(self, terms, key):
+    def __init__(self, terms, key, entity_ids=None):
        self.terms = terms
        self.key = key
        self.entity_ids = entity_ids
        self.entity_re = entity_id_pattern(entity_ids)
        self._cache = {}
    def _hit(self, gram: str) -> bool:
@@ -195,16 +249,64 @@ class TermScanner:
            self._cache[gram] = digest(gram, self.key) in self.terms
        return self._cache[gram]
    def _grams(self, text: str) -> dict:
        toks = [t.lower() for t in _TOKEN.findall(text)]
        hits = {}
        for n in range(1, _NGRAM_MAX + 1):
            for i in range(len(toks) - n + 1):
                gram = " ".join(toks[i:i + n])
                if self._hit(gram):
                    hits[gram] = hits.get(gram, 0) + 1
        return hits
    def scan(self, line: str) -> int:
        if self.key is None:
            return 0
-        toks = [t.lower() for t in _TOKEN.findall(line)]
+        hits = self._grams(line)
-        found = 0
+        if "\\" in line:
-        for n in range(1, _NGRAM_MAX + 1):
+            for gram, n in self._grams(_ESCAPE.sub(" ", line)).items():
-            for i in range(len(toks) - n + 1):
+                hits[gram] = max(hits.get(gram, 0), n)
-                if self._hit(" ".join(toks[i:i + n])):
+        return sum(hits.values())
-                    found += 1
+
-        return found
+
 # ── values inside base64 ─────────────────────────────────────────────────────
 # A run of base64 or base64url characters, with its padding. The run is decoded
 # at each of the four offsets, since it may be glued to the text before it or
 # cut by a window edge; an offset whose bytes are mostly printable UTF-8 is the
 # text. Random bytes almost never are, so identifiers, paths and digests that
 # happen to be base64-shaped decode to nothing worth reading.
 _B64_RUN = re.compile(r"[A-Za-z0-9+/_-]{24,}={0,2}")
 _B64_MIN = 24
 _B64_PRINTABLE = 0.85
 # A decoded text may itself hold base64; it is followed this many levels down.
 _B64_DEPTH = 2
 def _printable_share(text: str) -> float:
    if not text:
        return 0.0
    ok = sum(1 for ch in text if ch in "\t\n\r" or (ch.isprintable() and ch != "\ufffd"))
    return ok / len(text)
 def b64_texts(line: str):
    """The text of every base64 run in line that decodes to mostly printable
    UTF-8, one per run."""
    for m in _B64_RUN.finditer(line):
        run = m.group(0).rstrip("=").replace("-", "+").replace("_", "/")
        for off in range(4):
            body = run[off:]
            body = body[:len(body) - len(body) % 4] if len(body) % 4 == 1 else body
            if len(body) < _B64_MIN:
                break
            try:
                raw = base64.b64decode(body + "=" * (-len(body) % 4), validate=True)
            except (ValueError, binascii.Error):
                continue
            text = raw.decode("utf-8", "replace")
            if _printable_share(text) >= _B64_PRINTABLE:
                yield text
                break
 # A minified asset or a bundle member can hold one line of megabytes. It is read
@@ -241,15 +343,65 @@ def line_hits(line: str, scanner: TermScanner):
 # Classes whose line would carry the personal value itself: their FAIL line
 # names the file, the line and the class, never the text, so a CI log does not
-# publish what the gate keeps out.
+# publish what the gate keeps out. A line that holds one shows no text for any
 # finding, since another class's excerpt of the line would carry the value too.
 _QUIET = {"term", "email", "phone"}
-def line_findings(line: str, scanner: TermScanner):
+_PATH_SEP = re.compile(r"([/!])")
 def shown_path(label: str, scanner: TermScanner) -> str:
    """label as a FAIL line shows it. A path component (a directory, a file
    name, a bundle member's name) that holds a quiet value is replaced by a
    placeholder giving its depth, so the line says where without the text. A
    value that runs across components withholds them all."""
    def quiet(text):
        return any(cls in _QUIET for cls, _ in line_findings(text, scanner))
    if not quiet(label):
        return label
    parts = _PATH_SEP.split(label)
    def masked(every):
        out, depth = [], 0
        for p in parts:
            if p in ("/", "!"):
                out.append(p)
                continue
            depth += 1
            out.append("<withheld %d>" % depth if every or quiet(p) else p)
        return "".join(out)
    shown = masked(False)
    return masked(True) if quiet(shown) else shown
 def line_report(label: str, n: int, line: str, scanner: TermScanner, allow=(), used=None,
                skip_entity_ids: bool = False):
    """(class, FAIL text) for each finding on line n of label that no allow
    entry covers."""
    used = set() if used is None else used
    hits = list(line_hits(line, scanner))
    quiet = any(cls in _QUIET for _part, cls, _what in hits)
    out = []
    where = None
    for part, cls, what in hits:
        if (cls == "entity-id" and skip_entity_ids) or allowed(allow, label, cls, part, used):
            continue
        if where is None:
            where = shown_path(label, scanner)
        shown = "" if quiet else part.strip()[:140]
        out.append((cls, ("%s:%d  [%s: %s]  %s" % (where, n, cls, what, shown)).rstrip()))
    return out
 def line_findings(line: str, scanner: TermScanner, _depth: int = 0):
    out = []
    for _ in range(scanner.scan(line)):
        out.append(("term", "a listed term"))
-    for m in ENTITY_ID.finditer(line):
+    if scanner.entity_re is not None:
        for m in scanner.entity_re.finditer(line):
            out.append(("entity-id", m.group(0)))
    for m in PROVENANCE.finditer(line):
        out.append(("provenance", m.group(0)))
@@ -258,6 +410,10 @@ def line_findings(line: str, scanner: TermScanner):
    for m in PHONE.finditer(line):
        if (m.group(2) or m.group(5)) != "555":
            out.append(("phone", "a phone number"))
    if _depth < _B64_DEPTH:
        for text in b64_texts(line):
            for cls, what in line_findings(text, scanner, _depth + 1):
                out.append((cls, what if what.endswith("(in base64)") else what + " (in base64)"))
    return out
@@ -363,32 +519,61 @@ def self_test():
    fails = []
    probe_key = os.urandom(32)
    # Invented ids stand in for the reserved ones, which this file never holds.
    probe_ids = (9101, 9202, 9303)
    sc = TermScanner(frozenset(digest(g, probe_key) for g in ("zorblaxquent", "quilmar vesk", "tam orvel prax")),
-                     probe_key)
+                     probe_key, probe_ids)
    def classes(line):
        return {c for c, _ in line_findings(line, sc)}
    def b64(s: str, url: bool = False) -> str:
        enc = base64.urlsafe_b64encode if url else base64.b64encode
        return enc(live(s).encode("utf-8")).decode("ascii").rstrip("=" if url else "")
    positives = [(c, live(s)) for c, s in [
        ("term", "pays into the Zorblaxquent ledger"), ("term", "QUILMAR-vesk sent it"),
        ("term", "a tam_orvel.prax file"),
-        ("entity-id", "entity_id=|3,"), ("entity-id", '"entity_id": |128'), ("entity-id", "Entity |164's app"),
+        # A term right after a backslash escape, as a string literal or a JSON
-        ("entity-id", "kubectl -n entity-|3 exec"), ("entity-id", "pod entity-|3-0 restarted"), ("entity-id", "measured on Host #|163"),
+        # value writes it, and after a backslash that is not one.
-        ("entity-id", "gitea_user = 'host-|3'"), ("entity-id", "entity id |122 knocked"),
+        ("term", r'"line one\nZorblax|quent"'), ("term", r"col\tzorblax|quent\tcol"),
        ("term", r"end\r\nquilmar| vesk"), ("term", r"a\x41zorblax|quent"), ("term", r"x\u00e9zorblax|quent"),
        ("term", r"C:\Users\zorblax|quent\notes"),
        ("entity-id", "entity_id=|9101,"), ("entity-id", '"entity_id": |9202'), ("entity-id", "Entity |9303's app"),
        ("entity-id", "kubectl -n entity-|9101 exec"), ("entity-id", "pod entity-|9101-0 restarted"),
        ("entity-id", "measured on Host #|9303"), ("entity-id", "gitea_user = 'host-|9101'"),
        ("entity-id", "entity id |9202 knocked"),
        ("provenance", "the real| rows that fired on the Host"), ("provenance", "the REAL| row 9"),
        ("provenance", "verbatim| from the transcript"), ("provenance", "read| off entity 9001"),
        ("email", "mail ana.b@gm|ail.com today"), ("email", "x@proto|nmail.com"),
        ("phone", "call +1 (2|02) 555-|0100 or +1 30|1 444 1234"), ("phone", "+1|3014441234"),
    ]]
    # Every class is read inside base64 and base64url, padded or not, whole or
    # glued to the text before it.
    positives += [
        ("term", 'data: "%s"' % b64("a note that names Zorblax|quent outright")),
        ("term", "token=%s" % b64("pays QUILMAR| vesk ~~ été ok??", url=True)),
        ("term", "xyz" + b64("a note that names Zorblax|quent outright")),
        ("term", b64(b64("twice over: zorblax|quent, still found"))),
        ("entity-id", "blob %s end" % b64("the pod entity-|9101-0 restarted twice")),
        ("email", "%s" % b64("write to ana.b@gm|ail.com about it")),
    ]
    negatives = [live(s) for s in [
-        "zorblax quent", "quilmar", "a vesk quilmar", "orvel prax tam",
+        "zorblax quent", "quilmar", "a vesk quilmar", "orvel prax tam", r"zorblax\nquent",
-        "entity_id=9003", "entity_id=30", "entity 1023", "entity-3x", "host-31", "Host 7",
+        "entity_id=91011", "entity_id=910", "entity 92021", "entity-9101x", "host-91012", "Host 9104",
-        "entity-9310-0.svc", "the v0.3.128 tag", "entity_id = <id>",
+        "entity-9310-0.svc", "the v0.9.9101 tag", "entity_id = <id>",
        "real rows in a throwaway container", "the real row replaces the echo",
        "returns the text verbatim", "ana@example.com", "a•••@gmail.com shows the first letter",
        "+1 415 555 0199", "+1|5555550100", "ids 2026091812345", "port 8800, 4096 MiB",
        "MemoryHigh=15032385536 MemoryMax=17179869184", "2026-10-07 12:00",
        "widths 278 584 556 1015 667 722",
        # Base64-shaped text that is not base64 of text: identifiers, paths,
        # digests and random bytes are not decoded into findings.
        "TestPublishedSourceCarriesNoOldInternalNameAnywhere",
        "/usr/local/lib/python3/site-packages/zorbtool/__init__",
        "sha256:1ae0735f00daffa3aaf1363a5184c0d2dc55c78e3db4ec70241cdac97bf84b59",
        base64.b64encode(bytes(range(200, 256))).decode("ascii"),
        b64("a plain note with nothing listed in it at all"),
    ]]
    for cls, line in positives:
        if cls not in classes(line):
@@ -408,11 +593,41 @@ def self_test():
    # The real table is wired: every entry is a digest of the right shape.
    if not TERMS or not all(re.fullmatch(r"[0-9a-f]{32}", k) for k in TERMS):
        fails.append("self-test: TERMS is empty or malformed")
    # A line holding a quiet value shows no text for any finding on it: here a
    # provenance phrase beside a mailbox, and one base64 run holding both an
    # entity id and a term.
    blob = b64("the pod entity-|9101-0 belongs to zorblax|quent")
    for line, want, secret in ((live("the real| rows that came from ana.b@gm|ail.com"),
                                {"provenance", "email"}, live("ana.b@gm|ail")),
                               ("note %s end" % blob, {"entity-id", "term"}, blob[:24])):
        got = line_report("x.md", 1, line, sc)
        if {c for c, _ in got} != want or any(secret in t for _c, t in got):
            fails.append("self-test: a line with a quiet value is reported without its text, got %s" % got)
    # A path that holds a quiet value is shown with that component withheld,
    # by depth; one split across components withholds them all.
    for label, want in (("docs/zorblaxquent-notes.md", "docs/<withheld 2>"),
                        ("app.bundle!notes/" + live("ana.b@gm|ail.com") + ".txt", "app.bundle!notes/<withheld 3>"),
                        ("quilmar/vesk.md", "<withheld 1>/<withheld 2>"),
                        ("docs/readme.md", "docs/readme.md")):
        if shown_path(label, sc) != want:
            fails.append("self-test: path %r shown as %r, want %r" % (label, shown_path(label, sc), want))
    got = line_report("docs/" + live("ana.b@gm|ail.com") + ".md", 1, live("the real| rows that"), sc)
    if [t for _c, t in got] != [live("docs/<withheld 2>:1  [provenance: real| rows that]  the real| rows that")]:
        fails.append("self-test: a FAIL line's path is shown withheld, got %s" % got)
    # A term is counted once per place it stands, however many ways it is read.
    got = [c for c, _ in line_findings(live(r"a\nzorblax|quent b"), sc)]
    if got != ["term"]:
        fails.append("self-test: a term after an escape is one hit, got %s" % got)
    # Without a key the term class is off and nothing else changes.
-    keyless = TermScanner(sc.terms, None)
+    planted = live("the zorblax|quent ledger, entity_id=|9101, x@proto|nmail.com")
-    got = {c for c, _ in line_findings(live("the zorblax|quent ledger, entity_id=|3, x@proto|nmail.com"), keyless)}
+    keyless = TermScanner(sc.terms, None, probe_ids)
    got = {c for c, _ in line_findings(planted, keyless)}
    if got != {"entity-id", "email"}:
        fails.append("self-test: without a key only the term class is off, got %s" % sorted(got))
    # Without the ids the entity-id class is off and nothing else changes.
    got = {c for c, _ in line_findings(planted, TermScanner(sc.terms, probe_key))}
    if got != {"term", "email"}:
        fails.append("self-test: without the ids only the entity-id class is off, got %s" % sorted(got))
    # A key other than the one TERMS was made with is told apart from it.
    if key_matches(probe_key) or key_matches(b""):
        fails.append("self-test: a key TERMS was not made with passes the key check")
@@ -426,6 +641,18 @@ def self_test():
        if len(entries) != 1 or len(errors) != 2:
            fails.append("self-test: the allow file reader kept %d entries and %d errors, want 1 and 2"
                         % (len(entries), len(errors)))
        # The reserved ids are read one per line; a missing file turns the
        # class off, and a file that would match nothing is an error.
        idf = os.path.join(td, "ids")
        for body, want in (("# reserved\n9101\n\n 9202 \n", ((9101, 9202), False)),
                           ("9101\nentity-9202\n", (None, True)), ("# none\n\n", (None, True)), ("", (None, True))):
            with open(idf, "w", encoding="utf-8") as f:
                f.write(body)
            ids, err = load_reserved_ids(idf)
            if (ids, err is not None) != want:
                fails.append("self-test: reserved ids %r read as %r, %r" % (body, ids, err))
        if load_reserved_ids(os.path.join(td, "absent")) != (None, None):
            fails.append("self-test: a missing reserved-ids file is not (None, None)")
        # A zip-format bundle is opened by its signature, whatever it is named
        # (an app bundle, or a .zip such as a browser extension a site serves),
        # and its text members are read.
@@ -469,13 +696,20 @@ def main(argv) -> int:
    key = load_key()
    if key is None:
        print("  NOTE  term check skipped: no key at %s (set NO_PERSONAL_DATA_KEY_FILE); "
-              "the entity-id, provenance, email and phone checks still run" % KEY_FILE)
+              "the other checks still run" % KEY_FILE)
    elif not key_matches(key):
        fails.append("the key at %s is not the one TERMS was made with, so it would match "
                     "nothing: point NO_PERSONAL_DATA_KEY_FILE at that key, or remake TERMS and KEY_CHECK" % KEY_FILE)
        print("  FAIL  %s" % fails[-1])
        key = None
-    scanner = TermScanner(TERMS, key)
+    ids, ids_error = load_reserved_ids()
    if ids_error:
        fails.append(ids_error)
        print("  FAIL  %s" % ids_error)
    elif ids is None:
        print("  NOTE  entity-id check skipped: no reserved ids at %s (set RESERVED_ENTITY_IDS_FILE); "
              "the other checks still run" % RESERVED_ENTITY_IDS_FILE)
    scanner = TermScanner(TERMS, key, ids)
    used = set()
    findings = []
    counts = {}
@@ -488,21 +722,24 @@ def main(argv) -> int:
            continue
        for cls, what in line_findings(rel, scanner):
            if not allowed(allow, rel, cls, rel, used):
-                findings.append("%s  [%s: %s]  (file name)" % (rel, cls, what))
+                findings.append("%s  [%s: %s]  (file name)" % (shown_path(rel, scanner), cls, what))
                counts[cls] = counts.get(cls, 0) + 1
        for label, text in texts(root, rel):
            n_texts += 1
            if text is None:
                fails.append("%s has a zip signature but does not open as a zip, so its "
-                             "members cannot be read" % label)
+                             "members cannot be read" % shown_path(label, scanner))
                print("  FAIL  %s" % fails[-1])
                continue
            # A bundle member's name installs with it, like a file name.
            member = label[len(rel) + 1:]
            for cls, what in (line_findings(member, scanner) if member else ()):
                if not allowed(allow, label, cls, member, used):
                    findings.append("%s  [%s: %s]  (bundle member name)" % (shown_path(label, scanner), cls, what))
                    counts[cls] = counts.get(cls, 0) + 1
            for i, line in enumerate(text.splitlines(), 1):
-                for part, cls, what in line_hits(line, scanner):
+                for cls, finding in line_report(label, i, line, scanner, allow, used, label == allow_rel):
-                    if (cls == "entity-id" and label == allow_rel) or allowed(allow, label, cls, part, used):
+                    findings.append(finding)
                        continue
                    shown = "" if cls in _QUIET else part.strip()[:140]
                    findings.append(("%s:%d  [%s: %s]  %s" % (label, i, cls, what, shown)).rstrip())
                    counts[cls] = counts.get(cls, 0) + 1
    for f in findings:
        print("  FAIL  %s" % f)