diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 7ae8e6a..ca41ad1 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -17,6 +17,8 @@ env: PY_CURRENT: python@sha256:7c61056e61ac89e852de05f3dc6fa51a6dd2181797bceed46aa725dd7cb2cd3b # 3.13-slim # The PyYAML release bastion_agent/build.py vendors; the manifest loader imports it. PYYAML: pyyaml==6.0.3 + # The linter: every name a module imports is used, and none is undefined. + PYFLAKES: pyflakes==4.0.3 # The pull request's own head, or the pushed commit. WANT: ${{ gitea.event.pull_request.head.sha || gitea.sha }} @@ -36,17 +38,27 @@ jobs: run: | docker run --rm -v "$PWD":/src:ro -w /src "$PY_CURRENT" \ python -m unittest -v test_syntax_floor + - name: pyflakes + run: | + docker run --rm -v "$PWD":/src:ro -w /src -e PYFLAKES "$PY_CURRENT" sh -euc ' + pip install -q --disable-pip-version-check --root-user-action=ignore "$PYFLAKES" + python -m pyflakes bastion_agent pii scripts test_syntax_floor.py' - name: every test, on Python 3.8 and on a current Python - # age is installed so the seal round trips run instead of skipping. + # age is installed so the seal round trips run, and git so the release + # verifier's end-to-end tests do. Every test runs here: a skipped one + # fails the job (the only skips the suite has are for Windows, or for a + # tool missing, and this job installs the tools). run: | set -euo pipefail for img in "$PY_FLOOR" "$PY_CURRENT"; do docker run --rm -v "$PWD":/src:ro -w /src -e PYYAML "$img" sh -euc ' python --version apt-get -qq update >/dev/null - apt-get -qq install -y --no-install-recommends age >/dev/null + apt-get -qq install -y --no-install-recommends age git >/dev/null pip install -q --disable-pip-version-check --root-user-action=ignore "$PYYAML" - python -m unittest discover .' + python -m unittest discover -v . > /tmp/tests.log 2>&1 || { cat /tmp/tests.log; exit 1; } + grep -E "^(Ran|OK|FAILED)" /tmp/tests.log + if grep -E "\.\.\. skipped|^(OK|FAILED) \(.*skipped=" /tmp/tests.log; then echo "a test was skipped; CI runs every test"; exit 1; fi' done personal-data: @@ -62,9 +74,14 @@ jobs: test "$(git rev-parse HEAD)" = "$WANT" - name: no person's data in the tree # Reads files, runs nothing from the tree. The term check needs the - # maintainers' key, which a CI job never holds, so here it is skipped and - # the run says so; the entity-id, provenance, email and phone checks run. - run: python3 scripts/test_no_personal_data.py + # maintainers' key and the entity-id check the reserved ids, which a CI + # job never holds, so the job names files that do not exist: both + # classes are skipped and the run prints a NOTE for each; the + # provenance, email and phone checks run. Left at their defaults, a + # path that exists but cannot be read fails the run instead. + run: | + none="$PWD/.ci-holds-none/none" + NO_PERSONAL_DATA_KEY_FILE="$none" RESERVED_ENTITY_IDS_FILE="$none" python3 scripts/test_no_personal_data.py reproducible: needs: test diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..31c50f3 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,116 @@ +# The checks of .gitea/workflows/ci.yml, for the copy of this repository on +# GitHub. Gitea reads .gitea/workflows when it exists and then ignores this +# directory, so each forge runs one of the two files, never both. +# +# The jobs, images and commands are the same as there. No action from the +# marketplace is used: the commit is fetched with git from this repository's +# public URL, so nothing outside this file and the pinned images runs. The job +# token can only read. +name: ci + +on: + push: + branches: [main] + pull_request: + branches: [main] + +permissions: + contents: read + +env: + PY_FLOOR: python@sha256:1d52838af602b4b5a831beb13a0e4d073280665ea7be7f69ce2382f29c5a613f # 3.8-slim + PY_CURRENT: python@sha256:7c61056e61ac89e852de05f3dc6fa51a6dd2181797bceed46aa725dd7cb2cd3b # 3.13-slim + # The PyYAML release bastion_agent/build.py vendors; the manifest loader imports it. + PYYAML: pyyaml==6.0.3 + # The linter: every name a module imports is used, and none is undefined. + PYFLAKES: pyflakes==4.0.3 + # The pull request's own head, or the pushed commit. + WANT: ${{ github.event.pull_request.head.sha || github.sha }} + REPO_URL: ${{ github.server_url }}/${{ github.repository }}.git + +jobs: + test: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - name: fetch the commit + run: | + set -euo pipefail + git init -q . + git fetch -q --depth=1 "$REPO_URL" "$WANT" + git checkout -q --detach FETCH_HEAD + test "$(git rev-parse HEAD)" = "$WANT" + - name: the 3.8 syntax floor + run: | + docker run --rm -v "$PWD":/src:ro -w /src "$PY_CURRENT" \ + python -m unittest -v test_syntax_floor + - name: pyflakes + run: | + docker run --rm -v "$PWD":/src:ro -w /src -e PYFLAKES "$PY_CURRENT" sh -euc ' + pip install -q --disable-pip-version-check --root-user-action=ignore "$PYFLAKES" + python -m pyflakes bastion_agent pii scripts test_syntax_floor.py' + - name: every test, on Python 3.8 and on a current Python + # age is installed so the seal round trips run, and git so the release + # verifier's end-to-end tests do. Every test runs here: a skipped one + # fails the job (the only skips the suite has are for Windows, or for a + # tool missing, and this job installs the tools). + run: | + set -euo pipefail + for img in "$PY_FLOOR" "$PY_CURRENT"; do + docker run --rm -v "$PWD":/src:ro -w /src -e PYYAML "$img" sh -euc ' + python --version + apt-get -qq update >/dev/null + apt-get -qq install -y --no-install-recommends age git >/dev/null + pip install -q --disable-pip-version-check --root-user-action=ignore "$PYYAML" + python -m unittest discover -v . > /tmp/tests.log 2>&1 || { cat /tmp/tests.log; exit 1; } + grep -E "^(Ran|OK|FAILED)" /tmp/tests.log + if grep -E "\.\.\. skipped|^(OK|FAILED) \(.*skipped=" /tmp/tests.log; then echo "a test was skipped; CI runs every test"; exit 1; fi' + done + + personal-data: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - name: fetch the commit + run: | + set -euo pipefail + git init -q . + git fetch -q --depth=1 "$REPO_URL" "$WANT" + git checkout -q --detach FETCH_HEAD + test "$(git rev-parse HEAD)" = "$WANT" + - name: no person's data in the tree + # Reads files, runs nothing from the tree. The term check needs the + # maintainers' key and the entity-id check the reserved ids, which + # this job never holds, so both are skipped and the run prints a NOTE + # for each; the provenance, email and phone checks run. + run: | + set -euo pipefail + docker run --rm -v "$PWD":/src:ro -w /src "$PY_CURRENT" \ + python -I scripts/test_no_personal_data.py --root /src + + reproducible: + needs: test + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - name: fetch the commit + run: | + set -euo pipefail + git init -q . + git fetch -q --depth=1 "$REPO_URL" "$WANT" + git checkout -q --detach FETCH_HEAD + test "$(git rev-parse HEAD)" = "$WANT" + - name: two builds on each Python produce one sha256 + # Anyone must be able to rebuild a release and get the published bytes, + # so the .pyz may not depend on the build or the Python that made it. + run: | + set -euo pipefail + for img in "$PY_FLOOR" "$PY_CURRENT"; do + docker run --rm -v "$PWD":/src:ro -w /src -e PYYAML "$img" sh -euc ' + pip install -q --disable-pip-version-check --root-user-action=ignore "$PYYAML" + python -m bastion_agent.build --out /tmp/a | awk "{print \$NF}" + python -m bastion_agent.build --out /tmp/b | awk "{print \$NF}"' + done | tee digests + test "$(wc -l < digests)" -eq 4 + test "$(sort -u digests | wc -l)" -eq 1 + echo "every build is $(head -1 digests)" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..de7d862 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,174 @@ +# Attach the release files to the GitHub release of a tag v. +# +# GitHub only: Gitea reads .gitea/workflows when it exists and then ignores +# this directory, so the forge's copy of this repository never runs this file. +# +# The files are not uploaded from anywhere else. This job rebuilds them from +# the tag and attaches them only when they are the published release: +# +# 1. bastion_agent/verify_release.py fetches the live manifest +# (privatae.ai/bastion-agent/version.json) and its signature, checks the +# signature against the release keys the bastion pins, refuses unless the +# manifest names this tag's version and the tag is annotated with the line +# `built from `, rebuilds the .pyz from the tag with that +# tag's own build.py, and compares the sha256 with the manifest; +# 2. only then are the rebuilt .pyz, its .sha256, the manifest (its exact +# bytes) and the signature file attached to the release, which is created +# when the tag has none, with that version's CHANGELOG section. +# +# The live manifest names only the newest release, so a release can be given +# its files only while it is the published one. Older releases, whose manifest +# is no longer served, cannot be filled in afterwards. A run that failed (the +# tag pushed before the release went live, say) is run again from the Actions +# tab with the tag as input. +# +# No action from the marketplace is used, and nothing is fetched with a +# credential: the repository is read from its public URL. The job may write +# to the repository's releases; the token is handed to the last step only. +name: release + +on: + push: + tags: ["v*.*.*"] + workflow_dispatch: + inputs: + tag: + description: "the release tag, vX.Y.Z" + required: true + +permissions: + contents: read + +env: + REPO_URL: ${{ github.server_url }}/${{ github.repository }}.git + TAG: ${{ github.event.inputs.tag || github.ref_name }} + +defaults: + run: + shell: bash -euo pipefail {0} + +jobs: + release: + runs-on: ubuntu-24.04 + timeout-minutes: 20 + permissions: + contents: write + steps: + - name: the tag is a release tag + run: | + printf '%s\n' "$TAG" | grep -Exq 'v[0-9]+\.[0-9]+\.[0-9]+' \ + || { echo "not a release tag: $TAG"; exit 1; } + - name: fetch the repository and its tags + # verify_release.py is run from the commit this workflow runs from: the + # tag itself on a tag push, the default branch on a manual run. It + # builds the tag's own source in a separate worktree. + run: | + git init -q . + git remote add origin "$REPO_URL" + git fetch -q origin "+refs/tags/*:refs/tags/*" "$GITHUB_SHA" + git checkout -q --detach "$GITHUB_SHA" + test "$(git rev-parse HEAD)" = "$GITHUB_SHA" + git rev-parse -q --verify "refs/tags/$TAG^{commit}" >/dev/null \ + || { echo "no tag $TAG in this repository"; exit 1; } + - name: check the signed manifest and rebuild the .pyz from the tag + run: | + # The PyYAML release bastion_agent/build.py vendors, as a wheel only + # and only with one of these sha256s (PyPI's, for Linux on x86_64 and + # arm64, Python 3.12 and 3.13): the one package this job takes from + # outside the repository, and this step runs before the upload step + # that holds the token. A different PyYAML would only make the + # rebuild differ and the step refuse. + cat > "$RUNNER_TEMP/requirements.txt" <<'REQ' + pyyaml==6.0.3 \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c + REQ + /usr/bin/python3 -m venv /tmp/venv + /tmp/venv/bin/pip install -q --disable-pip-version-check --only-binary :all: \ + --require-hashes -r "$RUNNER_TEMP/requirements.txt" + /tmp/venv/bin/python -m bastion_agent.verify_release \ + --expect-version "$TAG" --out "$RUNNER_TEMP/release" + ls -l "$RUNNER_TEMP/release" + - name: the release notes, from that version's CHANGELOG section + run: | + /usr/bin/git show "$TAG:CHANGELOG.md" > "$RUNNER_TEMP/CHANGELOG.md" + /usr/bin/python3 -I - "${TAG#v}" "$RUNNER_TEMP/CHANGELOG.md" > "$RUNNER_TEMP/notes.md" <<'PY' + import re, sys + ver, path = sys.argv[1], sys.argv[2] + text = open(path, encoding="utf-8").read() + m = re.search(r"^## %s[ \t]*\n(.*?)(?=^## |\Z)" % re.escape(ver), text, re.M | re.S) + if not m: + sys.exit("CHANGELOG.md at the tag has no section for %s" % ver) + print(m.group(1).strip()) + print() + print("---") + print("The files below are this tag rebuilt by this repository's release workflow, " + "attached only after their sha256 matched the signed manifest " + "(`version.json` and its `.sig`). To check them yourself, run " + "`python3 -m bastion_agent.verify_release` from a clone: " + "see [Verify a release](https://github.com/%s#verify-a-release)." + % "Privatae/bastion-agent") + PY + - name: attach the files to the release + # The one step with the token. Its shell and Python are named by path, + # and Python runs isolated (-I), so nothing an earlier step put on PATH + # or in the environment chooses what runs here. + shell: /usr/bin/bash -euo pipefail {0} + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + run: | + /usr/bin/python3 -I - "$RUNNER_TEMP/release" "$RUNNER_TEMP/notes.md" <<'PY' + import json, os, sys, urllib.error, urllib.parse, urllib.request + + out_dir, notes = sys.argv[1], sys.argv[2] + api = os.environ.get("GITHUB_API_URL", "https://api.github.com").rstrip("/") + repo, tag, token = os.environ["GH_REPO"], os.environ["TAG"], os.environ["GH_TOKEN"] + + def call(method, url, body=None, ctype="application/json"): + data = body if isinstance(body, bytes) or body is None else json.dumps(body).encode() + req = urllib.request.Request(url, data=data, method=method, headers={ + "Authorization": "Bearer " + token, "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", "Content-Type": ctype, + "User-Agent": "bastion-agent-release"}) + try: + with urllib.request.urlopen(req, timeout=120) as r: + raw = r.read() + return r.status, json.loads(raw) if raw else None + except urllib.error.HTTPError as e: + return e.code, None + + st, rel = call("GET", "%s/repos/%s/releases/tags/%s" % (api, repo, tag)) + if st == 404: + body = open(notes, encoding="utf-8").read() + st, rel = call("POST", "%s/repos/%s/releases" % (api, repo), { + "tag_name": tag, "name": "bastion agent " + tag[1:], "body": body, + "draft": False, "prerelease": False}) + if st != 201: + sys.exit("creating the release for %s failed: HTTP %d" % (tag, st)) + print("created the release for %s" % tag) + elif st != 200: + sys.exit("reading the release for %s failed: HTTP %d" % (tag, st)) + else: + print("the release for %s exists; its text is left as it is" % tag) + + st, assets = call("GET", "%s/repos/%s/releases/%d/assets?per_page=100" % (api, repo, rel["id"])) + if st != 200: + sys.exit("listing the release's files failed: HTTP %d" % st) + have = {a["name"]: a["id"] for a in assets} + upload = rel["upload_url"].split("{", 1)[0] + for name in sorted(os.listdir(out_dir)): + if name in have: + st, _ = call("DELETE", "%s/repos/%s/releases/assets/%d" % (api, repo, have[name])) + if st != 204: + sys.exit("replacing %s failed: HTTP %d" % (name, st)) + with open(os.path.join(out_dir, name), "rb") as fh: + data = fh.read() + st, a = call("POST", upload + "?" + urllib.parse.urlencode({"name": name}), data, + "application/octet-stream") + if st != 201: + sys.exit("uploading %s failed: HTTP %d" % (name, st)) + print("attached %s (%d bytes)" % (a["name"], a["size"])) + PY diff --git a/CHANGELOG.md b/CHANGELOG.md index d2ae0da..c349b3c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,89 @@ Changes to the release tooling (`publish.sh`, `build.py`, `verify_release.py`) and to the documentation are listed under the version whose source first carried them, although they do not change what the `.pyz` does. +## 0.1.16 + +- **Every `search` and `read_range` reply names its sanitize mode** + (`sanitize_mode`: `flag` or `sterilize`, the mode the sanitize stage + recorded for the text in the reply; `unknown` when that record does not + vouch for it; a search with no hits names the manifest's mode, or `none`), + so the bastion and the Host can say what was done to the text. Switching + the manifest's mode does not relabel text an earlier sweep wrote. In `flag` + mode, the mode the bastion deploys, names, street addresses and dates of + birth remain. The field is new; nothing else in either reply changes. +- **Each GitHub release carries its files.** Pushing a release tag starts + `.github/workflows/release.yml`, which runs the release check against the + published manifest, rebuilds the `.pyz` from the tag, and only when the + sha256 matches attaches the `.pyz`, its `.sha256`, `version.json` and its + signature to the release (making the release, with this file's section for + it, when the tag has none). It can be run again by hand for the version + the manifest names; older releases, whose manifest is no longer served, + cannot be given their files afterwards. The manifest's `notes` now link to + that GitHub release instead of the tag page on the forge. +- **`verify_release` refuses a tag without its release record** (an annotated + tag with the line `built from `, as every release tag from + 0.1.14 on has), instead of building from any tag of that name. It gains + `--expect-version`, which refuses a manifest naming another version, and + `--out DIR`, which writes the rebuilt `.pyz` and `.sha256`, the manifest's + exact bytes and its signature once everything matched. +- **CI runs every test and pyflakes.** The test job installs `git` too, so + the release verifier's seven end-to-end tests, which were skipped in CI, now + run, and a skipped test fails the job. pyflakes 4.0.3 runs on both forges; + the 31 findings it had (unused imports and variables) are fixed, and the + packages' re-exports are listed in `__all__`. +- **The README says what it means.** The agent is deployed from a grant and + can also be run by hand; what crosses the tunnel is the catalog cards and + sanitized excerpts; building and testing need PyYAML 6.0.3; the CI badge is + the GitHub one. +- **The rules pass finds more personal data on its own,** which matters + because it is all that `flag` mode, the mode the bastion deploys, runs. + Newly found: international phone numbers (`+44 7700 900123`, + `+49 (0)30 1234567`, no separators, dots or dashes), national numbers with a + trunk `0` (`07700 900123`), a seven-digit local number after a word such as + call or phone, IBANs written in lower case, card numbers written with dots, + SSNs written with spaces, and nine-digit SSNs with no separators when `SSN` + or `social security` comes just before them. A bare nine-digit number, a + seven-digit number with no phone word before it (`Order 555-0142`), a number + that fails its checksum and a date are still left alone; a reference number + that starts with `0` (`0123-456789`) is now taken for a national phone + number. An expiry date or security code beside a card is still not found. +- **A dotted number right after a version word is no longer taken for an IP + address** (`version 1.2.3.4`), and SSNs that are never issued (group `00`, + serial `0000`) are no longer masked. +- **Every file is sanitized again once after this upgrade** (`SANITIZE_VERSION` + 3), so text kept by the older rules is redone. +- **The docs say what the deployed mode leaves in.** The README, SECURITY.md + and the sanitize, serve and PII docs say plainly that `flag` mode does not + find names, street addresses or dates of birth, and that `sterilize` mode + adds a model that looks for them. +- **The PII service listens on 127.0.0.1 by default** instead of every + interface; `PII_BIND` still chooses another address. Its container image runs + as uid 10001 instead of root, so a volume mounted on `/data` must be writable + by that user, and a container that should be reachable needs `PII_BIND` set. +- **CI also runs on GitHub** (`.github/workflows/ci.yml`): the same jobs, the + same pinned images. +- **The README's license paragraph and SECURITY.md name each kind of Host** + (docs only) instead of only Privatae's platform: a + C-Series Host runs on your own machine and is open source (Privatae/cseries, + AGPL-3.0-only), C-Series Cloud and a network membership are optional and + paid, and a Host on Privatae's own platform is the case where the Host's + software is not open source. +- **Security reports go to security@privatae.ai,** the address + privatae.ai/.well-known/security.txt names, and SECURITY.md links the + bastion's policy on GitHub, where anyone can read it. +- **How this repository is published.** From this release on, GitHub receives + each change merged since the last release as its own commit, titled as the + change was, instead of one commit per release, and the README says so. +- **The personal-data check reads what it used to miss.** A listed term + written right after a backslash escape (`\n`, `\t`, `\u00e9`) is found, and + so is any value inside a run of base64 or base64url that decodes to text. + The reserved entity ids are read from `/etc/privatae/reserved-entity-ids` + (one per line, `RESERVED_ENTITY_IDS_FILE` to point elsewhere) instead of + being written in the script; where that file is absent, as in public CI, + the entity-id check is skipped and the run prints a NOTE, as the term check + already does without its key. A file that holds no id, or a line that is + not one, fails the run. + ## 0.1.15 - **No change to what the agent does.** The README says how this repository diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 79ed029..2098f3f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -11,6 +11,7 @@ From the repository root: ```sh python3 -m unittest discover . # every test, including the 3.8 floor +python3 -m pyflakes bastion_agent pii scripts test_syntax_floor.py # pyflakes==4.0.3, as CI runs it python3 -m bastion_agent.build --out /tmp/agent # the .pyz and its .sha256 python3 /tmp/agent/bastion-agent-*.pyz --version bash -n bastion_agent/publish.sh pii/verify-live.sh && sh -n bastion_agent/install.sh @@ -19,7 +20,10 @@ bash -n bastion_agent/publish.sh pii/verify-live.sh && sh -n bastion_agent/insta - **All tests pass.** They need PyYAML (the manifest loader uses it), at the version `build.py` pins (`PYYAML_VERSION`) for the build tests. The round-trip seal tests are skipped where `age` and `age-keygen` are not - installed, so install them to run the whole suite. + installed, the TLS tests where `openssl` is not, and the release verifier's + end-to-end tests where `git` is not, so install them to run the whole suite. + CI installs them and fails on any skipped test. +- **pyflakes finds nothing.** No unused import or variable, no undefined name. - **The Python 3.8 floor.** `test_syntax_floor.py` parses every module in `bastion_agent/` and `pii/` at 3.8 and denies the newer standard-library calls that parse on 3.8 and then fail at run time. The agent ships to diff --git a/README.md b/README.md index c34b5ce..de4bb60 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,18 @@ # bastion agent -[![ci](https://git.privatae.ai/privatae/bastion-agent/actions/workflows/ci.yml/badge.svg?branch=main)](https://git.privatae.ai/privatae/bastion-agent/actions) +[![ci](https://github.com/Privatae/bastion-agent/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/Privatae/bastion-agent/actions/workflows/ci.yml) The small program the bastion runs on your own machine, for one folder. -It is deployed **from a grant, never installed by a person**: the bastion fetches +It is deployed **from a grant**, and you do not install it yourself: the bastion fetches `privatae.ai/bastion-agent/version.json`, checks its ed25519 release signature (`version.json..sig`) against the keys compiled into the bastion, downloads the `.pyz` and checks it against the sha256 in that signed manifest, mints a P-256 identity per store, writes and signs a manifest from the grant you -approved, and supervises `sweep` and `serve`. Your files never leave the machine; -what crosses the tunnel is the index and what the grant's mask allows. +approved, and supervises `sweep` and `serve`. Your files stay on the machine; +what crosses the tunnel is what the grant's mask allows: catalog cards, and +the sanitized excerpts that `search` and `read_range` return, each reply naming +the sanitize mode that made it (`sanitize_mode`, [serve.md](docs/serve.md)). The published version is whatever `privatae.ai/bastion-agent/version.json` names. `bastion_agent/VERSION` on `main` is the next release, which may not be @@ -18,7 +20,7 @@ published yet. Release manifests are published signed from 0.1.13 on, and the `.pyz` can be rebuilt from this source and compared ([Verify a release](#verify-a-release)). -This repository is free software, licensed under the GNU General Public License version 3 or (at your option) any later version (see `LICENSE`). Copyright Privatae LLC. You may run it, read it, change it and redistribute it under that license. The Privatae platform the bastion connects to is separate, proprietary software and is not covered by this license. A member runs their own build on their own machine — nothing here depends on a Privatae-held credential or a Privatae-side build. +This repository is free software, licensed under the GNU General Public License version 3 or (at your option) any later version (see `LICENSE`). Copyright Privatae LLC. You may run it, read it, change it and redistribute it under that license. The Host the bastion connects to is separate software and is not covered by this license: a C-Series Host is [Privatae/cseries](https://github.com/Privatae/cseries), under AGPL-3.0-only, and a Host on Privatae's own platform is proprietary software. A member runs their own build on their own machine — nothing here depends on a Privatae-held credential or a Privatae-side build. ## What it does @@ -30,13 +32,13 @@ failure stops the run: |---|---|---| | walk | the map of the folder: files, sizes, types, what the grant excludes | [walk.md](docs/walk.md) | | extract | text out of each file, with built-in extractors or local tools | [extract.md](docs/extract.md) | -| sanitize | finds personal data and replaces it with tokens; the originals stay in a vault on this machine | [sanitize.md](docs/sanitize.md), [pii-sterilization.md](docs/pii-sterilization.md) | +| sanitize | replaces the personal data its mode finds with tokens; the originals stay in a vault on this machine. In the mode the bastion deploys it does not find names, street addresses or dates of birth ([below](#how-the-bastion-runs-it)) | [sanitize.md](docs/sanitize.md), [pii-sterilization.md](docs/pii-sterilization.md) | | chunk, embed | splits the text and embeds it with a model on this machine | [embed.md](docs/embed.md) | | catalog | the curated dataset built from the stages above | [catalog.md](docs/catalog.md) | | seal | packs an allowlisted bundle of the catalog and encrypts it with age to the owner and the manifest's named recipients; the one way a dataset leaves the machine | [seal.md](docs/seal.md) | -`serve` answers queries over the index with mutual TLS, returning sanitized -chunks and catalog cards ([serve.md](docs/serve.md)). The gate refuses an +`serve` answers queries over the index with mutual TLS, returning chunks and +catalog cards from the sanitized text ([serve.md](docs/serve.md)). The gate refuses an unsigned, edited or foreign manifest, and the bastion never starts the agent without a valid one. (`--allow-unsigned` exists for development only; the bastion never passes it.) See [runtime.md](docs/runtime.md) and @@ -45,12 +47,20 @@ bastion never passes it.) See [runtime.md](docs/runtime.md) and ## How the bastion runs it The bastion installs and runs the agent itself from a folder grant you -approved; nobody installs it by hand. For each shared folder the bastion: +approved; you do not install it by hand. (The `.pyz` can also be installed and +run by hand, to try it outside the bastion: [packaging.md](docs/packaging.md).) +For each shared folder the bastion: - downloads the `.pyz` only through the signed manifest, and never replaces an installed agent with an older version; - mints the agent its own identity and writes and signs its manifest from the grant, so the agent works only on what you granted; +- sets the sanitize stage to `flag` mode: rules only, no model. It replaces + e-mail addresses, phone numbers, card, bank and social security numbers, IP + and MAC addresses and keys. **It does not find names, street addresses or + dates of birth**, so text a Host receives from the agent still contains them. + `sterilize` mode adds a model on this machine that looks for those, and the + bastion does not set it ([sanitize.md](docs/sanitize.md#modes-manifest-sanitizemode)); - points the embed stage at the model server on this machine (a local ollama) and at nothing else; - runs `serve` on the loopback address only, with mutual TLS, so the bastion is @@ -81,12 +91,16 @@ package, which is what the built artifact has always looked like inside. ## Build and test ```sh +pip install pyyaml==6.0.3 # the PyYAML build.py vendors; the manifest loader imports it python3 -m unittest discover . # every module's tests, plus the 3.8 floor python3 -m bastion_agent.build --out dist # produces dist/bastion-agent-.pyz + .sha256 python3 dist/bastion-agent-.pyz --version ``` -Every push runs the same checks in CI (`.gitea/workflows/ci.yml`). +A test that needs `age`, `openssl` or `git` is skipped where that tool is not +installed. Every push runs the same checks in CI (`.gitea/workflows/ci.yml` on +git.privatae.ai, `.github/workflows/ci.yml` on GitHub), with the tools +installed, so there no test is skipped, plus `pyflakes`. The build is reproducible: the same commit gives the same bytes on any machine with PyYAML at the pinned version (`PYYAML_VERSION` in `build.py`). @@ -113,8 +127,9 @@ it); no key or account of ours. It: `bastion_agent/verify_release.py`, copied from `client-go/release_sign.go` in the bastion's repository; 2. checks out the source of that version in a separate worktree: the tag - `v`, or else the commit the manifest names as `built_from` if the - clone has it; + `v`, which must be annotated with the line + `built from `, or else the commit the manifest names as + `built_from` if the clone has it; 3. rebuilds the `.pyz` with that version's own `build.py`, and compares its sha256 with every `.pyz` the manifest lists. @@ -122,21 +137,38 @@ A match means the file every bastion downloads is exactly what this source produces, and the signature means the release key holder published it. The exit code is 0 only when both hold. -The public repository is a copy of each release's source, not the development -history, so `built_from` names a commit that exists only in the development -repository. From this release on, each public snapshot carries the tag -`v`, which marks the source the release was built from. A snapshot -published before that has no tag, and can be checked from `built_from` only -where that commit is available. A manifest from before signing has no +From 0.1.16 on, each GitHub release carries the `.pyz`, its `.sha256`, the +signed manifest `version.json` and its signature `version.json..sig`. The +release workflow (`.github/workflows/release.yml`) attaches them when the tag +is pushed, after running this same check against the published manifest and +rebuilding the `.pyz` from the tag; it attaches nothing if a digest differs. +To check downloaded files yourself, `sha256sum -c bastion-agent-.pyz.sha256` +checks the `.pyz` against its `.sha256`, the manifest lists the sha256 of both, +and `check_signature` in `bastion_agent/verify_release.py` checks the manifest +against the pinned release keys: + +```sh +python3 -c 'import sys; from bastion_agent import verify_release as v; print(v.check_signature(open(sys.argv[1], "rb").read(), open(sys.argv[2]).read())[0]["files"])' version.json version.json.*.sig +``` + +The public repository does not carry the development history, so +`built_from` names a commit that exists only in the development repository. +Each published release from 0.1.14 on carries the tag `v`, which +marks the source the release was built from. Earlier releases have no tag +here, and can be checked from `built_from` only where that commit is +available. A manifest from before signing has no signature file beside it, and the check refuses it. ## How this repo is published -Development happens in a private repository. This repository receives the -source of each published release as one snapshot commit, tagged -`v`, whose tree is the source the signed manifest says the release -was built from; that is why its history is short. The same snapshot is on -GitHub at [github.com/Privatae/bastion-agent](https://github.com/Privatae/bastion-agent). +Development happens in a private repository, and its history is not +published. When a release is published, each change merged since the last one +arrives on GitHub, at +[github.com/Privatae/bastion-agent](https://github.com/Privatae/bastion-agent), +as a commit with the same files, titled as the change was and dated when it +was merged; the copy at git.privatae.ai receives the release as one commit. +In both, the release is tagged `v` on a commit whose tree is the +source the signed manifest says the release was built from. Every published `.pyz` can be rebuilt from its tag and compared ([Verify a release](#verify-a-release)). diff --git a/SECURITY.md b/SECURITY.md index 38bd642..180b265 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -4,7 +4,7 @@ Please do not open a public issue for a security problem. -Email **info@privatae.ai** with: +Email **security@privatae.ai** with: - the agent's version (`python3 bastion-agent-.pyz --version`) and platform, @@ -50,13 +50,22 @@ example: - **The bastion**, which installs, updates and supervises the agent, signs its manifest and holds its keys. It is a separate program with its own policy: - [privatae-bastion's SECURITY.md](https://git.privatae.ai/privatae/privatae-bastion/src/branch/main/SECURITY.md). + [the bastion's SECURITY.md](https://github.com/Privatae/bastion/blob/main/SECURITY.md). Report there, or to the same address. -- **The Privatae platform** the bastion connects to. It is separate software - with its own reporting path; reports about it sent to the address above are - forwarded. -- **What the design already says it does not do.** The rules-only `flag` mode - does not find names, and the recall pass is probabilistic +- **The Host** the bastion connects to, which is separate software. A + C-Series Host runs on your own machine, in its own Linux virtual machine; + its source is [Privatae/cseries](https://github.com/Privatae/cseries), under + AGPL-3.0-only, `npx cseries` installs the bastion beside it and pairs, + anchors and enrols it, and no Privatae account is needed. C-Series Cloud (a + Host on a server of its own that Privatae sets up and keeps patched) and a + Privatae network membership are optional and paid. A Host on Privatae's own + platform is the case where the Host's software is not open source; the + bastion keeps it to your grant and your approvals whatever the platform + does. Reports about a Host sent to the address above are forwarded. +- **What the design already says it does not do.** The rules-only `flag` mode, + which is the mode the bastion deploys, does not find names, street addresses + or dates of birth ([README](README.md#how-the-bastion-runs-it)), and the + recall pass of `sterilize` mode is probabilistic ([docs/pii-sterilization.md](docs/pii-sterilization.md#limits)); the extract store in the output folder is plaintext by design ([docs/runtime.md](docs/runtime.md#where-an-agent-lives-rootagent)); the diff --git a/bastion_agent/VERSION b/bastion_agent/VERSION index c34958a..e8e277f 100644 --- a/bastion_agent/VERSION +++ b/bastion_agent/VERSION @@ -1 +1 @@ -0.1.15 +0.1.16 diff --git a/bastion_agent/__init__.py b/bastion_agent/__init__.py index dee463b..40189fd 100644 --- a/bastion_agent/__init__.py +++ b/bastion_agent/__init__.py @@ -10,3 +10,9 @@ from .manifest import ( # noqa: F401 signing_payload, validate_document, ) + +# The package's public names: the manifest's schema and its loader. +__all__ = [ + "STAGES", "MASK_OPS", "SWEEPS", "GRANT_OPS", "Manifest", "ManifestError", + "load_manifest", "signing_digest", "signing_payload", "validate_document", +] diff --git a/bastion_agent/catalog.py b/bastion_agent/catalog.py index 6398dfe..d193052 100644 --- a/bastion_agent/catalog.py +++ b/bastion_agent/catalog.py @@ -340,7 +340,6 @@ def catalog(manifest, out_dir, env=None, summarizer=None, now=None): continue key = (w["root"], w["path"]) counts["files"] += 1 - name = w["path"].rsplit("/", 1)[-1] shown_path, changed = mask_path(w["path"], detect) masked = shown_path.rsplit("/", 1)[-1] counts["names_masked"] += changed diff --git a/bastion_agent/doctor.py b/bastion_agent/doctor.py index f1d4eec..3a4a647 100644 --- a/bastion_agent/doctor.py +++ b/bastion_agent/doctor.py @@ -7,11 +7,13 @@ tesseract, the gliner package) — their absence is a recorded gap in the stage output, not a failure here. """ +import importlib import json import os import platform import shutil import sys +import urllib.error import urllib.request PY_FLOOR = (3, 8) @@ -25,7 +27,6 @@ def _probe_http(url, path, timeout=5): """Reachable means the server answered — any HTTP status counts (a proxy that only permits the embed path answers 403 on '/'). Only a connection failure is 'not reachable'.""" - import urllib.error try: with urllib.request.urlopen(url.rstrip("/") + path, timeout=timeout) as r: return True, "HTTP %d" % r.status @@ -65,7 +66,7 @@ def checks(env=None): add(tool, bool(p), p or ("not on PATH — " + why), required=req, stage=stage) try: - import gliner # noqa: F401 + importlib.import_module("gliner") add("gliner package", True, "importable (PII_RECALL=gliner available)", required=False, stage="sanitize") except ImportError: add("gliner package", False, "not installed — sanitize sterilize on a laptop needs " diff --git a/bastion_agent/extract.py b/bastion_agent/extract.py index 29b9e6e..337b446 100644 --- a/bastion_agent/extract.py +++ b/bastion_agent/extract.py @@ -41,7 +41,6 @@ import os import re import shutil import subprocess -import sys import tempfile import time import unicodedata diff --git a/bastion_agent/publish.sh b/bastion_agent/publish.sh index 6583935..aeeb86b 100755 --- a/bastion_agent/publish.sh +++ b/bastion_agent/publish.sh @@ -192,7 +192,7 @@ installers = {n: {"url": "%s/%s" % (base_new, n), "sha256": h(os.path.join(pkg, for n in ("install.sh", "install.ps1")} json.dump({"version": ver, "built_from": sha, "python_floor": "3.8", "files": files, "installers": installers, - "notes": "https://git.privatae.ai/privatae/bastion-agent/releases/tag/v%s" % ver}, + "notes": "https://github.com/Privatae/bastion-agent/releases/tag/v%s" % ver}, sys.stdout, indent=2) PY diff --git a/bastion_agent/sanitize.py b/bastion_agent/sanitize.py index 76bcf72..b66316f 100644 --- a/bastion_agent/sanitize.py +++ b/bastion_agent/sanitize.py @@ -52,7 +52,7 @@ from .manifest import signing_digest # Part of the reuse key: bump it whenever the pipeline would sterilize the # same text differently, or a stored result from older code is kept as is. -SANITIZE_VERSION = 2 +SANITIZE_VERSION = 3 SANITIZE_CHUNK_CHARS = 20000 # per sterilize call; under the service's 2 MiB body cap SERVICE_TIMEOUT = 600.0 diff --git a/bastion_agent/serve.py b/bastion_agent/serve.py index f1feebd..4d9d0de 100644 --- a/bastion_agent/serve.py +++ b/bastion_agent/serve.py @@ -17,6 +17,12 @@ Ops (JSON over HTTPS): POST /read_range {doc, start, end} read_range (owner, grantee w/ op) from the SANITIZED store only, <= READ_RANGE_MAX bytes + Both answer with `sanitize_mode`, the mode the sanitize stage recorded for + the text in the reply ("flag" or "sterilize"; the weakest across a + search's hits; "unknown" when the record does not vouch for that text; a + search with no hits names the manifest's mode, or "none"), so a caller can + say what the text has had done to it: in flag mode names, street addresses + and dates of birth remain. GET /catalog[?prefix=&offset=&limit=] owner only: the file cards (paged) GET /catalog/dirs[?...] owner only: the directory cards (paged) GET /status owner only: run.state.json @@ -65,6 +71,83 @@ class ServeError(Exception): pass +def sanitize_mode(manifest): + """The sanitize mode the manifest sets: "flag", "sterilize", or "none" when + it names no sanitize stage. A reply with no text in it (a search with no + hits) carries this; a reply with text carries served_mode.""" + s = getattr(manifest, "sanitize", None) + return getattr(s, "mode", None) or "none" + + +# Weakest first: a reply that mixes texts names the weakest mode among them. +_MODE_RANK = {"unknown": 0, "flag": 1, "sterilize": 2} +_index_lock = threading.Lock() +_index_cache = {} # sanitize.jsonl path -> ((ino, mtime_ns, size), {doc: mode}) + + +def _sanitize_index(out_dir): + """({doc sha: mode} for every document the last sanitize run wrote with + status ok, the index's mtime_ns), from /sanitize.jsonl; ({}, None) + when there is none. Re-read only when the file changes.""" + path = os.path.join(out_dir, "sanitize.jsonl") + try: + st = os.stat(path) + except OSError: + return {}, None + sig = (st.st_ino, st.st_mtime_ns, st.st_size) + with _index_lock: + hit = _index_cache.get(path) + if hit and hit[0] == sig: + return hit[1], st.st_mtime_ns + modes = {} + try: + with open(path, encoding="utf-8") as fh: + for line in fh: + try: + rec = json.loads(line) + except ValueError: + continue + if isinstance(rec, dict) and rec.get("status") == "ok" and rec.get("sha256"): + modes[rec["sha256"]] = rec.get("mode") + except OSError: + return {}, None + _index_cache[path] = (sig, modes) + return modes, st.st_mtime_ns + + +def served_mode(out_dir, doc, start=None, text=None): + """The sanitize mode that wrote the text served for doc, as the sanitize + stage recorded it, or "unknown" when that record does not vouch for the + text. The manifest's mode says what the NEXT sweep will do; the store can + still hold an earlier sweep's text (the manifest switched to sterilize and + the sweep has not run, or stopped; a sweep is rewriting the store now; the + chunks a search reads were cut before the latest sanitize run). + + Unknown when: no status-ok record for doc, the file is newer than the + index (a run that has not finished rewrote it), or, for a search hit, + `text` at `start` is not what the store holds now.""" + modes, index_mtime = _sanitize_index(out_dir) + mode = modes.get(doc) + if mode not in ("flag", "sterilize"): + return "unknown" + path = sanitized_store_path(out_dir, doc) + try: + if os.stat(path).st_mtime_ns > index_mtime: + return "unknown" + if text is not None: + with open(path, encoding="utf-8") as fh: + stored = fh.read() + if stored[start:start + len(text)] != text: + return "unknown" + except (OSError, TypeError, ValueError): + return "unknown" + return mode + + +def _weakest(modes): + return min(modes, key=lambda m: _MODE_RANK.get(m, 0)) + + def parse_bind(bind): bind = bind.strip() if bind.startswith("["): @@ -312,7 +395,13 @@ class Handler(BaseHTTPRequestHandler): except (_embed.EmbedError, OSError) as e: self._send(502, {"error": "search unavailable: %s" % e}, cn, "search", {"q": qhash}) return - self._send(200, {"hits": hits, "k": k}, cn, "search", {"q": qhash, "hits": len(hits)}) + if hits: + mode = _weakest([served_mode(self.out_dir, h.get("doc"), h.get("start"), h.get("text")) + for h in hits]) + else: + mode = sanitize_mode(self.policy.manifest) + self._send(200, {"hits": hits, "k": k, "sanitize_mode": mode}, + cn, "search", {"q": qhash, "hits": len(hits)}) return if self.path == "/read_range": cn, denied = self._authorize("read_range") @@ -340,7 +429,8 @@ class Handler(BaseHTTPRequestHandler): text = fh.read() piece = text[start:end] self._send(200, {"doc": doc, "start": start, "end": start + len(piece), - "total": len(text), "text": piece}, + "total": len(text), "text": piece, + "sanitize_mode": served_mode(self.out_dir, doc)}, cn, "read_range", {"doc": doc[:12], "chars": len(piece)}) return self._send(404, {"error": "not found"}, self._cn(), None) diff --git a/bastion_agent/test_run.py b/bastion_agent/test_run.py index 1333fa2..a6431b4 100644 --- a/bastion_agent/test_run.py +++ b/bastion_agent/test_run.py @@ -12,7 +12,6 @@ import shutil import subprocess import tempfile import unittest -from unittest import mock import yaml diff --git a/bastion_agent/test_sanitize.py b/bastion_agent/test_sanitize.py index 90bcb26..f086bed 100644 --- a/bastion_agent/test_sanitize.py +++ b/bastion_agent/test_sanitize.py @@ -153,6 +153,19 @@ class FlagModeTests(Case): self.assertEqual(s2["counts"]["ok"], s1["counts"]["ok"]) self.assertFalse(idx["note.txt"].get("reused", False)) + def test_text_kept_by_the_older_rules_is_redone(self): + # version 2 kept an international number and a lowercase IBAN in the + # clear; a store it wrote must not be reused by the wider rules + put(self.root, "intl.txt", "Mobile +44 7700 900123, iban gb82west12345698765432\n") + with mock.patch.object(sz, "SANITIZE_VERSION", 2): + self.run_stage(manifest(self.root, "flag")) + s, idx = self.run_stage(manifest(self.root, "flag")) + self.assertEqual(s["counts"]["reused"], 0) + out = self.text_of(idx["intl.txt"]) + self.assertNotIn("7700 900123", out) + self.assertNotIn("gb82west", out) + self.assertEqual(idx["intl.txt"]["types"], {"PHONE": 1, "BANK_ACCOUNT": 1}) + def test_tokens_stable_across_files_in_scope(self): put(self.root, "again.txt", "ping m@example.com again\n") _, idx = self.run_stage(manifest(self.root, "flag")) diff --git a/bastion_agent/test_seal.py b/bastion_agent/test_seal.py index b9ea00e..c329ee6 100644 --- a/bastion_agent/test_seal.py +++ b/bastion_agent/test_seal.py @@ -175,7 +175,7 @@ class RoundTripTests(Case): def test_tampered_member_refused(self): ident, pub = keygen(self.out) env = {"FREEAGENT_OWNER_RECIPIENT": pub} - s = sl.seal(manifest(self.root), self.out, env=env) + sl.seal(manifest(self.root), self.out, env=env) # rebuild a bundle with a modified catalog but the old inventory, seal it, and unseal plain, _ = sl.build_bundle(manifest(self.root), self.out, "x") buf = io.BytesIO() diff --git a/bastion_agent/test_serve.py b/bastion_agent/test_serve.py index c18d7be..e181dd5 100644 --- a/bastion_agent/test_serve.py +++ b/bastion_agent/test_serve.py @@ -22,7 +22,6 @@ from . import run as rn from . import serve as sv from .catalog import catalog from .extract import extract -from .manifest import validate_document from .sanitize import sanitize from .test_embed import _Stub as EmbedStub # deterministic bag-of-words embedder from .walk import walk @@ -112,12 +111,12 @@ class ServeTests(unittest.TestCase): shutil.rmtree(cls.work, ignore_errors=True) @classmethod - def write_manifest(cls, grants): + def write_manifest(cls, grants, mode="flag"): import yaml doc = {"freeagent": 1, "id": "t", "owner": {"host": "owner-host"}, "scope": {"roots": [cls.root]}, "mask": ["read", "search", "index"], "stages": ["walk", "extract", "sanitize", "chunk", "embed", "catalog", "seal"], - "sanitize": {"mode": "flag"}, + "sanitize": {"mode": mode}, "embed": {"model": "stub-model", "dim": 64, "chunk": {"size_tokens": 8, "overlap_tokens": 2}}, "sweep": "manual", "sink": {"catalog": "host", "vectors": "local", "plaintext": "never"}, @@ -238,6 +237,93 @@ class ServeTests(unittest.TestCase): self.assertEqual(self.call("owner", "POST", "/search", {"query": ""})[0], 400) self.assertEqual(self.call("owner", "POST", "/nope", {})[0], 404) + GRANTS = [{"host": "partner-host", "ops": ["search"], "until": "2026-12-31"}, + {"host": "old-host", "ops": ["search", "read_range"], "until": "2026-01-01"}] + + def modes(self): + """(owner search, owner read_range, partner search) sanitize_mode for the apples doc.""" + st, body = self.call("owner", "POST", "/search", {"query": "apples", "k": 1}) + self.assertEqual(st, 200) + doc = body["hits"][0]["doc"] + st, rr = self.call("owner", "POST", "/read_range", {"doc": doc, "start": 0, "end": 10}) + self.assertEqual(st, 200) + st, pb = self.call("partner", "POST", "/search", {"query": "apples", "k": 1}) + self.assertEqual(st, 200) + return doc, (body["sanitize_mode"], rr["sanitize_mode"], pb["sanitize_mode"]) + + def rewrite_index(self, fn): + """Rewrite sanitize.jsonl through fn(record); returns the original bytes.""" + path = os.path.join(self.out, "sanitize.jsonl") + with open(path, "rb") as fh: + orig = fh.read() + recs = [fn(json.loads(l)) for l in orig.decode("utf-8").splitlines() if l.strip()] + with open(path, "w", encoding="utf-8") as fh: + fh.write("".join(json.dumps(r, sort_keys=True) + "\n" for r in recs)) + return orig + + def restore_index(self, orig): + with open(os.path.join(self.out, "sanitize.jsonl"), "wb") as fh: + fh.write(orig) + + def test_a_reply_names_the_mode_that_wrote_its_text_not_the_manifests(self): + # The store was swept in flag mode. Switching the manifest to + # sterilize changes what the NEXT sweep does, not the text already + # served: until a sweep rewrites it, every reply still says flag, so + # the bastion never calls flag-mode text sanitized. + try: + for mode in ("flag", "sterilize", "flag"): + self.write_manifest(grants=self.GRANTS, mode=mode) + self.assertEqual(self.modes()[1], ("flag", "flag", "flag"), mode) + finally: + self.write_manifest(grants=self.GRANTS) + + def test_a_store_the_sanitize_stage_wrote_in_sterilize_mode_is_named_sterilize(self): + orig = self.rewrite_index(lambda r: dict(r, mode="sterilize")) + try: + self.write_manifest(grants=self.GRANTS, mode="sterilize") + self.assertEqual(self.modes()[1], ("sterilize", "sterilize", "sterilize")) + finally: + self.restore_index(orig) + self.write_manifest(grants=self.GRANTS) + + def test_text_the_sanitize_record_does_not_vouch_for_is_unknown(self): + doc, modes = self.modes() + self.assertEqual(modes, ("flag", "flag", "flag")) + spath = sv.sanitized_store_path(self.out, doc) + st = os.stat(spath) + index = os.path.join(self.out, "sanitize.jsonl") + # a sweep still running has rewritten the file after the index + later = os.stat(index).st_mtime_ns + 10 ** 9 + os.utime(spath, ns=(later, later)) + try: + self.assertEqual(self.modes()[1], ("unknown", "unknown", "unknown")) + finally: + os.utime(spath, ns=(st.st_atime_ns, st.st_mtime_ns)) + # the chunks a search reads were cut from text the store no longer holds + with open(spath, encoding="utf-8") as fh: + text = fh.read() + with open(spath, "w", encoding="utf-8") as fh: + fh.write("X" * len(text)) + os.utime(spath, ns=(st.st_atime_ns, st.st_mtime_ns)) + try: + got = self.modes()[1] + self.assertEqual((got[0], got[2]), ("unknown", "unknown")) + finally: + with open(spath, "w", encoding="utf-8") as fh: + fh.write(text) + os.utime(spath, ns=(st.st_atime_ns, st.st_mtime_ns)) + # the last run did not write it (stopped: not_run), so the file is an older run's + orig = self.rewrite_index(lambda r: dict(r, status="not_run") if r.get("sha256") == doc else r) + try: + self.assertEqual(self.modes()[1], ("unknown", "unknown", "unknown")) + finally: + self.restore_index(orig) + self.assertEqual(self.modes()[1], ("flag", "flag", "flag")) + + def test_sanitize_mode_of_a_manifest_without_the_stage_is_none(self): + self.assertEqual(sv.sanitize_mode(None), "none") + self.assertEqual(sv.sanitize_mode(type("M", (), {"sanitize": None})()), "none") + def test_silent_tcp_connection_does_not_stall_other_clients(self): # with the handshake on the accept loop one idle raw TCP # connection blocked every client; now the handshake runs in the @@ -333,7 +419,7 @@ class UnitTests(unittest.TestCase): def test_parse_bind(self): self.assertEqual(sv.parse_bind("[::]:8470"), ("::", 8470)) self.assertEqual(sv.parse_bind("10.0.0.5:9"), ("10.0.0.5", 9)) - self.assertEqual(sv.parse_bind("[fd00:1::31]:8470"), ("fd00:1::31", 8470)) + self.assertEqual(sv.parse_bind("[2001:db8::31]:8470"), ("2001:db8::31", 8470)) with self.assertRaises(sv.ServeError): sv.parse_bind("nope") diff --git a/bastion_agent/test_verify_release.py b/bastion_agent/test_verify_release.py index d7c987c..341914b 100644 --- a/bastion_agent/test_verify_release.py +++ b/bastion_agent/test_verify_release.py @@ -169,16 +169,26 @@ class EndToEndTests(unittest.TestCase): "files": {name: {"sha256": digest or self.digest}, name + ".sha256": {"sha256": "0" * 64}}}).encode() - def verify(self, body, seed=None): + def verify(self, body, seed=None, args=()): msg = vr.statement("bastion-agent", self.version, hashlib.sha256(body).hexdigest()) - sig = base64.b64encode(_sign(seed or self.SEED, msg)) + sig = base64.b64encode(_sign(seed or self.SEED, msg)) + b"\n" served = {"/version.json": body, "/" + vr.sig_name(body): sig} + self.served = served out, err = io.StringIO(), io.StringIO() with mock.patch.object(vr, "fetch", lambda url: served[url[len("https://test"):]]), \ contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): - code = vr.main(["--base", "https://test"], repo=self.repo, keys=(self.pub,)) + code = vr.main(["--base", "https://test"] + list(args), repo=self.repo, keys=(self.pub,)) return code, out.getvalue() + err.getvalue() + def release_tag(self, built_from, annotated=True): + tag = "v" + self.version + if annotated: + self.run_git("tag", "-a", tag, "-m", "bastion agent %s\n\nbuilt from %s" % (self.version, built_from)) + else: + self.run_git("tag", tag) + self.addCleanup(subprocess.run, ("git", "-C", self.repo, "tag", "-d", tag), + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + def test_tag_first_then_built_from(self): code, out = self.verify(self.manifest("0000000")) self.assertEqual(code, 1, out) @@ -189,13 +199,57 @@ class EndToEndTests(unittest.TestCase): self.assertIn("source: built_from", out) self.assertIn("matches %s" % self.digest, out) - self.run_git("tag", "v" + self.version) - try: - code, out = self.verify(self.manifest("0000000")) - self.assertEqual(code, 0, out) - self.assertIn("source: tag v%s" % self.version, out) - finally: - self.run_git("tag", "-d", "v" + self.version) + self.release_tag("0000000") + code, out = self.verify(self.manifest("0000000")) + self.assertEqual(code, 0, out) + self.assertIn("source: tag v%s" % self.version, out) + + def test_a_tag_without_the_release_record_is_refused(self): + # A lightweight tag, or one recording another build, says nothing + # about which release it is: refused, not passed over for built_from. + self.release_tag("0000000", annotated=False) + code, out = self.verify(self.manifest(self.commit[:7])) + self.assertEqual(code, 1, out) + self.assertIn("does not carry the release record", out) + self.assertNotIn("rebuild:", out) + self.run_git("tag", "-d", "v" + self.version) + self.release_tag("1111111") + code, out = self.verify(self.manifest(self.commit[:7])) + self.assertEqual(code, 1, out) + self.assertIn("built from %s" % self.commit[:7], out) + + def test_another_version_than_expected_is_refused_before_any_checkout(self): + code, out = self.verify(self.manifest(self.commit[:7]), args=["--expect-version", "v9.9.9"]) + self.assertEqual(code, 1, out) + self.assertIn("names v%s, not v9.9.9" % self.version, out) + self.assertNotIn("source:", out) + code, out = self.verify(self.manifest(self.commit[:7]), args=["--expect-version", "v" + self.version]) + self.assertEqual(code, 0, out) + + def test_out_writes_what_a_release_carries(self): + name = "bastion-agent-%s.pyz" % self.version + sums = ("%s %s\n" % (self.digest, name)).encode() + out_dir = os.path.join(self.tmp, "release-out") + self.addCleanup(shutil.rmtree, out_dir, True) + body = json.dumps({"version": self.version, "built_from": self.commit[:7], + "files": {name: {"sha256": self.digest}, + name + ".sha256": {"sha256": hashlib.sha256(sums).hexdigest()}}}).encode() + code, out = self.verify(body, args=["--out", out_dir]) + self.assertEqual(code, 0, out) + sig = vr.sig_name(body) + self.assertEqual(sorted(os.listdir(out_dir)), sorted([name, name + ".sha256", "version.json", sig])) + with open(os.path.join(out_dir, "version.json"), "rb") as fh: + self.assertEqual(fh.read(), body) + with open(os.path.join(out_dir, sig), "rb") as fh: + self.assertEqual(fh.read(), self.served["/" + sig]) + with open(os.path.join(out_dir, name), "rb") as fh: + self.assertEqual(hashlib.sha256(fh.read()).hexdigest(), self.digest) + + def test_out_writes_nothing_when_the_rebuild_differs(self): + out_dir = os.path.join(self.tmp, "release-out-bad") + code, out = self.verify(self.manifest(self.commit[:7], digest="1" * 64), args=["--out", out_dir]) + self.assertEqual(code, 1, out) + self.assertFalse(os.path.exists(out_dir)) def test_a_different_build_is_reported(self): code, out = self.verify(self.manifest(self.commit, digest="1" * 64)) diff --git a/bastion_agent/test_walk.py b/bastion_agent/test_walk.py index c9dc40a..d6874d0 100644 --- a/bastion_agent/test_walk.py +++ b/bastion_agent/test_walk.py @@ -10,13 +10,12 @@ import hashlib import json import os import shutil -import sys import tempfile import unittest from unittest import mock from .manifest import validate_document -from .walk import Excluder, Walker, _glob_to_regex, load_hash_cache, walk +from .walk import Excluder, _glob_to_regex, load_hash_cache, walk IS_WINDOWS = os.name == "nt" diff --git a/bastion_agent/verify_release.py b/bastion_agent/verify_release.py index 31c02b6..8484d98 100644 --- a/bastion_agent/verify_release.py +++ b/bastion_agent/verify_release.py @@ -15,9 +15,15 @@ and the signature says the release key holder published it. The source is the tag v. The public repository is a snapshot of the development one, so the commit the manifest names (built_from) exists only -there; the tag marks the same tree in the public history. When the tag is -absent and built_from is present (a clone of the development repository), -built_from is used instead. +there; the tag marks the same tree in the public history, and it counts only +as an annotated tag whose message has the line `built from `. When +the tag is absent and built_from is present (a clone of the development +repository), built_from is used instead. + +`--expect-version` refuses a manifest that names another version, and `--out +DIR` writes what a release carries once everything matched: the rebuilt .pyz +and its .sha256, the manifest's exact bytes and its signature file. The +GitHub release workflow (.github/workflows/release.yml) runs it that way. The signature check is ed25519 in pure Python, because the standard library has no ed25519 and this has to run where nothing else is installed. It only @@ -228,15 +234,33 @@ def _commit(repo, ref): return r.stdout.strip() if r.returncode == 0 else "" +def tag_records(repo, tag, built_from): + """True when tag is an annotated tag whose message has the line + `built from `, the record publishing writes on every release + tag. A tag without it says nothing about which release it is.""" + if not built_from: + return False + if _git(repo, "cat-file", "-t", "refs/tags/" + tag, check=False).stdout.strip() != "tag": + return False + msg = _git(repo, "tag", "-l", "--format=%(contents)", tag, check=False).stdout + return ("built from " + built_from) in msg.splitlines() + + def pick_source(repo, version, built_from): """(label, commit) of the source to rebuild: the tag v, else the - manifest's built_from, else VerifyError.""" + manifest's built_from, else VerifyError. The tag counts only when it + records the manifest's built_from (tag_records); one that does not is + refused rather than passed over.""" tag = "v" + version # Fetch the tag if the clone lacks it; a failure (offline, no such tag) # just leaves the local refs to decide. _git(repo, "fetch", "-q", "origin", "refs/tags/%s:refs/tags/%s" % (tag, tag), check=False) c = _commit(repo, "refs/tags/" + tag) if c: + if not tag_records(repo, tag, built_from): + raise VerifyError("the tag %s does not carry the release record (an annotated tag " + "with the line 'built from %s'); nothing was built" + % (tag, built_from or "(none named)")) return "tag " + tag, c if built_from: c = _commit(repo, built_from) @@ -249,7 +273,8 @@ def pick_source(repo, version, built_from): def rebuild(repo, commit, version, work): - """sha256 of the .pyz that commit's own build.py produces.""" + """sha256 of the .pyz that commit's own build.py produces. The build's + output stays in /dist.""" src = os.path.join(work, "src") _git(repo, "worktree", "add", "-q", "--detach", src, commit) try: @@ -269,20 +294,52 @@ def rebuild(repo, commit, version, work): _git(repo, "worktree", "remove", "--force", src, check=False) +def write_release(out, dist, version, files, body, sig): + """The files a release carries, into out: the rebuilt .pyz and its .sha256 + (each checked against the manifest's digest for that name), the manifest + as it was served, byte for byte, and its signature under the name it is + served by. Called only after the signature and the rebuild both passed.""" + os.makedirs(out, exist_ok=True) + for name in ("bastion-agent-%s.pyz" % version, "bastion-agent-%s.pyz.sha256" % version): + with open(os.path.join(dist, name), "rb") as fh: + data = fh.read() + want = (files.get(name) or {}).get("sha256") + if hashlib.sha256(data).hexdigest() != want: + raise VerifyError("the rebuilt %s does not match the manifest's sha256 for it" % name) + with open(os.path.join(out, name), "wb") as fh: + fh.write(data) + with open(os.path.join(out, "version.json"), "wb") as fh: + fh.write(body) + with open(os.path.join(out, sig_name(body)), "wb") as fh: + fh.write(sig) + print("release: wrote %s" % ", ".join(sorted(os.listdir(out)))) + + def main(argv=None, repo=REPO, keys=None): ap = argparse.ArgumentParser(prog="bastion_agent.verify_release", description="Rebuild the published agent and compare it with the release manifest.") ap.add_argument("--base", default=os.environ.get("MANIFEST_BASE", DEFAULT_BASE), help="where version.json is served (default %s)" % DEFAULT_BASE) + ap.add_argument("--expect-version", metavar="VERSION", + help="refuse unless the published manifest names this version (v0.1.16 or 0.1.16)") + ap.add_argument("--out", metavar="DIR", + help="when everything matches, write the rebuilt .pyz and its .sha256, the " + "manifest's exact bytes and its signature file into DIR") args = ap.parse_args(argv) base = args.base.rstrip("/") work = tempfile.mkdtemp(prefix="agent-verify-") try: body = fetch(base + "/version.json") - sig = fetch(base + "/" + sig_name(body)).decode("ascii", "replace") + sig_raw = fetch(base + "/" + sig_name(body)) + sig = sig_raw.decode("ascii", "replace") m, key = check_signature(body, sig, keys) ver, built_from = m["version"], m.get("built_from") or "" print("published: v%s built from %s" % (ver, built_from or "(not named)")) + want = (args.expect_version or "").strip() + if want.startswith("v"): + want = want[1:] + if want and want != ver: + raise VerifyError("the published manifest names v%s, not v%s" % (ver, want)) print("signature: ok (release key %d of %d, %s)" % (key + 1, len(RELEASE_KEYS if keys is None else keys), "primary" if key == 0 else "spare")) files = m.get("files") if isinstance(m.get("files"), dict) else {} @@ -295,12 +352,16 @@ def main(argv=None, repo=REPO, keys=None): print("source: %s (%s)" % (label, commit[:12])) got = rebuild(repo, commit, ver, work) bad = 0 - for name, want in pyz.items(): - same = got == want + for name, digest in pyz.items(): + same = got == digest bad += not same print("rebuild: %-28s %s" % (name, "matches %s" % got if same - else "DIFFERS (rebuilt %s, published %s)" % (got, want))) - return 1 if bad else 0 + else "DIFFERS (rebuilt %s, published %s)" % (got, digest))) + if bad: + return 1 + if args.out: + write_release(args.out, os.path.join(work, "dist"), ver, files, body, sig_raw) + return 0 except VerifyError as exc: print("verify: %s" % exc, file=sys.stderr) return 1 diff --git a/bastion_agent/walk.py b/bastion_agent/walk.py index 859f93f..5477941 100644 --- a/bastion_agent/walk.py +++ b/bastion_agent/walk.py @@ -25,7 +25,6 @@ import json import os import platform import re -import sys import tempfile from .manifest import AGENT_DIR, signing_digest diff --git a/docs/packaging.md b/docs/packaging.md index b841692..00c33b5 100644 --- a/docs/packaging.md +++ b/docs/packaging.md @@ -122,9 +122,12 @@ checks each edge's bytes against the manifest it serves. Where it publishes is not written into the script: the edges, the ssh key, the web root and the backup directory come from its environment, and it stops if one is missing. -The manifest's `notes` link to the tag `v` on the public repository, -which marks the source of that release. `publish.sh` does not make the tag; it -is made when that release's source is published to the public repository. +The manifest's `notes` link to the GitHub release of the tag `v`, +which marks the source of that release and carries its files. `publish.sh` +does not make the tag; it is made when that release's source is published to +the public repository, and pushing it starts `.github/workflows/release.yml`, +which rebuilds the `.pyz` from the tag and attaches it, with its `.sha256`, the +signed manifest and its signature, once its sha256 matches the manifest. ### The legacy name diff --git a/docs/pii-sterilization.md b/docs/pii-sterilization.md index 2bffc61..a9f1af8 100644 --- a/docs/pii-sterilization.md +++ b/docs/pii-sterilization.md @@ -15,7 +15,9 @@ personal data is replaced with stable placeholder tokens, the real values stay in a local vault, and `/rehydrate` puts them back, locally, on demand. A cloud model can then work over the text without the values in it. Patterns alone cannot do this: "my daughter goes to Lincoln Elementary" contains no pattern, -only meaning. +only meaning. The agent the bastion deploys runs the rules pass only (`flag` +mode), so names, street addresses and dates of birth stay in the text it +returns ([sanitize.md](sanitize.md#modes-manifest-sanitizemode)). ## Pipeline (`pii/pipeline.py`) @@ -116,7 +118,7 @@ Configuration, from the environment: | Variable | Default | Meaning | |---|---|---| | `PII_API_KEY` | (none) | Required. Without it the service refuses to start: `/rehydrate` returns real values, so there is no unauthenticated mode. (`PII_ALLOW_NOAUTH=1` exists for the unit tests only.) | -| `PII_BIND` | `0.0.0.0:8091` | The address the service listens on; this is its exposure control. | +| `PII_BIND` | `127.0.0.1:8091` | The address the service listens on; this is its exposure control. Another address only when set. | | `PII_DB` | `/data/vault.db` | The vault. | | `PII_RECALL` | `llm` | The recall backend (below). | | `LLM_URL` | `http://llm:8080` | The model server for `llm`. | @@ -125,7 +127,9 @@ Configuration, from the environment: Two ways to run it, the same code: - **Container**: `pii/Dockerfile` (a pinned `python:*-slim` image, no pip - install; the vault on a mounted `/data`). + install; the vault on a mounted `/data`, writable by uid 10001, the + unprivileged user it runs as). Inside a container the default loopback bind + reaches nothing else, so set `PII_BIND` and publish the port deliberately. - **systemd**: `pii/pii.service`, with install steps in its header. It runs with `DynamicUser`, a private 0700 state directory for the vault (`/var/lib/privatae-pii/vault.db`), loopback bind (`127.0.0.1:8091`), and @@ -167,9 +171,18 @@ the machine. Every detector is anchored by a checksum, a prefix or a fixed shape; none depends on a model. -- **E-mail, phone, SSN, IPv4/IPv6, MAC.** -- **Cards** → `CREDIT_CARD`, checked with Luhn. -- **IBAN** → `BANK_ACCOUNT`: ISO 13616 mod-97 plus a per-country length +- **E-mail, IPv4/IPv6, MAC.** A dotted number right after a version word + (`version 1.2.3.4`, `v1.2.3.4`) is not an IP address. +- **Phone**: North American and other grouped national forms; international + numbers with `+` and a country code, any grouping, an optional `(0)` trunk + prefix; national numbers with a trunk `0` (`07700 900123`); a seven-digit + local number (`555-0142`) only after a word such as call, phone or tel. +- **SSN**: dashed or spaced; nine bare digits only after `SSN` or `social + security` within 40 characters. Numbers never issued (area 000, 666, 900+; + group 00; serial 0000) are left alone. +- **Cards** → `CREDIT_CARD`, with spaces, dots or dashes, checked with Luhn. + An expiry date or security code beside the card is not detected. +- **IBAN** → `BANK_ACCOUNT`, in either case: ISO 13616 mod-97 plus a per-country length table. It runs before the card detector, so an IBAN's digits cannot be claimed as a card number. - **US ABA routing number** → `BANK_ACCOUNT`: the weighted checksum and a diff --git a/docs/sanitize.md b/docs/sanitize.md index 2a16ed0..7288eeb 100644 --- a/docs/sanitize.md +++ b/docs/sanitize.md @@ -51,9 +51,12 @@ stages, there is no extract index, or the stage **stopped**; `2` completed with - **`flag`**: the rules pass **only**: deterministic, no model, milliseconds. It replaces the anchors a pattern can find: e-mail addresses, phone numbers, SSNs, cards, IBANs, routing numbers, crypto addresses, IP and MAC - addresses, API keys, private-key blocks, JWTs. **It does not find a name.** - A flag-mode catalog is not free of personal data, and the manifest doc says - so too. + addresses, API keys, private-key blocks, JWTs. It can also replace an + ordinary number that looks like a phone or ID number. **It does not find names, + street addresses, dates of birth**, or anything else that takes reading for + meaning (an employer, a school, a medical fact). It is the mode the bastion + writes into every manifest it deploys. A flag-mode catalog is not free of + personal data, and the manifest doc says so too. ## Where the sterilizer runs diff --git a/docs/serve.md b/docs/serve.md index c89d3a9..6bb5d39 100644 --- a/docs/serve.md +++ b/docs/serve.md @@ -2,7 +2,9 @@ `serve` is how a host reads what the agent prepared without the data moving: a host (the owner's, or a grantee's) sends a query and gets **sanitized** -chunks and catalog cards back. Nothing else has a route: not the extract +chunks and catalog cards back (sanitized by the manifest's mode: in `flag` +mode, which the bastion deploys, names, street addresses and dates of birth +remain; [sanitize.md](sanitize.md#modes-manifest-sanitizemode)). Nothing else has a route: not the extract store, not the vault, not the vectors table. It is `bastion_agent/serve.py`, mutual TLS with the standard library's `ssl`. @@ -31,12 +33,26 @@ There is no other credential: no token, no password, no header. | Route | Body | Returns | |---|---|---| -| `POST /search` | `{query, k?}` (k defaults to 5 and is held to 1–50) | `{hits, k}`: the top k sanitized windows with score, document id, chunk id, index and offsets. The same search as the CLI, through the same `EMBED_URL` model ([embed.md](embed.md)); 502 if the model cannot be reached. | -| `POST /read_range` | `{doc, start?, end}` | Up to 65,536 characters of the **sanitized** text of that document, with `total`. | +| `POST /search` | `{query, k?}` (k defaults to 5 and is held to 1–50) | `{hits, k, sanitize_mode}`: the top k sanitized windows with score, document id, chunk id, index and offsets, and the sanitize mode that wrote them. The same search as the CLI, through the same `EMBED_URL` model ([embed.md](embed.md)); 502 if the model cannot be reached. | +| `POST /read_range` | `{doc, start?, end}` | Up to 65,536 characters of the **sanitized** text of that document, with `total` and `sanitize_mode`. | | `GET /catalog`, `GET /catalog/dirs` | query `prefix`, `offset`, `limit` (default 500, at most 2,000) | The file cards or the directory cards (owner only), a page at a time: `{cards, kind, total, offset, limit, next_offset, prefix}`, `next_offset` being `null` on the last page. 404 until the first sweep has written a catalog. | | `GET /status` | | `{state}`: `run.state.json` (owner only). | | `GET /health` | | `{ok, id, serving}` for any client that chained. | +`sanitize_mode` says what was done to the text in that reply: the mode the +sanitize stage recorded for that document in `sanitize.jsonl`, `flag` or +`sterilize`. A search names the weakest mode among its hits. It is `unknown` +when that record does not vouch for the text: the last sanitize run did not +write the document (it stopped, or the file is an older run's), the file was +rewritten after the record (a sweep is running), or a search hit's text is no +longer what the store holds (the chunks predate the latest sanitize run). A +search with no hits names the manifest's `sanitize.mode`, or `none` when the +manifest has no sanitize stage. In `flag` mode, the one the bastion deploys, +names, street addresses and dates of birth remain in the text; only +`sterilize` looks for them. Changing the manifest's mode does not change the +field: after a change of mode the next sweep sanitizes every file again, and +until it has, the text served is the previous mode's and the reply says so. + `doc` is the document's content hash (64 lowercase hex), the same id the extract and sanitized stores share. The route reads only the `sanitize/` directory, so the plaintext for that id, which exists on disk, is never what diff --git a/pii/Dockerfile b/pii/Dockerfile index bb8ca91..d3f7efa 100644 --- a/pii/Dockerfile +++ b/pii/Dockerfile @@ -9,9 +9,18 @@ WORKDIR /app # boot (recall.py was, once) COPY __init__.py detect.py llmclient.py pipeline.py recall.py service.py vault.py /app/pii/ -# Root inside the container is deliberate: the vault db and its directory -# are created 0700/0600 (service sets umask 077) on the mounted /data, -# and the single-purpose process is bind-scoped by PII_BIND. Revisit if -# the service ever grows a second concern. +# The service runs as its own unprivileged user. The vault and its directory +# are created 0700/0600 (the service sets umask 077), so a volume mounted on +# /data must be writable by uid 10001. +RUN groupadd --system --gid 10001 pii \ + && useradd --system --uid 10001 --gid 10001 --no-create-home \ + --home-dir /nonexistent --shell /usr/sbin/nologin pii \ + && install -d -m 0700 -o pii -g pii /data +USER 10001:10001 + +# The service listens on 127.0.0.1 unless PII_BIND says otherwise, and inside +# a container that is reachable from nothing else. To publish it, choose the +# address and the host side explicitly, for example: +# docker run -e PII_BIND=0.0.0.0:8091 -p 127.0.0.1:8091:8091 ... ENV PYTHONUNBUFFERED=1 CMD ["python", "-m", "pii.service"] diff --git a/pii/__init__.py b/pii/__init__.py index 91ae805..22e0e5d 100644 --- a/pii/__init__.py +++ b/pii/__init__.py @@ -2,3 +2,9 @@ from . import detect, pipeline # noqa: F401 from .detect import PII_TYPES, Finding # noqa: F401 from .pipeline import TOKEN_RE, ground, rehydrate, replace_values, sterilize # noqa: F401 from .vault import Vault # noqa: F401 + +# The package's public names: the modules and what the agent calls in them. +__all__ = [ + "detect", "pipeline", "PII_TYPES", "Finding", "TOKEN_RE", "ground", "rehydrate", + "replace_values", "sterilize", "Vault", +] diff --git a/pii/detect.py b/pii/detect.py index adfa5d0..70ba612 100644 --- a/pii/detect.py +++ b/pii/detect.py @@ -54,23 +54,54 @@ IPV4_RE = re.compile(r"(?= 900: + if _ssn_ok(m.group(1), m.group(2), m.group(3)): + yield Finding(m.group(0), "SSN", m.start(), m.end()) + for m in SSN_SPACED_RE.finditer(text): + if _ssn_ok(m.group(1), m.group(2), m.group(3)): + yield Finding(m.group(0), "SSN", m.start(), m.end()) + for m in SSN_BARE_RE.finditer(text): + if not _ssn_ok(m.group(1), m.group(2), m.group(3)): + continue + if not SSN_CUE_RE.search(text[max(0, m.start() - 40):m.start()]): continue yield Finding(m.group(0), "SSN", m.start(), m.end()) @@ -225,6 +284,18 @@ def _find_phones(text): if not 10 <= len(digits) <= 15: continue yield Finding(m.group(0), "PHONE", m.start(), m.end()) + for m in INTL_PHONE_RE.finditer(text): + # E.164 allows at most 15 digits; a (0) trunk prefix is not one of them + digits = re.sub(r"\D", "", m.group(0).replace("(0)", "", 1)) + if 8 <= len(digits) <= 15: + yield Finding(m.group(0), "PHONE", m.start(), m.end()) + for m in NATIONAL_PHONE_RE.finditer(text): + digits = re.sub(r"\D", "", m.group(0)) + if 10 <= len(digits) <= 11: + yield Finding(m.group(0), "PHONE", m.start(), m.end()) + for m in LOCAL_PHONE_RE.finditer(text): + if PHONE_CUE_RE.search(text[max(0, m.start() - 30):m.start()]): + yield Finding(m.group(0), "PHONE", m.start(), m.end()) # Priority order: earlier detectors own their spans outright. Secrets go diff --git a/pii/recall.py b/pii/recall.py index 6f78ab4..77f7b1e 100644 --- a/pii/recall.py +++ b/pii/recall.py @@ -22,7 +22,7 @@ downgrade the sensor. Stdlib-only here; the gliner package is the one optional third-party import in the module and it is loaded lazily. """ -from typing import Callable, List, Tuple +from typing import List, Tuple from . import llmclient from .detect import PII_TYPES diff --git a/pii/service.py b/pii/service.py index 6e2a116..4b5e98e 100644 --- a/pii/service.py +++ b/pii/service.py @@ -19,7 +19,8 @@ exists for the unit-test harness only. /rehydrate returns REAL values — callers are trusted surfaces only; every call lands in the vault's audit table. -Config (env): PII_BIND (host:port, default 0.0.0.0:8091), LLM_URL +Config (env): PII_BIND (host:port, default 127.0.0.1:8091; another address +only when set), LLM_URL (default http://llm:8080), PII_DB (default /data/vault.db), PII_LLM_TIMEOUT seconds (default 300). """ @@ -38,7 +39,7 @@ _MAX_BODY = 2 * 1024 * 1024 class Config: def __init__(self, env): - self.bind = env.get("PII_BIND", "0.0.0.0:8091") + self.bind = env.get("PII_BIND", "127.0.0.1:8091") self.llm_url = env.get("LLM_URL", "http://llm:8080") self.db = env.get("PII_DB", "/data/vault.db") self.api_key = env.get("PII_API_KEY", "") diff --git a/pii/test_pii.py b/pii/test_pii.py index 01e75e4..f468941 100644 --- a/pii/test_pii.py +++ b/pii/test_pii.py @@ -47,12 +47,12 @@ class DetectTests(unittest.TestCase): self.assertEqual(self.find("ref 4111 1111 1111 1112"), set()) def test_ipv4_owns_dotted_span(self): - found = self.find("host at 192.168.68.54 responded") - self.assertIn(("192.168.68.54", "IP_ADDRESS"), found) - self.assertNotIn(("192.168.68.54", "PHONE"), found) + found = self.find("host at 198.51.100.54 responded") + self.assertIn(("198.51.100.54", "IP_ADDRESS"), found) + self.assertNotIn(("198.51.100.54", "PHONE"), found) def test_ipv4_invalid_octet(self): - self.assertNotIn(("999.168.68.541", "IP_ADDRESS"), self.find("x 999.168.68.541")) + self.assertNotIn(("999.51.100.541", "IP_ADDRESS"), self.find("x 999.51.100.541")) def test_ipv6_and_mac_disambiguation(self): found = self.find("fe80::1ff:fe23:4567:890a from aa:bb:cc:dd:ee:ff") @@ -127,6 +127,87 @@ class DetectTests(unittest.TestCase): self.assertEqual(self.find("the skate park (AKA the bowl) opens at 9"), set()) +class RulesOnlyFormatTests(unittest.TestCase): + """The rules pass is all that flag mode runs, so each format here must be + found with no recall model at all, and each lookalike must be left alone.""" + + FOUND = [ + ("UK mobile, international", "Mobile: +44 7700 900123", "+44 7700 900123", "PHONE"), + ("German, (0) trunk prefix", "Tel. +49 (0)30 1234567", "+49 (0)30 1234567", "PHONE"), + ("international, no separators", "ring +442079460958 today", "+442079460958", "PHONE"), + ("international, dots", "fax +33.6.39.98.12.34 now", "+33.6.39.98.12.34", "PHONE"), + ("international, dashes", "cell +44-20-7946-0958.", "+44-20-7946-0958", "PHONE"), + ("UK national, trunk 0", "call 07700 900123 later", "07700 900123", "PHONE"), + ("German national, trunk 0", "Tel 030 1234567", "030 1234567", "PHONE"), + ("German national, (0xx)", "Tel (030) 1234567", "(030) 1234567", "PHONE"), + ("7-digit local after a phone word", "Ring 555-0142 for the gate code.", "555-0142", "PHONE"), + ("7-digit local, call", "call me on 555 0199", "555 0199", "PHONE"), + ("IBAN lowercase", "iban: gb82west12345698765432", "gb82west12345698765432", "BANK_ACCOUNT"), + ("IBAN lowercase, spaced", "pay de89 3704 0044 0532 0130 00 now", + "de89 3704 0044 0532 0130 00", "BANK_ACCOUNT"), + ("IBAN followed by a four-letter word", "to BE68 5390 0754 7034 then", + "BE68 5390 0754 7034", "BANK_ACCOUNT"), + ("card with dots", "Card 4111.1111.1111.1111", "4111.1111.1111.1111", "CREDIT_CARD"), + ("card with dashes", "card 5555-5555-5555-4444 ok", "5555-5555-5555-4444", "CREDIT_CARD"), + ("card after no.", "card no.4111 1111 1111 1111", "4111 1111 1111 1111", "CREDIT_CARD"), + ("card, 4-6-5 grouping", "amex 3782 822463 10005", "3782 822463 10005", "CREDIT_CARD"), + ("SSN with spaces", "SSN 219 09 9999", "219 09 9999", "SSN"), + ("SSN, no separators, keyword", "Social security number 219099999.", "219099999", "SSN"), + ("SSN, no separators, SSN:", "ssn: 219099999", "219099999", "SSN"), + ] + + LEFT_ALONE = [ + ("order number", "Order 555-0142 shipped"), + ("order number, digits only", "order #20431877 is packed"), + ("9-digit invoice id", "invoice 219099999 is due"), + ("16 digits failing Luhn, spaces", "ref 4111 1111 1111 1112"), + ("16 digits failing Luhn, dots", "ref 4111.1111.1111.1112"), + ("IBAN shape, bad checksum, lowercase", "iban gb82west12345698765433"), + ("IBAN shape, bad checksum, uppercase", "iban GB82WEST12345698765433 ok"), + ("version string", "version 1.2.3.4 shipped"), + ("version string, v", "upgrade to v1.2.3.4 today"), + ("ISO date", "released 2026-08-03 at noon"), + ("US date", "Date of birth: 03/14/1987"), + ("dotted date", "seen on 14.03.1987 in the hall"), + ("SSN shape, group 00", "ssn 219-00-9999"), + ("short number after +", "add +12 345 to it"), + ("spaced digits inside a longer run", "lot 4 219 09 9999 7"), + ] + + def find(self, text): + return {(f.value, f.type) for f in detect.detect(text)} + + def test_each_format_is_found(self): + for label, text, value, typ in self.FOUND: + with self.subTest(label): + self.assertIn((value, typ), self.find(text)) + + def test_lookalikes_are_left_alone(self): + for label, text in self.LEFT_ALONE: + with self.subTest(label): + self.assertEqual(self.find(text), set()) + + def test_rules_only_sterilize_leaves_no_value(self): + # what flag mode does: the pipeline with no recall model + with tempfile.TemporaryDirectory() as tmp: + vault = Vault(os.path.join(tmp, "v.db")) + for label, text, value, _ in self.FOUND: + with self.subTest(label): + out, _ = pipeline.sterilize(text, "s", vault, lambda t: [], + escape_held_literals=True) + self.assertNotIn(value, out) + self.assertEqual(pipeline.rehydrate(out, "s", vault)[0], text) + for label, text in self.LEFT_ALONE: + with self.subTest(label): + out, _ = pipeline.sterilize(text, "s", vault, lambda t: [], + escape_held_literals=True) + self.assertEqual(out, text) + + def test_ip_address_is_still_found_after_other_words(self): + self.assertIn(("203.0.113.42", "IP_ADDRESS"), self.find("Server at 203.0.113.42 refused.")) + self.assertIn(("203.0.113.42", "IP_ADDRESS"), self.find("server 203.0.113.42 version 2")) + + class VaultTests(unittest.TestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory() @@ -672,6 +753,14 @@ class ServiceTests(unittest.TestCase): with self.assertRaises(SystemExit): service.Config({"PII_DB": "x"}) + def test_default_bind_is_loopback(self): + # /rehydrate returns real values over plain HTTP: only an explicit + # setting may open the service beyond this machine + self.assertEqual(service.Config({"PII_DB": "x", "PII_API_KEY": "k"}).bind, + "127.0.0.1:8091") + self.assertEqual(service.Config({"PII_DB": "x", "PII_API_KEY": "k", + "PII_BIND": "0.0.0.0:8091"}).bind, "0.0.0.0:8091") + def test_fail_closed_on_unstartable_recall_backend(self): with self.assertRaises(SystemExit) as ctx: service.Config({"PII_DB": "x", "PII_API_KEY": "k", "PII_RECALL": "nope"}) diff --git a/scripts/test_no_personal_data.py b/scripts/test_no_personal_data.py index 074bae1..ca60b18 100644 --- a/scripts/test_no_personal_data.py +++ b/scripts/test_no_personal_data.py @@ -17,31 +17,41 @@ lands here is published far past where it came from. This fails on: - term a word or phrase from TERMS: the owner's and members' details that - were once copied into a repository (bank, employer, account - digits, places, names, addresses, a device, a customer, a person's - project, a Host's content, record ids). They are held as keyed + term a word or phrase from TERMS, a list of details about real people + that must not appear in this repository. They are held as keyed digests, so this file does not publish what it keeps out, and a guess cannot be tested against the table without the key: every 1-3 token n-gram of a line is normalised (lowercase letters and digits, joined by one space), HMAC-SHA256'd with the maintainers' - key and looked up. The key lives outside the repository + key and looked up. A backslash escape (\\n, \\t, \\x41, \\u00e9...) + separates tokens as whitespace would, so a term written right + after one is still found. The key lives outside the repository (NO_PERSONAL_DATA_KEY_FILE, default /etc/privatae/keys/no-personal-data.hmac), as do the plain list and the script that hashes it. Where the key is absent (a CI job, which holds no secrets) this class is skipped and the run says - so; the other classes always run. A key other than the one TERMS + so; the other classes still run. A key other than the one TERMS was made with fails the run (KEY_CHECK), since it would match nothing. - entity-id a real entity id that labels a person's Host (RESERVED_ENTITY_IDS) - in entity-N, entity N, entity_id=N, "entity_id": N, entity id N, - Host N, Host #N or host-N form; examples and tests use 9000 and up + entity-id a reserved entity id, one that labels a person's Host, in + entity-N, entity N, entity_id=N, "entity_id": N, entity id N, + Host N, Host #N or host-N form; examples and tests use 9000 and + up. The ids live outside the repository, one per line + (RESERVED_ENTITY_IDS_FILE, default + /etc/privatae/reserved-entity-ids). Where that file is absent (a + CI job) this class is skipped and the run says so; a file that + holds no id, or a line that is not one, fails the run, since it + would match nothing. provenance a line that says its data is real: "real rows" followed by "that" or "from", an all-caps "REAL" row, "verbatim" from a transcript or a run, a row "read off" an entity... email a mailbox at a personal mail provider (gmail, yahoo, proton...) phone a North American number outside the 555 fiction range +A run of 24 or more base64 or base64url characters that decodes to mostly +printable UTF-8 is decoded and its text checked for every class as well, so a +value is not let through by being encoded. + Every tracked text file is scanned, file names included, and every text member of each zip-format bundle (an app bundle is one), since a bundle is what installs. Lockfiles and other binary files are not. @@ -63,6 +73,8 @@ Exit 0 = clean; exit 1 + FAIL lines otherwise. from __future__ import annotations +import base64 +import binascii import hmac import os import re @@ -118,13 +130,43 @@ TERMS: frozenset[str] = frozenset({ }) _NGRAM_MAX = 3 -# Entity ids of real Hosts that belong to people (the operator's own, members', -# the platform's Security entity and the two security-test Hosts). -RESERVED_ENTITY_IDS = (3, 102, 122, 123, 128, 163, 164) -_IDS = "|".join(str(i) for i in RESERVED_ENTITY_IDS) -ENTITY_ID = re.compile( - r"\b(?:entit(?:y|ies)(?:[-_ ]|_id\s*==?\s*|_id[\"']?\s*:\s*|\s+id\s+|\s+#)" - r"|Host\s+#?|host-)(?:%s)\b" % _IDS, re.I) +# The reserved entity ids are read from a file outside the repository, one id +# per line; blank lines and lines starting with # are ignored. +RESERVED_ENTITY_IDS_FILE = os.environ.get("RESERVED_ENTITY_IDS_FILE", "/etc/privatae/reserved-entity-ids") + + +def load_reserved_ids(path: str = RESERVED_ENTITY_IDS_FILE): + """(ids, error). A missing file is (None, None): the class is skipped. A + file that cannot be read, holds a line that is not an id, or holds no id + at all is (None, error): a list that matches nothing must not pass.""" + try: + with open(path, encoding="utf-8") as f: + lines = f.read().splitlines() + except FileNotFoundError: + return None, None + except (OSError, UnicodeDecodeError) as e: + return None, "the reserved entity ids at %s cannot be read: %s" % (path, e) + ids = [] + for n, raw in enumerate(lines, 1): + s = raw.strip() + if not s or s.startswith("#"): + continue + if not s.isdigit(): + return None, "%s:%d is not an entity id (one id per line)" % (path, n) + ids.append(int(s)) + if not ids: + return None, "%s holds no entity id, so the entity-id check would match nothing" % path + return tuple(ids), None + + +def entity_id_pattern(ids): + """The entity-id class for these ids, or None (the class is off).""" + if not ids: + return None + alts = "|".join(str(i) for i in sorted(set(ids), key=lambda i: -len(str(i)))) + return re.compile( + r"\b(?:entit(?:y|ies)(?:[-_ ]|_id\s*==?\s*|_id[\"']?\s*:\s*|\s+id\s+|\s+#)" + r"|Host\s+#?|host-)(?:%s)\b" % alts, re.I) PROVENANCE = re.compile( r"\breal rows? (?:that|from|read|copied|taken|off)\b" @@ -181,13 +223,25 @@ def key_matches(key: bytes) -> bool: return hmac.compare_digest(digest(KEY_CHECK_TEXT, key), KEY_CHECK) +# A backslash escape as source and data files write it: a named control +# character, a hex or unicode escape, or an octal one. "a\nb" is two words, but +# the tokenizer alone reads "a" and "nb". The line is read once as written and +# once with each escape turned into a space; a hit in either counts. Reading it +# as written too keeps a term that follows a backslash which is not an escape +# (a Windows path: C:\name) whole. +_ESCAPE = re.compile(r"\\(?:x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8}|[0-7]{1,3}|[abefnrtv])") + + class TermScanner: """Finds TERMS in a line by hashing its 1-3 token n-grams. Digests are - cached, so a repeated n-gram is hashed once.""" + cached, so a repeated n-gram is hashed once. It also carries the + entity-id pattern for the reserved ids (None: that class is off).""" - def __init__(self, terms, key): + def __init__(self, terms, key, entity_ids=None): self.terms = terms self.key = key + self.entity_ids = entity_ids + self.entity_re = entity_id_pattern(entity_ids) self._cache = {} def _hit(self, gram: str) -> bool: @@ -195,16 +249,64 @@ class TermScanner: self._cache[gram] = digest(gram, self.key) in self.terms return self._cache[gram] + def _grams(self, text: str) -> dict: + toks = [t.lower() for t in _TOKEN.findall(text)] + hits = {} + for n in range(1, _NGRAM_MAX + 1): + for i in range(len(toks) - n + 1): + gram = " ".join(toks[i:i + n]) + if self._hit(gram): + hits[gram] = hits.get(gram, 0) + 1 + return hits + def scan(self, line: str) -> int: if self.key is None: return 0 - toks = [t.lower() for t in _TOKEN.findall(line)] - found = 0 - for n in range(1, _NGRAM_MAX + 1): - for i in range(len(toks) - n + 1): - if self._hit(" ".join(toks[i:i + n])): - found += 1 - return found + hits = self._grams(line) + if "\\" in line: + for gram, n in self._grams(_ESCAPE.sub(" ", line)).items(): + hits[gram] = max(hits.get(gram, 0), n) + return sum(hits.values()) + + +# ── values inside base64 ───────────────────────────────────────────────────── +# A run of base64 or base64url characters, with its padding. The run is decoded +# at each of the four offsets, since it may be glued to the text before it or +# cut by a window edge; an offset whose bytes are mostly printable UTF-8 is the +# text. Random bytes almost never are, so identifiers, paths and digests that +# happen to be base64-shaped decode to nothing worth reading. +_B64_RUN = re.compile(r"[A-Za-z0-9+/_-]{24,}={0,2}") +_B64_MIN = 24 +_B64_PRINTABLE = 0.85 +# A decoded text may itself hold base64; it is followed this many levels down. +_B64_DEPTH = 2 + + +def _printable_share(text: str) -> float: + if not text: + return 0.0 + ok = sum(1 for ch in text if ch in "\t\n\r" or (ch.isprintable() and ch != "\ufffd")) + return ok / len(text) + + +def b64_texts(line: str): + """The text of every base64 run in line that decodes to mostly printable + UTF-8, one per run.""" + for m in _B64_RUN.finditer(line): + run = m.group(0).rstrip("=").replace("-", "+").replace("_", "/") + for off in range(4): + body = run[off:] + body = body[:len(body) - len(body) % 4] if len(body) % 4 == 1 else body + if len(body) < _B64_MIN: + break + try: + raw = base64.b64decode(body + "=" * (-len(body) % 4), validate=True) + except (ValueError, binascii.Error): + continue + text = raw.decode("utf-8", "replace") + if _printable_share(text) >= _B64_PRINTABLE: + yield text + break # A minified asset or a bundle member can hold one line of megabytes. It is read @@ -241,16 +343,66 @@ def line_hits(line: str, scanner: TermScanner): # Classes whose line would carry the personal value itself: their FAIL line # names the file, the line and the class, never the text, so a CI log does not -# publish what the gate keeps out. +# publish what the gate keeps out. A line that holds one shows no text for any +# finding, since another class's excerpt of the line would carry the value too. _QUIET = {"term", "email", "phone"} -def line_findings(line: str, scanner: TermScanner): +_PATH_SEP = re.compile(r"([/!])") + + +def shown_path(label: str, scanner: TermScanner) -> str: + """label as a FAIL line shows it. A path component (a directory, a file + name, a bundle member's name) that holds a quiet value is replaced by a + placeholder giving its depth, so the line says where without the text. A + value that runs across components withholds them all.""" + def quiet(text): + return any(cls in _QUIET for cls, _ in line_findings(text, scanner)) + + if not quiet(label): + return label + parts = _PATH_SEP.split(label) + + def masked(every): + out, depth = [], 0 + for p in parts: + if p in ("/", "!"): + out.append(p) + continue + depth += 1 + out.append("" % depth if every or quiet(p) else p) + return "".join(out) + + shown = masked(False) + return masked(True) if quiet(shown) else shown + + +def line_report(label: str, n: int, line: str, scanner: TermScanner, allow=(), used=None, + skip_entity_ids: bool = False): + """(class, FAIL text) for each finding on line n of label that no allow + entry covers.""" + used = set() if used is None else used + hits = list(line_hits(line, scanner)) + quiet = any(cls in _QUIET for _part, cls, _what in hits) + out = [] + where = None + for part, cls, what in hits: + if (cls == "entity-id" and skip_entity_ids) or allowed(allow, label, cls, part, used): + continue + if where is None: + where = shown_path(label, scanner) + shown = "" if quiet else part.strip()[:140] + out.append((cls, ("%s:%d [%s: %s] %s" % (where, n, cls, what, shown)).rstrip())) + return out + + +def line_findings(line: str, scanner: TermScanner, _depth: int = 0): out = [] for _ in range(scanner.scan(line)): out.append(("term", "a listed term")) - for m in ENTITY_ID.finditer(line): - out.append(("entity-id", m.group(0))) + if scanner.entity_re is not None: + for m in scanner.entity_re.finditer(line): + out.append(("entity-id", m.group(0))) for m in PROVENANCE.finditer(line): out.append(("provenance", m.group(0))) for m in EMAIL.finditer(line): @@ -258,6 +410,10 @@ def line_findings(line: str, scanner: TermScanner): for m in PHONE.finditer(line): if (m.group(2) or m.group(5)) != "555": out.append(("phone", "a phone number")) + if _depth < _B64_DEPTH: + for text in b64_texts(line): + for cls, what in line_findings(text, scanner, _depth + 1): + out.append((cls, what if what.endswith("(in base64)") else what + " (in base64)")) return out @@ -363,32 +519,61 @@ def self_test(): fails = [] probe_key = os.urandom(32) + # Invented ids stand in for the reserved ones, which this file never holds. + probe_ids = (9101, 9202, 9303) sc = TermScanner(frozenset(digest(g, probe_key) for g in ("zorblaxquent", "quilmar vesk", "tam orvel prax")), - probe_key) + probe_key, probe_ids) def classes(line): return {c for c, _ in line_findings(line, sc)} + def b64(s: str, url: bool = False) -> str: + enc = base64.urlsafe_b64encode if url else base64.b64encode + return enc(live(s).encode("utf-8")).decode("ascii").rstrip("=" if url else "") + positives = [(c, live(s)) for c, s in [ ("term", "pays into the Zorblaxquent ledger"), ("term", "QUILMAR-vesk sent it"), ("term", "a tam_orvel.prax file"), - ("entity-id", "entity_id=|3,"), ("entity-id", '"entity_id": |128'), ("entity-id", "Entity |164's app"), - ("entity-id", "kubectl -n entity-|3 exec"), ("entity-id", "pod entity-|3-0 restarted"), ("entity-id", "measured on Host #|163"), - ("entity-id", "gitea_user = 'host-|3'"), ("entity-id", "entity id |122 knocked"), + # A term right after a backslash escape, as a string literal or a JSON + # value writes it, and after a backslash that is not one. + ("term", r'"line one\nZorblax|quent"'), ("term", r"col\tzorblax|quent\tcol"), + ("term", r"end\r\nquilmar| vesk"), ("term", r"a\x41zorblax|quent"), ("term", r"x\u00e9zorblax|quent"), + ("term", r"C:\Users\zorblax|quent\notes"), + ("entity-id", "entity_id=|9101,"), ("entity-id", '"entity_id": |9202'), ("entity-id", "Entity |9303's app"), + ("entity-id", "kubectl -n entity-|9101 exec"), ("entity-id", "pod entity-|9101-0 restarted"), + ("entity-id", "measured on Host #|9303"), ("entity-id", "gitea_user = 'host-|9101'"), + ("entity-id", "entity id |9202 knocked"), ("provenance", "the real| rows that fired on the Host"), ("provenance", "the REAL| row 9"), ("provenance", "verbatim| from the transcript"), ("provenance", "read| off entity 9001"), ("email", "mail ana.b@gm|ail.com today"), ("email", "x@proto|nmail.com"), ("phone", "call +1 (2|02) 555-|0100 or +1 30|1 444 1234"), ("phone", "+1|3014441234"), ]] + # Every class is read inside base64 and base64url, padded or not, whole or + # glued to the text before it. + positives += [ + ("term", 'data: "%s"' % b64("a note that names Zorblax|quent outright")), + ("term", "token=%s" % b64("pays QUILMAR| vesk ~~ été ok??", url=True)), + ("term", "xyz" + b64("a note that names Zorblax|quent outright")), + ("term", b64(b64("twice over: zorblax|quent, still found"))), + ("entity-id", "blob %s end" % b64("the pod entity-|9101-0 restarted twice")), + ("email", "%s" % b64("write to ana.b@gm|ail.com about it")), + ] negatives = [live(s) for s in [ - "zorblax quent", "quilmar", "a vesk quilmar", "orvel prax tam", - "entity_id=9003", "entity_id=30", "entity 1023", "entity-3x", "host-31", "Host 7", - "entity-9310-0.svc", "the v0.3.128 tag", "entity_id = ", + "zorblax quent", "quilmar", "a vesk quilmar", "orvel prax tam", r"zorblax\nquent", + "entity_id=91011", "entity_id=910", "entity 92021", "entity-9101x", "host-91012", "Host 9104", + "entity-9310-0.svc", "the v0.9.9101 tag", "entity_id = ", "real rows in a throwaway container", "the real row replaces the echo", "returns the text verbatim", "ana@example.com", "a•••@gmail.com shows the first letter", "+1 415 555 0199", "+1|5555550100", "ids 2026091812345", "port 8800, 4096 MiB", "MemoryHigh=15032385536 MemoryMax=17179869184", "2026-10-07 12:00", "widths 278 584 556 1015 667 722", + # Base64-shaped text that is not base64 of text: identifiers, paths, + # digests and random bytes are not decoded into findings. + "TestPublishedSourceCarriesNoOldInternalNameAnywhere", + "/usr/local/lib/python3/site-packages/zorbtool/__init__", + "sha256:1ae0735f00daffa3aaf1363a5184c0d2dc55c78e3db4ec70241cdac97bf84b59", + base64.b64encode(bytes(range(200, 256))).decode("ascii"), + b64("a plain note with nothing listed in it at all"), ]] for cls, line in positives: if cls not in classes(line): @@ -408,11 +593,41 @@ def self_test(): # The real table is wired: every entry is a digest of the right shape. if not TERMS or not all(re.fullmatch(r"[0-9a-f]{32}", k) for k in TERMS): fails.append("self-test: TERMS is empty or malformed") + # A line holding a quiet value shows no text for any finding on it: here a + # provenance phrase beside a mailbox, and one base64 run holding both an + # entity id and a term. + blob = b64("the pod entity-|9101-0 belongs to zorblax|quent") + for line, want, secret in ((live("the real| rows that came from ana.b@gm|ail.com"), + {"provenance", "email"}, live("ana.b@gm|ail")), + ("note %s end" % blob, {"entity-id", "term"}, blob[:24])): + got = line_report("x.md", 1, line, sc) + if {c for c, _ in got} != want or any(secret in t for _c, t in got): + fails.append("self-test: a line with a quiet value is reported without its text, got %s" % got) + # A path that holds a quiet value is shown with that component withheld, + # by depth; one split across components withholds them all. + for label, want in (("docs/zorblaxquent-notes.md", "docs/"), + ("app.bundle!notes/" + live("ana.b@gm|ail.com") + ".txt", "app.bundle!notes/"), + ("quilmar/vesk.md", "/"), + ("docs/readme.md", "docs/readme.md")): + if shown_path(label, sc) != want: + fails.append("self-test: path %r shown as %r, want %r" % (label, shown_path(label, sc), want)) + got = line_report("docs/" + live("ana.b@gm|ail.com") + ".md", 1, live("the real| rows that"), sc) + if [t for _c, t in got] != [live("docs/:1 [provenance: real| rows that] the real| rows that")]: + fails.append("self-test: a FAIL line's path is shown withheld, got %s" % got) + # A term is counted once per place it stands, however many ways it is read. + got = [c for c, _ in line_findings(live(r"a\nzorblax|quent b"), sc)] + if got != ["term"]: + fails.append("self-test: a term after an escape is one hit, got %s" % got) # Without a key the term class is off and nothing else changes. - keyless = TermScanner(sc.terms, None) - got = {c for c, _ in line_findings(live("the zorblax|quent ledger, entity_id=|3, x@proto|nmail.com"), keyless)} + planted = live("the zorblax|quent ledger, entity_id=|9101, x@proto|nmail.com") + keyless = TermScanner(sc.terms, None, probe_ids) + got = {c for c, _ in line_findings(planted, keyless)} if got != {"entity-id", "email"}: fails.append("self-test: without a key only the term class is off, got %s" % sorted(got)) + # Without the ids the entity-id class is off and nothing else changes. + got = {c for c, _ in line_findings(planted, TermScanner(sc.terms, probe_key))} + if got != {"term", "email"}: + fails.append("self-test: without the ids only the entity-id class is off, got %s" % sorted(got)) # A key other than the one TERMS was made with is told apart from it. if key_matches(probe_key) or key_matches(b""): fails.append("self-test: a key TERMS was not made with passes the key check") @@ -426,6 +641,18 @@ def self_test(): if len(entries) != 1 or len(errors) != 2: fails.append("self-test: the allow file reader kept %d entries and %d errors, want 1 and 2" % (len(entries), len(errors))) + # The reserved ids are read one per line; a missing file turns the + # class off, and a file that would match nothing is an error. + idf = os.path.join(td, "ids") + for body, want in (("# reserved\n9101\n\n 9202 \n", ((9101, 9202), False)), + ("9101\nentity-9202\n", (None, True)), ("# none\n\n", (None, True)), ("", (None, True))): + with open(idf, "w", encoding="utf-8") as f: + f.write(body) + ids, err = load_reserved_ids(idf) + if (ids, err is not None) != want: + fails.append("self-test: reserved ids %r read as %r, %r" % (body, ids, err)) + if load_reserved_ids(os.path.join(td, "absent")) != (None, None): + fails.append("self-test: a missing reserved-ids file is not (None, None)") # A zip-format bundle is opened by its signature, whatever it is named # (an app bundle, or a .zip such as a browser extension a site serves), # and its text members are read. @@ -469,13 +696,20 @@ def main(argv) -> int: key = load_key() if key is None: print(" NOTE term check skipped: no key at %s (set NO_PERSONAL_DATA_KEY_FILE); " - "the entity-id, provenance, email and phone checks still run" % KEY_FILE) + "the other checks still run" % KEY_FILE) elif not key_matches(key): fails.append("the key at %s is not the one TERMS was made with, so it would match " "nothing: point NO_PERSONAL_DATA_KEY_FILE at that key, or remake TERMS and KEY_CHECK" % KEY_FILE) print(" FAIL %s" % fails[-1]) key = None - scanner = TermScanner(TERMS, key) + ids, ids_error = load_reserved_ids() + if ids_error: + fails.append(ids_error) + print(" FAIL %s" % ids_error) + elif ids is None: + print(" NOTE entity-id check skipped: no reserved ids at %s (set RESERVED_ENTITY_IDS_FILE); " + "the other checks still run" % RESERVED_ENTITY_IDS_FILE) + scanner = TermScanner(TERMS, key, ids) used = set() findings = [] counts = {} @@ -488,21 +722,24 @@ def main(argv) -> int: continue for cls, what in line_findings(rel, scanner): if not allowed(allow, rel, cls, rel, used): - findings.append("%s [%s: %s] (file name)" % (rel, cls, what)) + findings.append("%s [%s: %s] (file name)" % (shown_path(rel, scanner), cls, what)) counts[cls] = counts.get(cls, 0) + 1 for label, text in texts(root, rel): n_texts += 1 if text is None: fails.append("%s has a zip signature but does not open as a zip, so its " - "members cannot be read" % label) + "members cannot be read" % shown_path(label, scanner)) print(" FAIL %s" % fails[-1]) continue + # A bundle member's name installs with it, like a file name. + member = label[len(rel) + 1:] + for cls, what in (line_findings(member, scanner) if member else ()): + if not allowed(allow, label, cls, member, used): + findings.append("%s [%s: %s] (bundle member name)" % (shown_path(label, scanner), cls, what)) + counts[cls] = counts.get(cls, 0) + 1 for i, line in enumerate(text.splitlines(), 1): - for part, cls, what in line_hits(line, scanner): - if (cls == "entity-id" and label == allow_rel) or allowed(allow, label, cls, part, used): - continue - shown = "" if cls in _QUIET else part.strip()[:140] - findings.append(("%s:%d [%s: %s] %s" % (label, i, cls, what, shown)).rstrip()) + for cls, finding in line_report(label, i, line, scanner, allow, used, label == allow_rel): + findings.append(finding) counts[cls] = counts.get(cls, 0) + 1 for f in findings: print(" FAIL %s" % f)