commit 3f1d48bf9b79e0233f3b5c7ba697a00ca46d110b Author: Privatae Date: Sat Sep 26 18:31:40 2026 +0000 bastion agent 0.1.14 diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml new file mode 100644 index 0000000..5a885cd --- /dev/null +++ b/.gitea/workflows/ci.yml @@ -0,0 +1,77 @@ +# The checks the README asks for, run on every push to main and every pull +# request into main. It holds no secrets: the only credential is the job's own +# token, used to fetch the commit under test. +# +# The agent runs on whatever Python a member's machine has, down to 3.8, so the +# tests run on the floor and on a current release. Images are pinned by digest. +name: ci + +on: + push: + branches: [main] + pull_request: + branches: [main] + +env: + PY_FLOOR: python@sha256:1d52838af602b4b5a831beb13a0e4d073280665ea7be7f69ce2382f29c5a613f # 3.8-slim + PY_CURRENT: python@sha256:7c61056e61ac89e852de05f3dc6fa51a6dd2181797bceed46aa725dd7cb2cd3b # 3.13-slim + # The PyYAML release bastion_agent/build.py vendors; the manifest loader imports it. + PYYAML: pyyaml==6.0.3 + # The pull request's own head, or the pushed commit. + WANT: ${{ gitea.event.pull_request.head.sha || gitea.sha }} + +jobs: + test: + runs-on: platform + steps: + - name: fetch the commit + run: | + set -euo pipefail + git init -q . + git -c http.extraHeader="Authorization: token ${{ gitea.token }}" \ + fetch -q --depth=1 "${{ gitea.server_url }}/${{ gitea.repository }}.git" "$WANT" + git checkout -q --detach FETCH_HEAD + test "$(git rev-parse HEAD)" = "$WANT" + - name: the 3.8 syntax floor + run: | + docker run --rm -v "$PWD":/src:ro -w /src "$PY_CURRENT" \ + python -m unittest -v test_syntax_floor + - name: every test, on Python 3.8 and on a current Python + # age is installed so the seal round trips run instead of skipping. + run: | + set -euo pipefail + for img in "$PY_FLOOR" "$PY_CURRENT"; do + docker run --rm -v "$PWD":/src:ro -w /src -e PYYAML "$img" sh -euc ' + python --version + apt-get -qq update >/dev/null + apt-get -qq install -y --no-install-recommends age >/dev/null + pip install -q --disable-pip-version-check --root-user-action=ignore "$PYYAML" + python -m unittest discover .' + done + + reproducible: + needs: test + runs-on: platform + steps: + - name: fetch the commit + run: | + set -euo pipefail + git init -q . + git -c http.extraHeader="Authorization: token ${{ gitea.token }}" \ + fetch -q --depth=1 "${{ gitea.server_url }}/${{ gitea.repository }}.git" "$WANT" + git checkout -q --detach FETCH_HEAD + test "$(git rev-parse HEAD)" = "$WANT" + - name: two builds on each Python produce one sha256 + # Anyone must be able to rebuild a release and get the published bytes, + # so the .pyz may not depend on the build or the Python that made it. + run: | + set -euo pipefail + for img in "$PY_FLOOR" "$PY_CURRENT"; do + docker run --rm -v "$PWD":/src:ro -w /src -e PYYAML "$img" sh -euc ' + pip install -q --disable-pip-version-check --root-user-action=ignore "$PYYAML" + python -m bastion_agent.build --out /tmp/a | awk "{print \$NF}" + python -m bastion_agent.build --out /tmp/b | awk "{print \$NF}"' + done | tee digests + test "$(wc -l < digests)" -eq 4 + test "$(sort -u digests | wc -l)" -eq 1 + echo "every build is $(head -1 digests)" diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..21faee5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +dist/ +__pycache__/ +*.pyc diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..1c51658 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,205 @@ +# Changelog + +All notable changes to the bastion agent. + +The version in `bastion_agent/VERSION` on `main` is the next release, not +necessarily the one machines run. A merged change waits there until it is +published. The published version is whatever + names at the moment, and it +is the only version a bastion installs. When a second release is merged before +the first is published, only the later one is published, and it carries the +changes of both. This file does not record which versions were published. + +Changes to the release tooling (`publish.sh`, `build.py`, `verify_release.py`) +and to the documentation are listed under the version whose source first +carried them, although they do not change what the `.pyz` does. + +## 0.1.14 + +- **Rollout order: upgrade any running PII service before publishing this + agent.** The sanitize stage now asks the service to escape every + placeholder-shaped string in a document and stops, fail closed, when the + service does not confirm it (`stats.escape_held_literals`). An older service + does not, so a `sterilize` manifest pointed at one stops at its first file. +- **Anyone can check a release.** `python3 -m bastion_agent.verify_release` + fetches the published manifest and its signature, checks the ed25519 + signature against the release keys the bastion pins, checks out the tag + `v` (or the manifest's `built_from` commit where it exists), + rebuilds the `.pyz` with that version's `build.py`, and compares its sha256 + with every `.pyz` the manifest lists. It needs Python, git and PyYAML at the + pinned version, nothing else; the signature check is ed25519 in pure Python, + tested against RFC 8032's vectors and the vector the bastion's signing tool + and the bastion share. `build.py` leaves it out of the `.pyz`. +- `SECURITY.md` (how to report a problem in the agent, and what is in scope), + `CONTRIBUTING.md` (the checks a change must pass and the rules it must keep) + and this changelog. +- The documents under `docs/` describe the program as it is. Test commands are + the real ones (`python3 -m unittest …`), and the status lines, dates, live + measurements from the development machine and lists of work that is now + done are gone. Descriptions that no longer matched the code are corrected, + among them: the built-in PDF reader, the Office limits, the default output + directory of each command, the `/catalog` paging and `--port-file` of + `serve`, which environment variables the PII service reads, and when a chunk + keeps its vector. +- Comments and help text follow the same rule: the `--out` help names the real + default, and no comment names a machine, a model deployment or a file that is + not in this repository. No change in behaviour. +- `pii/verify-live.sh` takes the service's address as an argument (default + `127.0.0.1:8091`, where `pii.service` binds) and the key from + `PII_API_KEY`, instead of reading a file outside this repository. +- `sweep` without `--out` now uses `/.agent`, the same folder as + `serve`. +- PII: overlapping matches, including a value that overlaps itself, become one + placeholder; no match may split an existing placeholder; placeholder-looking + text in a document always survives a round trip (through the service, only + text the vault does not hold; `escape_held_literals: true` asks for all of + it), and a chunk is never cut inside a placeholder; a phone number, IPv4 or + MAC address at the end of a sentence is now detected, and one followed by + `.txt`-style text is not. +- The `residual_tokenized` and `verify_tokenized` stats count what was actually + replaced. There is a new stat, `literal_tokens_escaped`, and `/sterilize` + echoes `stats.escape_held_literals`. +- The sanitize stage re-sanitizes every file once after this upgrade + (`SANITIZE_VERSION` 2): output written by older code is not reused. +- The legacy address `privatae.ai/freeagent/` is retired, so this release is + published under the new file name only (`publish.sh` finds the route gone + on every edge and the public 404 by itself). A bastion older than 0.20.0 + keeps the agent it has. + +## 0.1.13 + +- No change in what the agent does. The README describes it as it is: the + stages and their order, how the bastion runs the agent, what `serve` + returns, verifying a release, and reporting a security problem. +- **Release manifests are signed.** 0.1.13 is the first release published + with a signature; none is served for the 0.1.12 manifest. `publish.sh` + builds the release, prints the command for the key holder to sign it on + their own computer (the release key never comes to the publishing machine), + and publishes only once the signature verifies against the keys the bastion + pins, both in the release being prepared and in the one machines run now. The signature is published + beside the manifest as `version.json..sig`, and + old signature files are never removed. +- **The build is reproducible on any machine.** `build.py` vendors PyYAML only + at the pinned version (`PYYAML_VERSION`) and refuses any other, so two + machines build the same bytes from the same source. +- `publish.sh` writes the legacy `freeagent-.pyz` file name only while + some edge still serves the legacy `/freeagent/` address, judged per edge from + the configuration its web server has loaded; `docs/packaging.md` has the + steps to retire that address. + +## 0.1.12 + +- No change in behaviour: comments no longer carry ticket numbers or name + machines, so the source can be published as it is. + +## 0.1.11 + +- **The manifest gate needs no openssl.** Reading the certificate, checking + its chain to `FREEAGENT_CA_ROOT` and verifying the ECDSA P-256 signature are + done in the standard library (`x509gate.py`), with strict DER parsing, so the + agent runs on machines without openssl, Windows included. `sign --value` + records a signature made elsewhere (the bastion signs with its own key); + `sign --key` still uses openssl. `doctor` no longer lists openssl as + required. +- **The built-in PDF reader reads real PDFs** (`pdftext.py`): object streams, + compressed streams, the page tree, fonts' `/ToUnicode` maps (so + browser-printed PDFs read), simple font encodings, glyph widths for spacing, + and form XObjects. Without poppler it used to find text only in old PDFs. It + reads on a budget per document (time, decoded bytes, operators, tokens, + CMap entries) and never raises; encrypted files are refused. Cached PDF + verdicts are redone (`EXTRACT_VERSION` 2). +- **Office files are read on a budget.** DOCX, XLSX, PPTX and OpenDocument + files are parsed as a stream, each finished paragraph or row dropped once + read, with caps on XML size, text size, time and parts per paragraph or row; + reaching a cap keeps the text read so far and says so. A DTD is refused, and + UTF-16 documents are read. + +## 0.1.10 + +- Licensed under the GNU General Public License, version 3 or later, in place + of PolyForm Internal Use. The `LICENSE` file is also inside the `.pyz`. + +## 0.1.9 + +- No change to the agent. Test fixtures use generic addresses, and + `publish.sh` takes its backup directory from the environment and refuses one + inside the web root. + +## 0.1.8 + +- Licensed under PolyForm Internal Use 1.0.0 in place of PolyForm + Noncommercial 1.0.0, which had forbidden running the agent for work. +- `publish.sh` names no publishing target of its own: the edges, key and web + root come from its environment, and it stops when one is missing. Documents + no longer cite files outside the repository, and test fixtures use invented + people. No change in behaviour. + +## 0.1.7 + +- **The program is named the bastion agent.** The package is `bastion_agent`, + the artifact `bastion-agent-.pyz`, the command `bastion-agent`, and + the address `privatae.ai/bastion-agent/`. The manifest's `freeagent: 1` + field, the `FREEAGENT_*` environment variables, the `freeagent` stamp in + output records and the `build/freeagent/` output directory keep the old + word: bastions write and set them, and output already on disk uses them. + Every publish lists the artifact under both file names, with the same bytes. +- The agent and the PII sterilizer became a repository of their own, licensed + under PolyForm Noncommercial 1.0.0. + +## 0.1.6 + +- `serve`'s `/catalog` and `/catalog/dirs` are paged: `prefix`, `offset` and + `limit`, answered with `total` and `next_offset`. + +## 0.1.5 + +- `serve` answers `/catalog/dirs`: the directory cards, the map a host plans + from. + +## 0.1.4 + +- `.agent` is excluded in any letter case on every OS (a case-insensitive Mac + volume has only one `.Agent`). +- The documents say plainly that the extract store in `.agent` is the plaintext + of every file in scope, and that a folder should be served from one machine. + +## 0.1.3 + +- **An agent lives with its folder.** For a manifest with one root, the output + directory defaults to `/.agent`, and the walk always excludes `.agent`. +- `FREEAGENT_VAULT_DIR` keeps the PII vault out of the folder, which may be + synced elsewhere. + +## 0.1.2 + +- `serve --port-file` writes the port it bound, so a supervisor can bind port + 0 and learn it. +- `publish.sh`: the agent is published at `privatae.ai/freeagent/`. + +## 0.1.1 + +- Each TLS handshake in `serve` runs in its connection's own thread with a + timeout, and an idle connection is closed after 60 s, so a client that says + nothing cannot hold up the others. +- The manifest gate requires `FREEAGENT_CA_ROOT` and a signing certificate + with exactly one CN, and a refused manifest is recorded in the run log. +- Every segment of a path is masked in the catalog, not only the file name. +- The sweep lock is never taken from a live process; `chunk` and `catalog` + require the `sanitize` stage; a malformed CSV is an error record instead of a + crash. + +## 0.1.0 + +The first version with a number: the whole pipeline in one file. + +- The job manifest, its loader and validator. +- The stages: `walk`, `extract`, `sanitize` (the PII sterilizer, in process or + as a service, with rules, a recall model and a local vault), `chunk`, + `embed` (a local model, vectors in SQLite) and `search`, `catalog`, and + `seal` (an age envelope to the owner and named recipients). +- `sweep`, the scheduler with the manifest signature gate and a lock, and a + systemd user unit. +- `serve`, the mutual-TLS query endpoint, with grants read from the signed + manifest on each request. +- Packaging: a reproducible single-file zipapp, the `doctor` preflight, and + installers for sh and PowerShell. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..79ed029 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,89 @@ +# Contributing + +Thank you for looking at the bastion agent. It runs on machines people own and +reads their files, so the bar is: nothing lands that lets personal data leave +the machine unsanitized, or lets the agent do more than its signed manifest +says. + +## Before you open a pull request + +From the repository root: + +```sh +python3 -m unittest discover . # every test, including the 3.8 floor +python3 -m bastion_agent.build --out /tmp/agent # the .pyz and its .sha256 +python3 /tmp/agent/bastion-agent-*.pyz --version +bash -n bastion_agent/publish.sh pii/verify-live.sh && sh -n bastion_agent/install.sh +``` + +- **All tests pass.** They need PyYAML (the manifest loader uses it), at the + version `build.py` pins (`PYYAML_VERSION`) for the build tests. The + round-trip seal tests are skipped where `age` and `age-keygen` are not + installed, so install them to run the whole suite. +- **The Python 3.8 floor.** `test_syntax_floor.py` parses every module in + `bastion_agent/` and `pii/` at 3.8 and denies the newer standard-library + calls that parse on 3.8 and then fail at run time. The agent ships to + whatever the owner runs; keep it there. +- **The build reproduces.** `test_build` builds twice and requires identical + bytes. Nothing that depends on the machine or the clock (a timestamp, an + unsorted directory listing, bytecode, a different PyYAML) may reach the + `.pyz`. Anyone checks a release by rebuilding it + (`python3 -m bastion_agent.verify_release`), so a build that drifts breaks + that check for everyone. +- **Standard library only** in both packages, apart from the vendored PyYAML + and the optional `gliner` import. A new dependency is new attack surface in + the process that holds the most sensitive data on the machine. + +A change to a stage's output adds a test for it, and a change to what a stage +refuses adds a test that it refuses. + +## What a change has to respect + +- **Bytes never leave the machine.** The files, their extracted text and the + vault stay local. What leaves is the sealed catalog, to the recipients the + manifest names, and what `serve` returns to a certificate the manifest + allows. There is no plaintext sink: `sink.plaintext` is `never` and has no + other value. +- **Sanitize before anything is written for later stages.** Chunks, vectors, + keywords, titles, summaries, the catalog and everything `serve` returns come + from the sanitized store, never the extract store. A test that searches an + output for the fixture's personal data should keep passing. +- **Fail closed.** A recall pass that fails stops the stage; a manifest that + does not verify runs nothing and serves nothing; a model with the wrong + dimension writes nothing. When in doubt, stop and say why in the run record. +- **No delete, no write.** The agent never changes, moves or deletes the files + it reads, and rows in the vault and the vectors table are only ever added. +- **Addresses come from the environment, never the manifest.** A manifest names + models and policy; where the model or the sterilizer runs is the machine's + business. +- **Stable wire names stay.** Installed bastions write `freeagent: 1` into + every manifest and set the `FREEAGENT_*` environment variables; renaming one + breaks them. The `build/freeagent/` default output directory keeps its name + so output a person already has is not orphaned. +- **The release keys match the bastion's.** `RELEASE_KEYS` in + `bastion_agent/verify_release.py` is a copy of `productionReleaseKeys` in the + bastion's `client-go/release_sign.go`. When the bastion's list changes, + change this one the same way. + +## Style + +- Comments explain why, in plain words. State the invariant and the failure it + prevents. No ticket numbers, no dates, no names of people or machines. +- Documentation describes the program as it is, including what it does not + do. It carries no status lines, dates or plans, which go stale the day they + land. +- One module per stage, with its own `test_.py` and its own document + under `docs/`. +- Error text a person reads says what was refused and what they can do about + it. + +## Reporting a problem + +Bugs and feature requests: the issue tracker on the repository. Security +problems: see [SECURITY.md](SECURITY.md), and please do not file them +publicly. + +## License + +By contributing you agree that your contribution is licensed under the GNU +General Public License, version 3 or later, like the rest of the repository. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..f288702 --- /dev/null +++ b/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/README.md b/README.md new file mode 100644 index 0000000..24ba4be --- /dev/null +++ b/README.md @@ -0,0 +1,159 @@ +# bastion agent + +[![ci](https://git.privatae.ai/privatae/bastion-agent/actions/workflows/ci.yml/badge.svg?branch=main)](https://git.privatae.ai/privatae/bastion-agent/actions) + +The small program the bastion runs on your own machine, for one folder. + +It is deployed **from a grant, never installed by a person**: the bastion fetches +`privatae.ai/bastion-agent/version.json`, checks its ed25519 release signature +(`version.json..sig`) against the keys compiled into the bastion, downloads the `.pyz` +and checks it against the sha256 in that signed manifest, +mints a P-256 identity per store, writes and signs a manifest from the grant you +approved, and supervises `sweep` and `serve`. Your files never leave the machine; +what crosses the tunnel is the index and what the grant's mask allows. + +The published version is whatever `privatae.ai/bastion-agent/version.json` +names. `bastion_agent/VERSION` on `main` is the next release, which may not be +published yet. Release manifests are published signed from 0.1.13 on, and the +`.pyz` can be rebuilt from this source and compared +([Verify a release](#verify-a-release)). + +This repository is free software, licensed under the GNU General Public License version 3 or (at your option) any later version (see `LICENSE`). Copyright Privatae LLC. You may run it, read it, change it and redistribute it under that license. The Privatae platform the bastion connects to is separate, proprietary software and is not covered by this license. A member runs their own build on their own machine — nothing here depends on a Privatae-held credential or a Privatae-side build. + +## What it does + +For one shared folder, on the manifest's cadence, the agent runs the stages its +manifest names, always in this order and starting with `walk`; the first +failure stops the run: + +| stage | what it does | doc | +|---|---|---| +| walk | the map of the folder: files, sizes, types, what the grant excludes | [walk.md](docs/walk.md) | +| extract | text out of each file, with built-in extractors or local tools | [extract.md](docs/extract.md) | +| sanitize | finds personal data and replaces it with tokens; the originals stay in a vault on this machine | [sanitize.md](docs/sanitize.md), [pii-sterilization.md](docs/pii-sterilization.md) | +| chunk, embed | splits the text and embeds it with a model on this machine | [embed.md](docs/embed.md) | +| catalog | the curated dataset built from the stages above | [catalog.md](docs/catalog.md) | +| seal | packs an allowlisted bundle of the catalog and encrypts it with age to the owner and the manifest's named recipients; the one way a dataset leaves the machine | [seal.md](docs/seal.md) | + +`serve` answers queries over the index with mutual TLS, returning sanitized +chunks and catalog cards ([serve.md](docs/serve.md)). The gate refuses an +unsigned, edited or foreign manifest, and the bastion never starts the agent +without a valid one. (`--allow-unsigned` exists for development only; the +bastion never passes it.) See [runtime.md](docs/runtime.md) and +[manifest.md](docs/manifest.md). + +## How the bastion runs it + +The bastion installs and runs the agent itself from a folder grant you +approved; nobody installs it by hand. For each shared folder the bastion: + +- downloads the `.pyz` only through the signed manifest, and never replaces an + installed agent with an older version; +- mints the agent its own identity and writes and signs its manifest from the + grant, so the agent works only on what you granted; +- points the embed stage at the model server on this machine (a local ollama) + and at nothing else; +- runs `serve` on the loopback address only, with mutual TLS, so the bastion is + the only client; +- keeps the personal-data vault in the bastion's own state directory rather + than in the folder, which may be synced elsewhere; +- on a machine where the bastion runs as root, runs the agent as the folder's + owner, never as root. + +The agent updates without asking: while a folder is shared, the bastion checks +the agent's signed manifest every ten minutes and installs any newer version. +The bastion's [SECURITY.md](https://git.privatae.ai/privatae/privatae-bastion/src/branch/main/SECURITY.md) +says what that means for what you trust. + +## Layout + +| path | what it is | +|---|---| +| `bastion_agent/` | the agent: walk, extract, sanitize, embed, catalog, seal, manifest, serve; plus `build.py` and `verify_release.py`, which make and check a release and are not part of the `.pyz` | +| `pii/` | the sterilizer the agent runs **in-process** — its rules, vault and recall backends | +| `docs/` | one document per module, plus `pii-sterilization.md` | +| `test_syntax_floor.py` | the Python 3.8 floor gate for both packages | + +`bastion_agent/` and `pii/` are siblings on purpose: `sanitize.py` and `catalog.py` +import `pii`, and `bastion_agent/build.py` bundles it into the `.pyz` as a top-level +package, which is what the built artifact has always looked like inside. + +## Build and test + +```sh +python3 -m unittest discover . # every module's tests, plus the 3.8 floor +python3 -m bastion_agent.build --out dist # produces dist/bastion-agent-.pyz + .sha256 +python3 dist/bastion-agent-.pyz --version +``` + +Every push runs the same checks in CI (`.gitea/workflows/ci.yml`). + +The build is reproducible: the same commit gives the same bytes on any machine +with PyYAML at the pinned version (`PYYAML_VERSION` in `build.py`). + +The floor matters: this ships to whatever the owner runs, and a Mac or Windows +box may lag years behind Debian stable. `test_syntax_floor.py` parses both +packages at 3.8 and denies stdlib calls that parse on 3.8 and then fail at +runtime. + +## Verify a release + +```sh +python3 -m bastion_agent.verify_release +``` + +Run it from a clone of this repository. It needs Python 3.8 or later, git, +network access, and PyYAML at the version `build.py` pins (the build vendors +it); no key or account of ours. It: + +1. fetches `privatae.ai/bastion-agent/version.json` and the signature beside it + (`version.json..sig`), and checks the ed25519 + signature against the release keys the bastion pins, the same check a + bastion makes before it installs an agent. The public keys are in + `bastion_agent/verify_release.py`, copied from `client-go/release_sign.go` + in the bastion's repository; +2. checks out the source of that version in a separate worktree: the tag + `v`, or else the commit the manifest names as `built_from` if the + clone has it; +3. rebuilds the `.pyz` with that version's own `build.py`, and compares its + sha256 with every `.pyz` the manifest lists. + +A match means the file every bastion downloads is exactly what this source +produces, and the signature means the release key holder published it. The +exit code is 0 only when both hold. + +The public repository is a copy of each release's source, not the development +history, so `built_from` names a commit that exists only in the development +repository. From this release on, each public snapshot carries the tag +`v`, which marks the source the release was built from. A snapshot +published before that has no tag, and can be checked from `built_from` only +where that commit is available. A manifest from before signing has no +signature file beside it, and the check refuses it. + +## Security and contributing + +Please do not open a public issue for a security problem: see +[SECURITY.md](SECURITY.md) for how to report one and what is in scope. +[CONTRIBUTING.md](CONTRIBUTING.md) lists the checks a change must pass and the +rules it must keep. [CHANGELOG.md](CHANGELOG.md) says what changed in each +version. + +## Where this came from + +The two packages were extracted from a larger repository they started in. The +bastion itself is a separate program in its own repository, and stays that way: +two programs, both open, kept separate on purpose. + +## The name + +The program is the **bastion agent**; this repository is named for it. It +was first called freeagent. The package (`bastion_agent`), the artifact +(`bastion-agent-.pyz`) and the published address +(`privatae.ai/bastion-agent/`) carry the new name. The legacy address +`privatae.ai/freeagent/` is retired and answers 404; a bastion older than +0.20.0, which knows only that address, keeps the agent it has +([docs/packaging.md](docs/packaging.md#the-legacy-name)). The environment +variables (`FREEAGENT_*`), the manifest's `freeagent: 1` field and the +bastion's heartbeat fields keep the old word as stable names: the bastion +sets and writes them, so renaming one would have to change the bastion and +the agent in step, for no change in behaviour. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..38bd642 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,84 @@ +# Security policy + +## Reporting a vulnerability + +Please do not open a public issue for a security problem. + +Email **info@privatae.ai** with: + +- the agent's version (`python3 bastion-agent-.pyz --version`) and + platform, +- what you found and how to reproduce it, +- what an attacker could do with it. + +You will get an acknowledgement within three business days. We will keep you +informed as the report is confirmed and fixed, and credit you in the release +notes if you wish. + +## What is in scope + +This repository: the bastion agent and the `.pyz` it is built into. The agent +reads a person's files, so anything that lets it read, keep or give out more +than its signed manifest and the person's grant allow is in scope. For +example: + +- **The walk** reading outside the manifest's roots (through a link, a + junction, a path trick), or descending into an excluded directory. +- **The extractors** (including the built-in PDF and Office readers) being + made to exhaust memory or time past their limits, to run a program other + than the named tools, or to write outside the output directory. +- **The sanitizer** (`pii/`) writing personal data it found into the sanitized + store, an index or a summary; a recall failure that does not stop the stage; + a value reachable from a token without the local vault; the vault readable + by another user. +- **The embedder and the summarizer** sending text that has not been sanitized + to a model. +- **The catalog and the sealed bundle** carrying a value, the extract store, + the vault or the vectors; a bundle readable by anyone but the owner and the + manifest's recipients. +- **`serve`** answering a client whose certificate does not chain to the + configured root, giving a grantee more than its grant or anything after its + expiry, returning plaintext instead of sanitized text, or continuing to serve + after the manifest stops verifying. +- **The manifest gate** accepting an unsigned, edited or foreign manifest, or a + certificate that does not chain to the configured root. +- **The artifact**: a `.pyz` that is not what this source builds, or a + weakness in `verify_release.py` that makes a mismatched build or a bad + signature pass. + +## What is out of scope + +- **The bastion**, which installs, updates and supervises the agent, signs its + manifest and holds its keys. It is a separate program with its own policy: + [privatae-bastion's SECURITY.md](https://git.privatae.ai/privatae/privatae-bastion/src/branch/main/SECURITY.md). + Report there, or to the same address. +- **The Privatae platform** the bastion connects to. It is separate software + with its own reporting path; reports about it sent to the address above are + forwarded. +- **What the design already says it does not do.** The rules-only `flag` mode + does not find names, and the recall pass is probabilistic + ([docs/pii-sterilization.md](docs/pii-sterilization.md#limits)); the extract + store in the output folder is plaintext by design + ([docs/runtime.md](docs/runtime.md#where-an-agent-lives-rootagent)); the + vault is not encrypted at rest. A report that shows one of these is worse + than documented is in scope. +- **Findings that need the owner's account already.** The agent runs as the + person whose folder it reads; someone who can act as that person can read + the folder without it. + +## Supported versions + +Only the current release receives fixes: the version + names. While a folder is +shared, the bastion checks that signed manifest every ten minutes and installs +any newer version, so a fixed release reaches every machine running the agent +without anyone installing it. `bastion_agent/VERSION` on `main` may be ahead +of the published version; see [CHANGELOG.md](CHANGELOG.md). + +## Checking what you run + +Every release manifest is signed with the bastion's release key (the bastion's +SECURITY.md says how that key is held and what trusting it means), and the +build is reproducible. `python3 -m bastion_agent.verify_release` checks the +signature against the pinned keys, rebuilds the published version from this +source, and compares the bytes; see the README. diff --git a/bastion_agent/VERSION b/bastion_agent/VERSION new file mode 100644 index 0000000..71d6a66 --- /dev/null +++ b/bastion_agent/VERSION @@ -0,0 +1 @@ +0.1.14 diff --git a/bastion_agent/__init__.py b/bastion_agent/__init__.py new file mode 100644 index 0000000..dee463b --- /dev/null +++ b/bastion_agent/__init__.py @@ -0,0 +1,12 @@ +from .manifest import ( # noqa: F401 + STAGES, + MASK_OPS, + SWEEPS, + GRANT_OPS, + Manifest, + ManifestError, + load_manifest, + signing_digest, + signing_payload, + validate_document, +) diff --git a/bastion_agent/__main__.py b/bastion_agent/__main__.py new file mode 100644 index 0000000..3c11efc --- /dev/null +++ b/bastion_agent/__main__.py @@ -0,0 +1,367 @@ +"""CLI: python3 -m bastion_agent {validate,show,payload,walk} [...]""" + +import argparse +import os +import sys + +from . import doctor as _doctor +from .catalog import CatalogError, catalog +from .embed import EmbedError, chunk, embed, search +from .extract import extract +from .manifest import ManifestError, load_manifest, signing_digest, signing_payload +from .run import GateError, RunError, default_out, sign_manifest_file, sweep, write_signature +from .sanitize import SanitizeError, sanitize +from .seal import SealError, seal, unseal +from .serve import ServeError, serve +from .walk import walk + + +def _show(m, path): + print(f"{path}: {m.id} (owner {m.owner_host})") + print(f" roots: {', '.join(m.scope.roots)}") + if m.scope.exclude: + print(f" exclude: {', '.join(m.scope.exclude)}") + print(f" mask: {' '.join(m.mask)}") + print(f" stages: {' -> '.join(m.stages)}") + if m.sanitize: + print(f" sanitize: {m.sanitize.mode} (scope {m.sanitize.scope}" + f"{', verify' if m.sanitize.verify else ''})") + if m.embed: + print(f" embed: {m.embed.model} {m.embed.dim}d, chunk " + f"{m.embed.chunk_size_tokens}/{m.embed.chunk_overlap_tokens}") + if m.summarize: + print(f" summarize: {m.summarize.model} per {', '.join(m.summarize.per)}") + print(f" sweep: {m.sweep}") + print(f" sink: catalog={m.sink.catalog} vectors={m.sink.vectors} " + f"plaintext={m.sink.plaintext}" + f"{' seal_to=' + ','.join(m.sink.seal_to) if m.sink.seal_to else ''}") + if m.grants: + for g in m.grants: + print(f" grant: {g.host} may {', '.join(g.ops)} until {g.until.isoformat()}") + else: + print(" grants: none") + print(f" signed: {'yes (' + m.signature.alg + ')' if m.signed else 'NO — runtime will refuse'}") + print(f" digest: sha256:{signing_digest(m.raw)}") + + +def _version(): + # pkgutil reads from a zipapp as well as a checkout; a plain open() cannot + import pkgutil + try: + data = pkgutil.get_data(__package__ or "bastion_agent", "VERSION") + return data.decode("utf-8").strip() if data else "0.0.0" + except (OSError, ValueError): + return "0.0.0" + + +def main(): + parser = argparse.ArgumentParser(prog="bastion-agent") + parser.add_argument("--version", action="version", version="bastion-agent %s" % _version()) + sub = parser.add_subparsers(dest="cmd", required=True) + p_doc = sub.add_parser("doctor", help="preflight: python, tools, endpoints, environment") + p_doc.add_argument("--json", action="store_true") + p_val = sub.add_parser("validate", help="validate manifest(s), exit 1 on any error") + p_val.add_argument("--require-signed", action="store_true", + help="also fail when the signature block is absent") + p_val.add_argument("paths", nargs="+") + p_show = sub.add_parser("show", help="print a manifest's effective plan") + p_show.add_argument("paths", nargs="+") + p_pay = sub.add_parser("payload", help="emit the canonical signing payload (bytes) to stdout") + p_pay.add_argument("path") + p_walk = sub.add_parser("walk", help="run the walk stage: filesystem + metadata map") + p_walk.add_argument("path") + p_walk.add_argument("--out", help="output directory (default /.agent, or build/freeagent/ for several roots)") + p_walk.add_argument("--no-hash", action="store_true", + help="metadata only; skip SHA-256 (fast survey)") + p_walk.add_argument("--require-signed", action="store_true", + help="refuse an unsigned manifest, as the runtime will") + p_ext = sub.add_parser("extract", help="run the extract stage: text out of walked files") + p_ext.add_argument("path") + p_ext.add_argument("--out", help="output directory (default /.agent, or build/freeagent/ for several roots)") + p_ext.add_argument("--no-external", action="store_true", + help="built-in extractors only; ignore pdftotext/tesseract on PATH") + p_ext.add_argument("--require-signed", action="store_true") + p_san = sub.add_parser("sanitize", help="run the sanitize stage: PII out of the extracted text") + p_san.add_argument("path") + p_san.add_argument("--out", help="output directory (default /.agent, or build/freeagent/ for several roots)") + p_san.add_argument("--require-signed", action="store_true") + for name, helptext in (("chunk", "run the chunk stage: sanitized text -> overlapping windows"), + ("embed", "run the embed stage: chunks -> local vectors.db")): + p = sub.add_parser(name, help=helptext) + p.add_argument("path") + p.add_argument("--out") + p.add_argument("--require-signed", action="store_true") + p_srch = sub.add_parser("search", help="query the local vectors (proves the lane end to end)") + p_srch.add_argument("path") + p_srch.add_argument("query") + p_srch.add_argument("--out") + p_srch.add_argument("-k", type=int, default=5) + p_cat = sub.add_parser("catalog", help="run the catalog stage: file + directory cards") + p_cat.add_argument("path") + p_cat.add_argument("--out") + p_cat.add_argument("--require-signed", action="store_true") + p_seal = sub.add_parser("seal", help="run the seal stage: catalog -> age envelope for the owner") + p_seal.add_argument("path") + p_seal.add_argument("--out") + p_seal.add_argument("--require-signed", action="store_true") + p_unseal = sub.add_parser("unseal", help="owner-side: open a sealed bundle with an age identity") + p_unseal.add_argument("sealed") + p_unseal.add_argument("--identity", required=True, help="age identity file (age-keygen output)") + p_unseal.add_argument("--dest", required=True) + p_sw = sub.add_parser("sweep", help="the runtime: gate the manifest, run its stages on its cadence") + p_sw.add_argument("path") + p_sw.add_argument("--out", help="output directory (default /.agent, or build/freeagent/ " + "for several roots; the same folder serve uses)") + p_sw.add_argument("--once", action="store_true", help="one run, then exit (any cadence)") + p_sw.add_argument("--allow-unsigned", action="store_true", + help="DEVELOPMENT ONLY: run an unsigned or unverifiable manifest; recorded in run.jsonl") + p_sign = sub.add_parser("sign", help="write the signature block into a manifest (host-side helper)") + p_sign.add_argument("path") + p_sign_by = p_sign.add_mutually_exclusive_group(required=True) + p_sign_by.add_argument("--key", help="PEM private key (signs with openssl)") + p_sign_by.add_argument("--value", help="a base64 DER signature over `payload`, made " + "elsewhere (the bastion signs with its own key)") + p_sign.add_argument("--cert", required=True, help="cert path to record in the manifest") + p_srv = sub.add_parser("serve", help="the query endpoint: search/read_range over mTLS for owner + grantees") + p_srv.add_argument("path") + p_srv.add_argument("--out") + p_srv.add_argument("--bind", help="host:port or [v6]:port (default FREEAGENT_BIND or [::]:8470)") + p_srv.add_argument("--allow-unsigned", action="store_true", + help="DEVELOPMENT ONLY: serve an unsigned/unverifiable manifest") + p_srv.add_argument("--port-file", help="write the bound port here (for a supervisor binding port 0)") + args = parser.parse_args() + + if args.cmd == "doctor": + return _doctor.main(["--json"] if args.json else []) + + if args.cmd == "serve": + try: + m = load_manifest(args.path) + except ManifestError as exc: + print(exc, file=sys.stderr) + return 1 + out = args.out or default_out(m) + try: + serve(args.path, out, allow_unsigned=args.allow_unsigned, bind=args.bind, + port_file=args.port_file) + except ServeError as exc: + print("serve: %s" % exc, file=sys.stderr) + return 1 + except KeyboardInterrupt: + pass + return 0 + + if args.cmd == "sign": + try: + if args.value: + import base64 + try: + der = base64.b64decode(args.value, validate=True) + except Exception: + print("sign: --value is not base64", file=sys.stderr) + return 1 + write_signature(args.path, der, args.cert) + else: + sign_manifest_file(args.path, args.key, args.cert) + except (GateError, ManifestError) as exc: + print("sign: %s" % exc, file=sys.stderr) + return 1 + print("%s: signed (cert %s)" % (args.path, args.cert)) + return 0 + + if args.cmd == "sweep": + try: + last = sweep(args.path, args.out, allow_unsigned=args.allow_unsigned, once=args.once) + except RunError as exc: + print("sweep: %s" % exc, file=sys.stderr) + return 1 + return 0 if (last and last.get("ok")) else 1 + + if args.cmd == "unseal": + try: + inv = unseal(args.sealed, args.identity, args.dest) + except SealError as exc: + print("unseal: %s" % exc, file=sys.stderr) + return 1 + print("%s: %d members verified -> %s (manifest %s, sealed %s)" + % (args.sealed, len(inv["members"]), args.dest, inv["manifest_id"], inv["sealed_at"])) + return 0 + + if args.cmd == "seal": + try: + m = load_manifest(args.path, require_signed=args.require_signed) + except ManifestError as exc: + print(exc, file=sys.stderr) + return 1 + out = args.out or default_out(m) + try: + s = seal(m, out) + except SealError as exc: + print("seal: %s" % exc, file=sys.stderr) + return 1 + print("%s: sealed %d members (%d bytes -> %d) to %d recipient(s)%s -> %s" + % (m.id, len(s["members"]), s["bytes_plain"], s["bytes_sealed"], + len(s["recipients"]), " + signature" if s["signature"] else "", + os.path.join(out, s["file"]))) + return 0 + + if args.cmd == "catalog": + try: + m = load_manifest(args.path, require_signed=args.require_signed) + except ManifestError as exc: + print(exc, file=sys.stderr) + return 1 + if "catalog" not in m.stages: + print("%s: 'catalog' is not in this manifest's stages" % args.path, file=sys.stderr) + return 1 + out = args.out or default_out(m) + try: + s = catalog(m, out) + except (FileNotFoundError, CatalogError) as exc: + print("catalog: %s" % exc, file=sys.stderr) + return 1 + c = s["counts"] + print("%s: %d file cards (%d with text), %d directory cards, %d names masked; " + "summaries %d new, %d reused [%s]; catalog %d bytes -> %s" + % (m.id, c["files"], c["with_text"], c["dirs"], c["names_masked"], + c["summaries"], c["summaries_reused"], s["summarizer"] or "none", + s["bytes_catalog"], out)) + if s["stopped"]: + print("catalog: summarizer STOPPED: %s" % s["stopped"], file=sys.stderr) + return 1 + return 0 + + if args.cmd in ("chunk", "embed", "search"): + try: + m = load_manifest(args.path, require_signed=getattr(args, "require_signed", False)) + except ManifestError as exc: + print(exc, file=sys.stderr) + return 1 + stage = "embed" if args.cmd == "search" else args.cmd + if stage not in m.stages: + print("%s: %r is not in this manifest's stages" % (args.path, stage), file=sys.stderr) + return 1 + out = args.out or default_out(m) + try: + if args.cmd == "chunk": + s = chunk(m, out) + c = s["counts"] + print("%s: %d docs -> %d chunks (%d/%d tokens), %d skipped -> %s" + % (m.id, c["docs"], c["chunks"], s["size_tokens"], s["overlap_tokens"], + c["skipped"], out)) + return 0 + if args.cmd == "embed": + s = embed(m, out) + c = s["counts"] + print("%s: %d chunks in %d docs -> %d embedded, %d reused [%s, %dd]; " + "%d stale vectors kept; store %s" + % (m.id, c["chunks"], c["docs"], c["embedded"], c["reused"], s["endpoint"], + s["dim"], s["stale_vectors"], s["store"])) + return 0 + hits = search(m, out, args.query, k=args.k) + for h in hits: + snippet = h["text"].replace("\n", " ")[:160] + print("%.4f %s#%d %s" % (h["score"], h["doc"][:12], h["idx"], snippet)) + if not hits: + print("no vectors for model %s" % m.embed.model, file=sys.stderr) + return 0 + except (FileNotFoundError, EmbedError) as exc: + print("%s: %s" % (args.cmd, exc), file=sys.stderr) + return 1 + + if args.cmd == "sanitize": + try: + m = load_manifest(args.path, require_signed=args.require_signed) + except ManifestError as exc: + print(exc, file=sys.stderr) + return 1 + if "sanitize" not in m.stages: + print("%s: 'sanitize' is not in this manifest's stages" % args.path, file=sys.stderr) + return 1 + out = args.out or default_out(m) + try: + s = sanitize(m, out) + except (FileNotFoundError, SanitizeError) as exc: + print("sanitize: %s" % exc, file=sys.stderr) + return 1 + c = s["counts"] + types = ", ".join("%s=%d" % kv for kv in sorted(s["types"].items())) or "none" + print("%s: %d files -> %d sanitized, %d reused, %d skipped, %d errors, %d not run " + "[%s, scope %s]; tokens: %s -> %s" + % (m.id, c["files"], c["ok"], c["reused"], c["skipped"], c["error"], c["not_run"], + s["sterilizer"], s["scope"], types, out)) + if s["stopped"]: + print("sanitize STOPPED (fail closed): %s" % s["stopped"], file=sys.stderr) + return 1 + return 2 if c["error"] else 0 + + if args.cmd == "extract": + try: + m = load_manifest(args.path, require_signed=args.require_signed) + except ManifestError as exc: + print(exc, file=sys.stderr) + return 1 + if "extract" not in m.stages: + print("%s: 'extract' is not in this manifest's stages" % args.path, file=sys.stderr) + return 1 + out = args.out or default_out(m) + try: + s = extract(m, out, use_external=not args.no_external) + except FileNotFoundError as exc: + print(exc, file=sys.stderr) + return 1 + c = s["counts"] + tools = ", ".join(k for k, v in s["tools"].items() if v) or "none" + print("%s: %d files -> %d extracted (%d chars), %d reused, %d empty, %d unsupported, " + "%d skipped, %d errors; tools: %s -> %s" + % (m.id, c["files"], c["ok"], s["chars"], c["reused"], c["empty"], + c["unsupported"], c["skipped"], c["error"], tools, out)) + return 2 if c["error"] else 0 + + if args.cmd == "walk": + try: + m = load_manifest(args.path, require_signed=args.require_signed) + except ManifestError as exc: + print(exc, file=sys.stderr) + return 1 + out = args.out or default_out(m) + s = walk(m, out, do_hash=not args.no_hash) + c = s["counts"] + print("%s: walked %d entries, %d files (%d bytes), %d dirs, %d links; " + "%d hashed, %d reused, %d too large, %d excluded, %d errors -> %s" + % (m.id, s["entries"], c["files"], s["bytes"], c["dirs"], c["symlinks"], + c["hashed"], c["hash_reused"], c["too_large"], c["excluded"], + c["errors"], out)) + for r in s["roots"]: + if r.get("error"): + print(" root %d %s: %s" % (r["index"], r["path"], r["error"]), file=sys.stderr) + return 2 if c["errors"] else 0 + + if args.cmd == "payload": + try: + m = load_manifest(args.path) + except ManifestError as exc: + print(exc, file=sys.stderr) + return 1 + sys.stdout.buffer.write(signing_payload(m.raw)) + return 0 + + rc = 0 + for path in args.paths: + try: + m = load_manifest(path, require_signed=getattr(args, "require_signed", False)) + except ManifestError as exc: + print(f"{path}:", file=sys.stderr) + print(exc, file=sys.stderr) + rc = 1 + continue + if args.cmd == "validate": + print(f"{path}: OK — {m.id}, {len(m.stages)} stages, {len(m.grants)} grants, " + f"{'signed' if m.signed else 'unsigned'}") + else: + _show(m, path) + return rc + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bastion_agent/bastion-agent.service b/bastion_agent/bastion-agent.service new file mode 100644 index 0000000..97729fb --- /dev/null +++ b/bastion_agent/bastion-agent.service @@ -0,0 +1,34 @@ +# bastion agent sweep — systemd USER unit (Linux). Runs the manifest's stages on +# its cadence under the owner's account; the corpus never leaves that user's +# permissions. +# +# mkdir -p ~/.config/systemd/user ~/.config/bastion-agent +# cp bastion-agent/bastion-agent.service ~/.config/systemd/user/ +# # environment (never in the manifest): recipients, endpoints, tools +# printf 'FREEAGENT_OWNER_RECIPIENT=age1...\nEMBED_URL=http://127.0.0.1:11434\n' > ~/.config/bastion-agent/env +# chmod 600 ~/.config/bastion-agent/env +# systemctl --user daemon-reload && systemctl --user enable --now bastion-agent +# journalctl --user -u bastion-agent -f +# +# Mac: a launchd agent with the same ExecStart (KeepAlive true); Windows: a +# Scheduled Task "At log on" running the same command. Neither file is in +# this repository. +[Unit] +Description=bastion agent sweep (%h/.config/bastion-agent/manifest.yaml) +After=network-online.target + +[Service] +Type=simple +WorkingDirectory=%h/.local/share/bastion-agent +EnvironmentFile=%h/.config/bastion-agent/env +# the repo checkout (or the packaged binary later); adjust to where it lives +ExecStart=/usr/bin/python3 -m bastion_agent sweep %h/.config/bastion-agent/manifest.yaml --out %h/.local/share/bastion-agent/out +Restart=on-failure +RestartSec=60 +# the vault, stores and envelope are 0600 already; keep the unit from widening them +UMask=0077 +NoNewPrivileges=true +PrivateTmp=true + +[Install] +WantedBy=default.target diff --git a/bastion_agent/build.py b/bastion_agent/build.py new file mode 100644 index 0000000..39971c2 --- /dev/null +++ b/bastion_agent/build.py @@ -0,0 +1,142 @@ +"""Build the bastion agent distributable: one reproducible zipapp. + + python3 -m bastion_agent.build [--out dist] + +Produces dist/bastion-agent-.pyz and dist/bastion-agent-.pyz.sha256. +The .pyz runs on any Python >= 3.8 on Mac, Windows or Linux: + + python3 bastion-agent-0.1.0.pyz doctor + python3 bastion-agent-0.1.0.pyz sweep manifest.yaml + +Inside: the `bastion_agent` package (no tests), the `pii` package (no tests, +no Dockerfile), and a vendored pure-Python PyYAML (MIT; the one import +the manifest loader needs that is not stdlib — the C accelerator is left +out on purpose so the same file runs everywhere). age and openssl stay +what they are: single external binaries `doctor` checks for. The repo's +`LICENSE` (GNU GPL v3 or later) is archived at the top level, so the +license travels with the program as GPL-3 sections 4 and 5 require. + +Reproducible: fixed timestamps, sorted members, no bytecode, and the +vendored PyYAML pinned to one release (PYYAML_VERSION). Two builds of the +same tree are byte-identical on any machine, so the .sha256 means something +and anyone can rebuild a release and compare. +A native per-OS binary (PyInstaller/Nuitka) needs a build machine per OS +and is not produced here; this is the artifact until then. +""" + +import argparse +import hashlib +import io +import os +import sys +import zipfile + +HERE = os.path.dirname(os.path.abspath(__file__)) +TOOLS = os.path.dirname(HERE) +FIXED_TIME = (2026, 1, 1, 0, 0, 0) +SHEBANG = b"#!/usr/bin/env python3\n" + +MAIN_SHIM = '''"""bastion agent zipapp entry point.""" +import sys +from bastion_agent.__main__ import main +sys.exit(main()) +''' + +YAML_NOTICE = """PyYAML (vendored, pure-Python modules only) +License: MIT — Copyright (c) 2017-2021 Ingy döt Net, Copyright (c) 2006-2016 Kirill Simonov +https://github.com/yaml/pyyaml — the C accelerator (_yaml) is deliberately not included. +""" + + +def version(): + with open(os.path.join(HERE, "VERSION"), encoding="utf-8") as fh: + return fh.read().strip() + + +def _package_files(src_dir, exclude_prefixes=("test_",), exclude_names=()): + for name in sorted(os.listdir(src_dir)): + if not name.endswith(".py") or name.startswith(exclude_prefixes) or name in exclude_names: + continue + with open(os.path.join(src_dir, name), "rb") as fh: + yield name, fh.read() + + +# The PyYAML release that is vendored. Its source is copied in from whatever +# `import yaml` finds, so without a pin two machines with different PyYAML +# builds would make different .pyz files from the same tree. Move it on +# purpose, in its own commit. +PYYAML_VERSION = "6.0.3" + + +def _yaml_files(): + import yaml + if getattr(yaml, "__version__", "") != PYYAML_VERSION: + raise SystemExit( + "build needs PyYAML %s to vendor (found %s at %s): pip install pyyaml==%s" + % (PYYAML_VERSION, getattr(yaml, "__version__", "?"), yaml.__file__, PYYAML_VERSION)) + d = os.path.dirname(yaml.__file__) + for name in sorted(os.listdir(d)): + if name.endswith(".py"): + with open(os.path.join(d, name), "rb") as fh: + yield name, fh.read() + + +def members(): + """(arcname, bytes) for everything in the zipapp, deterministic order.""" + out = [("__main__.py", MAIN_SHIM.encode("utf-8"))] + # build.py and verify_release.py are tools for making and checking a + # release; neither is part of the program a bastion runs. + for name, data in _package_files(HERE, exclude_names=("build.py", "verify_release.py")): + out.append(("bastion_agent/" + name, data)) + with open(os.path.join(HERE, "VERSION"), "rb") as fh: + out.append(("bastion_agent/VERSION", fh.read())) + for name, data in _package_files(os.path.join(TOOLS, "pii")): + out.append(("pii/" + name, data)) + for name, data in _yaml_files(): + out.append(("yaml/" + name, data)) + out.append(("yaml/LICENSE-NOTICE.txt", YAML_NOTICE.encode("utf-8"))) + with open(os.path.join(TOOLS, "LICENSE"), "rb") as fh: + out.append(("LICENSE", fh.read())) + return sorted(out, key=lambda m: m[0]) + + +def build_bytes(): + buf = io.BytesIO() + buf.write(SHEBANG) + with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf: + for arcname, data in members(): + info = zipfile.ZipInfo(arcname, date_time=FIXED_TIME) + info.compress_type = zipfile.ZIP_DEFLATED + info.external_attr = 0o644 << 16 + zf.writestr(info, data) + return buf.getvalue() + + +def build(out_dir="dist"): + os.makedirs(out_dir, exist_ok=True) + data = build_bytes() + ver = version() + path = os.path.join(out_dir, "bastion-agent-%s.pyz" % ver) + with open(path, "wb") as fh: + fh.write(data) + try: + os.chmod(path, 0o755) + except OSError: + pass + digest = hashlib.sha256(data).hexdigest() + with open(path + ".sha256", "w", encoding="utf-8") as fh: + fh.write("%s %s\n" % (digest, os.path.basename(path))) + return path, digest, len(data) + + +def main(argv=None): + ap = argparse.ArgumentParser(prog="bastion_agent.build") + ap.add_argument("--out", default="dist") + args = ap.parse_args(argv) + path, digest, size = build(args.out) + print("%s %d bytes sha256 %s" % (path, size, digest)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bastion_agent/catalog.py b/bastion_agent/catalog.py new file mode 100644 index 0000000..6398dfe --- /dev/null +++ b/bastion_agent/catalog.py @@ -0,0 +1,444 @@ +"""bastion agent `catalog` stage — the curated dataset. + +The catalog is what a host reasons over and what `seal` ships. One card +per file, one per directory, small and structured, built from: + + walk.jsonl metadata (path, size, mtime, mime, hash) + extract.jsonl whether text came out, and how + sanitize.jsonl entity counts (token types minted per file) + sanitize/.txt the SANITIZED text — the only text this stage reads + chunks.jsonl chunk count per document (optional) + +Outputs: + /catalog.jsonl file cards, sorted by (root, path) + /catalog.dirs.jsonl directory cards, sorted by (root, path) + /catalog.summary.json + +Every text-derived field (title, head, keywords, summary) comes from the +sanitized store, so the catalog carries PII tokens, never values. File +and directory NAMES are plaintext by nature; they go through the rules +pass and any precision anchor in a name (an email, a card number) is +masked as [PII_TYPE] in the card. Contextual PII in a filename +("Marisol_medical.pdf") is not caught by rules — the honest limit of the +sanitize stage applies here too, and the doc says so. + +The optional summarizer (manifest `summarize`) is a local chat model — a +sensor: it fills the `summary` string and nothing else, is reached via +SUMMARIZE_URL (never the manifest), sees only sanitized text, and is +cached per (document, model, version). The first failure stops +summarizing (cards still get written, `summary_error` set, exit 1). +""" + +import datetime as _dt +import hashlib +import json +import math +import os +import re +import tempfile +import urllib.error +import urllib.request + +from .manifest import signing_digest +from .sanitize import store_path as sanitized_store_path + + +def _rules_detect(): + """pii.detect.detect — a sibling package here and in the .pyz (see + sanitize.pii_modules for why the old `tools.pii` branch is gone).""" + from pii.detect import detect + return detect + +CATALOG_VERSION = 1 +HEAD_CHARS = 300 +TITLE_CHARS = 120 +KEYWORDS_PER_CARD = 8 +KEYWORDS_PER_DIR = 12 +SUMMARY_INPUT_CHARS = 12000 +SUMMARY_MAX_CHARS = 600 +DEFAULT_SUMMARIZE_URL = "http://127.0.0.1:11434" +SUMMARIZE_TIMEOUT = 300.0 + +WORD_RE = re.compile(r"[^\W\d_]{3,}", re.UNICODE) +TOKEN_MARK_RE = re.compile(r"\[PII_[A-Z_]+_\d+\]") +STOPWORDS = set(""" +the and for that with this from are was were will have has had not but you your our their they them +its his her she him who what which when where why how all any can could would should may might must +into onto over under than then there here also just more most much many some such very only own same +other another each both few between about after before during while above below off out too again +further once here per via use used using does did done being been each same so if or as at by on in +of to is it be an a we us me my i no yes do go get got let new old one two three first last next see +set run make made take took like well also back down up still even ever never always often sometimes +""".split()) + + +class CatalogError(Exception): + pass + + +def _atomic_write_text(path, text): + d = os.path.dirname(path) + os.makedirs(d, exist_ok=True) + fd, tmp = tempfile.mkstemp(prefix=".tmp-", dir=d) + try: + with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as fh: + fh.write(text) + os.replace(tmp, path) + except BaseException: + try: + os.unlink(tmp) + except OSError: + pass + raise + + +def _read_jsonl(out_dir, name, required=True): + path = os.path.join(out_dir, name) + if not os.path.exists(path): + if required: + raise FileNotFoundError("no %s at %s — run the earlier stages first" % (name, path)) + return [] + with open(path, encoding="utf-8") as fh: + return [json.loads(l) for l in fh] + + +# ── text facts ────────────────────────────────────────────────────────────── + +def title_of(text): + for line in text.splitlines(): + s = line.strip().lstrip("#*-= \t").strip() + if len(s) >= 3: + return s[:TITLE_CHARS] + return "" + + +def head_of(text): + s = re.sub(r"\s+", " ", text).strip() + return s[:HEAD_CHARS] + + +def term_counts(text): + counts = {} + text = TOKEN_MARK_RE.sub(" ", text) # [PII_*_n] placeholders are not keywords + for w in WORD_RE.findall(text.lower()): + if w in STOPWORDS: + continue + counts[w] = counts.get(w, 0) + 1 + return counts + + +def keywords(counts, df, ndocs, k): + """Top-k terms by TF-IDF; ties broken alphabetically for determinism.""" + scored = [] + for w, tf in counts.items(): + idf = math.log((1 + ndocs) / (1 + df.get(w, 0))) + 1.0 + scored.append((-(tf * idf), w)) + scored.sort() + return [w for _, w in scored[:k]] + + +def mask_name(name, detect): + """Rules pass over a file/dir name: precision anchors -> [PII_TYPE].""" + findings = detect(name) + if not findings: + return name + out = name + for f in sorted(findings, key=lambda f: -len(f.value)): + out = out.replace(f.value, "[PII_%s]" % f.type) + return out + + +def mask_path(path, detect): + """Every component of a relative path through mask_name — a parent + directory can carry an address as easily as a filename, and the file + card's path is what seal ships. Returns (masked + path, number of components changed).""" + if not path: + return path, 0 + out, changed = [], 0 + for seg in path.split("/"): + m = mask_name(seg, detect) if seg else seg + if m != seg: + changed += 1 + out.append(m) + return "/".join(out), changed + + +# ── the summarizer (a sensor) ─────────────────────────────────────────────── + +SUMMARY_PROMPT = ( + "Summarize the following document in at most three plain sentences for a catalog card: " + "what it is, what it covers, and who or what it concerns. Placeholders like [PII_PERSON_1] " + "stand for redacted personal data — keep them as they are, never guess what they hide. " + "Reply with the summary only.\n\n" +) +DIR_SUMMARY_PROMPT = ( + "These are catalog entries for the files in one folder. In at most three plain sentences " + "say what the folder is for and what kinds of material it holds. Placeholders like " + "[PII_PERSON_1] are redacted personal data — keep them as they are. Reply with the " + "summary only.\n\n" +) + + +class Summarizer: + def __init__(self, url, model, api=None, timeout=SUMMARIZE_TIMEOUT): + self.url = url.rstrip("/") + self.model = model + self.api = api + self.timeout = timeout + + def _post(self, path, body): + req = urllib.request.Request( + self.url + path, data=json.dumps(body).encode("utf-8"), + headers={"Content-Type": "application/json"}) + try: + with urllib.request.urlopen(req, timeout=self.timeout) as resp: + return json.load(resp), None + except urllib.error.HTTPError as e: + return None, "HTTP %d" % e.code + except (OSError, ValueError) as e: + return None, str(e) + + def _ollama(self, prompt): + data, err = self._post("/api/chat", { + "model": self.model, "stream": False, "options": {"temperature": 0}, + "messages": [{"role": "user", "content": prompt}]}) + if err: + return None, err + try: + return data["message"]["content"], None + except (KeyError, TypeError) as e: + return None, "malformed ollama response: %s" % e + + def _openai(self, prompt): + data, err = self._post("/v1/chat/completions", { + "model": self.model, "temperature": 0, + "messages": [{"role": "user", "content": prompt}]}) + if err: + return None, err + try: + return data["choices"][0]["message"]["content"], None + except (KeyError, IndexError, TypeError) as e: + return None, "malformed chat response: %s" % e + + def complete(self, prompt): + if self.api == "ollama": + text, err = self._ollama(prompt) + elif self.api == "openai": + text, err = self._openai(prompt) + else: + text, err = self._ollama(prompt) + if text is not None: + self.api = "ollama" + else: + text, err2 = self._openai(prompt) + if text is not None: + self.api = "openai" + else: + err = "ollama: %s; openai: %s" % (err, err2) + if text is None: + raise CatalogError("summarizer %s (model %s): %s" % (self.url, self.model, err)) + return clean_summary(text) + + def describe(self): + return "%s@%s(%s)" % (self.model, self.url, self.api or "auto") + + +def clean_summary(text): + text = re.sub(r".*?", "", text, flags=re.S) # thinking models + text = re.sub(r"\s+", " ", text).strip().strip('"') + return text[:SUMMARY_MAX_CHARS] + + +def build_summarizer(manifest, env): + if manifest.summarize is None: + return None + url = env.get("SUMMARIZE_URL", DEFAULT_SUMMARIZE_URL).strip() or DEFAULT_SUMMARIZE_URL + api = env.get("SUMMARIZE_API", "").strip().lower() or None + if api not in (None, "ollama", "openai"): + raise CatalogError("SUMMARIZE_API=%r is not ollama|openai" % api) + return Summarizer(url, manifest.summarize.model, api=api, + timeout=float(env.get("SUMMARIZE_TIMEOUT", SUMMARIZE_TIMEOUT))) + + +def _cache_path(out_dir): + return os.path.join(out_dir, "catalog.cache.json") + + +def _load_cache(out_dir): + try: + with open(_cache_path(out_dir), encoding="utf-8") as fh: + return json.load(fh) + except (OSError, ValueError): + return {} + + +# ── the stage ─────────────────────────────────────────────────────────────── + +def catalog(manifest, out_dir, env=None, summarizer=None, now=None): + env = dict(os.environ) if env is None else env + summ = summarizer if summarizer is not None else build_summarizer(manifest, env) + per = set(manifest.summarize.per) if manifest.summarize else set() + walk_rows = _read_jsonl(out_dir, "walk.jsonl") + extract_rows = {(r["root"], r["path"]): r for r in _read_jsonl(out_dir, "extract.jsonl")} + sanitize_rows = {(r["root"], r["path"]): r for r in _read_jsonl(out_dir, "sanitize.jsonl")} + chunk_counts = {} + for c in _read_jsonl(out_dir, "chunks.jsonl", required=False): + chunk_counts[c["doc"]] = chunk_counts.get(c["doc"], 0) + 1 + detect = _rules_detect() + cache = _load_cache(out_dir) + stamp = now or _dt.datetime.now(_dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + # pass 1: text facts per distinct sanitized document + texts = {} + doc_terms = {} + for key, s in sanitize_rows.items(): + if s.get("status") != "ok": + continue + sha = s["sha256"] + if sha in texts: + continue + with open(sanitized_store_path(out_dir, sha), encoding="utf-8") as fh: + texts[sha] = fh.read() + doc_terms[sha] = term_counts(texts[sha]) + df = {} + for counts in doc_terms.values(): + for w in counts: + df[w] = df.get(w, 0) + 1 + ndocs = len(doc_terms) + + counts = {"files": 0, "with_text": 0, "dirs": 0, "summaries": 0, "summaries_reused": 0, + "summary_errors": 0, "names_masked": 0} + stopped = None + summary_key = lambda sha, kind: "%s|%s|%s|v%d" % ( + kind, sha, manifest.summarize.model if manifest.summarize else "", CATALOG_VERSION) + + def summarize(kind, sha, prompt_body): + nonlocal stopped + if summ is None or stopped is not None: + return None, None + k = summary_key(sha, kind) + if k in cache: + counts["summaries_reused"] += 1 + return cache[k], None + try: + text = summ.complete((SUMMARY_PROMPT if kind == "file" else DIR_SUMMARY_PROMPT) + + prompt_body[:SUMMARY_INPUT_CHARS]) + except CatalogError as e: + stopped = str(e) + counts["summary_errors"] += 1 + return None, stopped + cache[k] = text + counts["summaries"] += 1 + return text, None + + # pass 2: file cards + cards = [] + dir_acc = {} + for w in walk_rows: + if w.get("kind") != "file": + continue + key = (w["root"], w["path"]) + counts["files"] += 1 + name = w["path"].rsplit("/", 1)[-1] + shown_path, changed = mask_path(w["path"], detect) + masked = shown_path.rsplit("/", 1)[-1] + counts["names_masked"] += changed + e = extract_rows.get(key, {}) + s = sanitize_rows.get(key, {}) + card = { + "root": w["root"], "path": shown_path, "name": masked, "kind": "file", + "size": w.get("size"), "mtime": w.get("mtime"), "mime": w.get("mime"), + "sha256": w.get("sha256"), "v": CATALOG_VERSION, + "text": e.get("status") if e else "not_extracted", + "extractor": e.get("extractor"), + "sanitized": s.get("status") if s else None, + } + sha = s.get("sha256") if s.get("status") == "ok" else None + if sha and sha in texts: + text = texts[sha] + counts["with_text"] += 1 + card.update({ + "title": title_of(text), + "head": head_of(text), + "chars": len(text), + "keywords": keywords(doc_terms[sha], df, ndocs, KEYWORDS_PER_CARD), + "entities": s.get("types", {}), + "chunks": chunk_counts.get(sha, 0), + }) + if "file" in per: + summary, err = summarize("file", sha, text) + card["summary"] = summary + if err: + card["summary_error"] = err + elif "file" in per: + card["summary"] = None # nothing to summarize + cards.append(card) + # directory aggregation: every ancestor, root included as "" + parts = w["path"].split("/")[:-1] + for depth in range(len(parts) + 1): + dpath = "/".join(parts[:depth]) + acc = dir_acc.setdefault((w["root"], dpath), { + "files": 0, "bytes": 0, "mimes": {}, "latest": "", "terms": {}, "entities": {}, + "children": [], "with_text": 0}) + acc["files"] += 1 + acc["bytes"] += w.get("size") or 0 + m = w.get("mime") or "?" + acc["mimes"][m] = acc["mimes"].get(m, 0) + 1 + if (w.get("mtime") or "") > acc["latest"]: + acc["latest"] = w["mtime"] + if sha and sha in doc_terms: + acc["with_text"] += 1 + for t, n in doc_terms[sha].items(): + acc["terms"][t] = acc["terms"].get(t, 0) + n + for t, n in (s.get("types") or {}).items(): + acc["entities"][t] = acc["entities"].get(t, 0) + n + if depth == len(parts): + acc["children"].append(card) + cards.sort(key=lambda c: (c["root"], c["path"])) + + # pass 3: directory cards + dcards = [] + for (root, dpath), acc in dir_acc.items(): + shown, _ = mask_path(dpath, detect) + masked = shown.rsplit("/", 1)[-1] if shown else "" + mimes = sorted(acc["mimes"].items(), key=lambda kv: (-kv[1], kv[0]))[:6] + card = { + "root": root, "path": shown, "name": masked, "kind": "dir", "v": CATALOG_VERSION, + "files": acc["files"], "with_text": acc["with_text"], "bytes": acc["bytes"], + "latest": acc["latest"], "types": dict(mimes), + "keywords": keywords(acc["terms"], df, ndocs, KEYWORDS_PER_DIR), + "entities": acc["entities"], + } + if "directory" in per: + body = "\n".join( + "- %s: %s" % (c["name"], (c.get("summary") or c.get("head") or "")[:200]) + for c in sorted(acc["children"], key=lambda c: c["path"])[:60]) + sha = hashlib.sha256( + ("%s|%s|" % (root, dpath)).encode("utf-8") + body.encode("utf-8")).hexdigest() + summary, err = summarize("dir", sha, body) + card["summary"] = summary + if err: + card["summary_error"] = err + dcards.append(card) + dcards.sort(key=lambda c: (c["root"], c["path"])) + counts["dirs"] = len(dcards) + + _atomic_write_text(os.path.join(out_dir, "catalog.jsonl"), "".join( + json.dumps(c, ensure_ascii=False, sort_keys=True) + "\n" for c in cards)) + _atomic_write_text(os.path.join(out_dir, "catalog.dirs.jsonl"), "".join( + json.dumps(c, ensure_ascii=False, sort_keys=True) + "\n" for c in dcards)) + _atomic_write_text(_cache_path(out_dir), json.dumps(cache, ensure_ascii=False, sort_keys=True)) + summary = { + "freeagent": 1, "stage": "catalog", "catalog_version": CATALOG_VERSION, + "manifest_id": manifest.id, + "manifest_digest": "sha256:" + signing_digest(manifest.raw), + "cataloged_at": stamp, + "summarizer": summ.describe() if summ else None, + "summarize_per": sorted(per), + "counts": counts, "stopped": stopped, + "bytes_catalog": sum(len(json.dumps(c, ensure_ascii=False)) + 1 for c in cards + dcards), + } + _atomic_write_text(os.path.join(out_dir, "catalog.summary.json"), + json.dumps(summary, ensure_ascii=False, indent=2, sort_keys=True) + "\n") + return summary diff --git a/bastion_agent/doctor.py b/bastion_agent/doctor.py new file mode 100644 index 0000000..f1d4eec --- /dev/null +++ b/bastion_agent/doctor.py @@ -0,0 +1,119 @@ +"""`bastion-agent doctor` — the preflight a member runs before the first sweep. + +Reports what this machine has and what it lacks, per stage, and never +changes anything. Required items are what a stage refuses to run +without; optional ones are the sensors that widen coverage (pdftotext, +tesseract, the gliner package) — their absence is a recorded gap in the +stage output, not a failure here. +""" + +import json +import os +import platform +import shutil +import sys +import urllib.request + +PY_FLOOR = (3, 8) + + +def _which(name): + return shutil.which(name) + + +def _probe_http(url, path, timeout=5): + """Reachable means the server answered — any HTTP status counts (a + proxy that only permits the embed path answers 403 on '/'). Only a + connection failure is 'not reachable'.""" + import urllib.error + try: + with urllib.request.urlopen(url.rstrip("/") + path, timeout=timeout) as r: + return True, "HTTP %d" % r.status + except urllib.error.HTTPError as e: + return True, "reachable (HTTP %d on %s)" % (e.code, path) + except Exception as e: + return False, str(e)[:120] + + +def checks(env=None): + env = dict(os.environ) if env is None else env + out = [] + + def add(name, ok, detail, required=True, stage=None): + out.append({"check": name, "ok": bool(ok), "detail": detail, "required": required, + "stage": stage}) + + v = sys.version_info[:3] + add("python >= %d.%d" % PY_FLOOR, v >= PY_FLOOR, "%s %s (%s)" % (platform.python_implementation(), + ".".join(map(str, v)), sys.executable)) + add("platform", True, "%s %s %s" % (platform.system(), platform.release(), platform.machine()), + required=False) + try: + import yaml # noqa: F401 + add("yaml (manifest loader)", True, "PyYAML %s" % getattr(yaml, "__version__", "?")) + except ImportError: + add("yaml (manifest loader)", False, "PyYAML not importable") + + for tool, stage, req, why in ( + ("age", "seal", True, "the envelope; https://age-encryption.org"), + # Not needed to RUN the agent: the gate verifies in the standard + # library (x509gate) and the bastion signs with its own key. + ("openssl", "sign/seal", False, "only for `sign --key` and SEAL_SIGN_KEY seal signatures"), + ("pdftotext", "extract", False, "PDF text (poppler); without it simple PDFs only"), + ("tesseract", "extract", False, "image OCR; without it images are recorded unsupported")): + p = _which(tool) + add(tool, bool(p), p or ("not on PATH — " + why), required=req, stage=stage) + + try: + import gliner # noqa: F401 + add("gliner package", True, "importable (PII_RECALL=gliner available)", required=False, stage="sanitize") + except ImportError: + add("gliner package", False, "not installed — sanitize sterilize on a laptop needs " + "PII_RECALL=gliner (pip install gliner) or a reachable LLM_URL / PII_ADDR", + required=False, stage="sanitize") + + embed_url = env.get("EMBED_URL", "http://127.0.0.1:11434") + ok, detail = _probe_http(embed_url, "/") + add("embedding endpoint %s" % embed_url, ok, detail, required=False, stage="embed") + if env.get("PII_ADDR"): + url = env["PII_ADDR"] if "://" in env["PII_ADDR"] else "http://" + env["PII_ADDR"] + ok, detail = _probe_http(url, "/health") + add("pii service %s" % url, ok, detail, required=False, stage="sanitize") + if env.get("SUMMARIZE_URL"): + ok, detail = _probe_http(env["SUMMARIZE_URL"], "/") + add("summarizer %s" % env["SUMMARIZE_URL"], ok, detail, required=False, stage="catalog") + + for var, stage, why in ( + ("FREEAGENT_OWNER_RECIPIENT", "seal", "owner's age public key"), + ("FREEAGENT_CA_ROOT", "sweep/serve", "CA root for the manifest gate and client certs"), + ("FREEAGENT_TLS_CERT", "serve", "this machine's cert"), + ("FREEAGENT_TLS_KEY", "serve", "this machine's key")): + val = env.get(var, "") + if var.endswith(("_ROOT", "_CERT", "_KEY")) and val: + ok, detail = os.path.exists(val), (val if os.path.exists(val) else val + " (missing file)") + else: + ok, detail = bool(val), (("set" if val else "unset — " + why)) + add(var, ok, detail, required=False, stage=stage) + return out + + +def render(results): + lines = [] + for r in results: + mark = "ok " if r["ok"] else ("MISS" if r["required"] else "opt ") + stage = (" [%s]" % r["stage"]) if r["stage"] else "" + lines.append("%s %-34s %s%s" % (mark, r["check"], r["detail"], stage)) + missing = [r for r in results if r["required"] and not r["ok"]] + lines.append("") + lines.append("required missing: %d" % len(missing) if missing else "all required checks pass") + return "\n".join(lines) + + +def main(argv=None, env=None): + argv = sys.argv[1:] if argv is None else argv + results = checks(env) + if "--json" in argv: + print(json.dumps(results, indent=2)) + else: + print(render(results)) + return 1 if any(r["required"] and not r["ok"] for r in results) else 0 diff --git a/bastion_agent/embed.py b/bastion_agent/embed.py new file mode 100644 index 0000000..5c85f57 --- /dev/null +++ b/bastion_agent/embed.py @@ -0,0 +1,417 @@ +"""bastion agent `chunk` + `embed` stages, and the local `search` that proves them. + +chunk sanitized store -> /chunks.jsonl + Overlapping windows sized in approximate tokens (manifest + embed.chunk.size_tokens / overlap_tokens). "Token" here is a + deterministic regex unit (a word or a punctuation mark), not a + model tokenizer: the stdlib has none, and the same text must chunk + identically on every machine. Chunk id = sha256(doc sha, window + text) — content-addressed, so an unchanged document keeps its + vectors; a changed document has a new sha and is embedded again. + +embed chunks -> /vectors.db (SQLite, stdlib) + The endpoint is resolved from the environment at run time + (EMBED_URL, default the local Ollama port), never the manifest; + the manifest pins the MODEL and DIM. Before the first real call + the stage embeds a probe and refuses to run if the served + dimension is not the manifest's — a wrong model never writes a + vector. Two wire formats, auto-detected: Ollama /api/embed and + OpenAI-style /v1/embeddings (llama.cpp server, TEI). + Rows are keyed (chunk_id, model): switching models ADDS rows and + never deletes the old ones, so going back to an earlier model + costs nothing and no vector is ever thrown away. + +search embed the query the same way, cosine over every vector of the + manifest's model, return the top-k sanitized chunks. Pure Python; + fine to tens of thousands of chunks, which is the point where a + real index is a follow-up. +""" + +import array +import datetime as _dt +import hashlib +import json +import math +import os +import re +import sqlite3 +import tempfile +import urllib.error +import urllib.request + +from .manifest import signing_digest +from .sanitize import store_path as sanitized_store_path + +CHUNK_VERSION = 1 +DEFAULT_EMBED_URL = "http://127.0.0.1:11434" # local Ollama; a sensor, resolved at run time +EMBED_BATCH = 32 +EMBED_TIMEOUT = 300.0 +PROBE_TEXT = "freeagent embedding probe" + +TOKEN_RE = re.compile(r"\w+|[^\w\s]", re.UNICODE) + + +class EmbedError(Exception): + """Fail-closed stop: the endpoint is unreachable or serves the wrong model.""" + + +# ── chunking ──────────────────────────────────────────────────────────────── + +def count_tokens(text): + return len(TOKEN_RE.findall(text)) + + +def chunk_text(text, size_tokens, overlap_tokens): + """Yield (start, end, window_text). Windows are cut at token + boundaries; the step is size - overlap tokens; the last window may be + shorter. Every character of text is inside at least one window.""" + spans = [(m.start(), m.end()) for m in TOKEN_RE.finditer(text)] + if not spans: + return + step = max(1, size_tokens - overlap_tokens) + i = 0 + n = len(spans) + while True: + j = min(n, i + size_tokens) + start = 0 if i == 0 else spans[i][0] + end = len(text) if j == n else spans[j - 1][1] + yield start, end, text[start:end] + if j >= n: + return + i += step + + +def chunk_id(doc_sha, window_text): + h = hashlib.sha256() + h.update(doc_sha.encode("ascii")) + h.update(b"\x00") + h.update(window_text.encode("utf-8")) + return h.hexdigest() + + +def _atomic_write_text(path, text): + d = os.path.dirname(path) + os.makedirs(d, exist_ok=True) + fd, tmp = tempfile.mkstemp(prefix=".tmp-", dir=d) + try: + with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as fh: + fh.write(text) + os.replace(tmp, path) + except BaseException: + try: + os.unlink(tmp) + except OSError: + pass + raise + + +def load_sanitized_index(out_dir): + path = os.path.join(out_dir, "sanitize.jsonl") + if not os.path.exists(path): + raise FileNotFoundError("no sanitize index at %s — run the sanitize stage first" % path) + with open(path, encoding="utf-8") as fh: + return [json.loads(l) for l in fh] + + +def chunk(manifest, out_dir, now=None): + cfg = manifest.embed + if cfg is None: + raise EmbedError("manifest has no embed section / stage") + records = load_sanitized_index(out_dir) + counts = {"docs": 0, "chunks": 0, "skipped": 0} + seen_docs = set() + rows = [] + for rec in records: + if rec.get("status") != "ok": + counts["skipped"] += 1 + continue + sha = rec["sha256"] + if sha in seen_docs: + continue # duplicate content chunks once; paths resolve through sanitize.jsonl + seen_docs.add(sha) + counts["docs"] += 1 + with open(sanitized_store_path(out_dir, sha), encoding="utf-8") as fh: + text = fh.read() + for idx, (start, end, window) in enumerate( + chunk_text(text, cfg.chunk_size_tokens, cfg.chunk_overlap_tokens)): + rows.append({"chunk_id": chunk_id(sha, window), "doc": sha, "idx": idx, + "start": start, "end": end, "chars": len(window), + "tokens": count_tokens(window), "v": CHUNK_VERSION}) + counts["chunks"] += 1 + rows.sort(key=lambda r: (r["doc"], r["idx"])) + _atomic_write_text(os.path.join(out_dir, "chunks.jsonl"), "".join( + json.dumps(r, sort_keys=True) + "\n" for r in rows)) + summary = { + "freeagent": 1, "stage": "chunk", "chunk_version": CHUNK_VERSION, + "manifest_id": manifest.id, + "manifest_digest": "sha256:" + signing_digest(manifest.raw), + "chunked_at": now or _dt.datetime.now(_dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "size_tokens": cfg.chunk_size_tokens, "overlap_tokens": cfg.chunk_overlap_tokens, + "counts": counts, + } + _atomic_write_text(os.path.join(out_dir, "chunk.summary.json"), + json.dumps(summary, indent=2, sort_keys=True) + "\n") + return summary + + +# ── the vector store ──────────────────────────────────────────────────────── + +_SCHEMA = """ +CREATE TABLE IF NOT EXISTS chunks ( + chunk_id TEXT PRIMARY KEY, + doc TEXT NOT NULL, + idx INTEGER NOT NULL, + start INTEGER NOT NULL, + end INTEGER NOT NULL, + text TEXT NOT NULL +); +CREATE INDEX IF NOT EXISTS chunks_doc ON chunks(doc, idx); +CREATE TABLE IF NOT EXISTS vectors ( + chunk_id TEXT NOT NULL, + model TEXT NOT NULL, + dim INTEGER NOT NULL, + vec BLOB NOT NULL, + created TEXT NOT NULL, + PRIMARY KEY (chunk_id, model) +); +CREATE INDEX IF NOT EXISTS vectors_model ON vectors(model); +""" + + +def vectors_path(out_dir): + return os.path.join(out_dir, "vectors.db") + + +def open_store(out_dir): + os.makedirs(out_dir, exist_ok=True) + conn = sqlite3.connect(vectors_path(out_dir), timeout=30) + conn.executescript(_SCHEMA) + return conn + + +def pack(vec): + a = array.array("f", vec) + return a.tobytes() + + +def unpack(blob): + a = array.array("f") + a.frombytes(blob) + return a + + +# ── the endpoint (a sensor) ───────────────────────────────────────────────── + +class Endpoint: + """Local embedding endpoint. `api` is 'ollama' or 'openai'; None + auto-detects on the probe.""" + + def __init__(self, url, model, api=None, timeout=EMBED_TIMEOUT): + self.url = url.rstrip("/") + self.model = model + self.api = api + self.timeout = timeout + + def _post(self, path, body): + req = urllib.request.Request( + self.url + path, data=json.dumps(body).encode("utf-8"), + headers={"Content-Type": "application/json"}) + try: + with urllib.request.urlopen(req, timeout=self.timeout) as resp: + return json.load(resp), None + except urllib.error.HTTPError as e: + detail = "" + try: + detail = e.read().decode("utf-8", "replace")[:200] + except Exception: + pass + return None, "HTTP %d %s" % (e.code, detail) + except (OSError, ValueError) as e: + return None, str(e) + + def _ollama(self, texts): + data, err = self._post("/api/embed", {"model": self.model, "input": texts}) + if err: + return None, err + try: + return [list(map(float, v)) for v in data["embeddings"]], None + except (KeyError, TypeError, ValueError) as e: + return None, "malformed ollama response: %s" % e + + def _openai(self, texts): + data, err = self._post("/v1/embeddings", {"model": self.model, "input": texts}) + if err: + return None, err + try: + items = sorted(data["data"], key=lambda d: d.get("index", 0)) + return [list(map(float, d["embedding"])) for d in items], None + except (KeyError, TypeError, ValueError) as e: + return None, "malformed embeddings response: %s" % e + + def embed(self, texts): + if self.api == "ollama": + vecs, err = self._ollama(texts) + elif self.api == "openai": + vecs, err = self._openai(texts) + else: + vecs, err = self._ollama(texts) + if vecs is not None: + self.api = "ollama" + else: + vecs, err2 = self._openai(texts) + if vecs is not None: + self.api = "openai" + else: + err = "ollama: %s; openai: %s" % (err, err2) + if vecs is None: + raise EmbedError("embedding endpoint %s (model %s): %s" % (self.url, self.model, err)) + if len(vecs) != len(texts): + raise EmbedError("endpoint returned %d vectors for %d inputs" % (len(vecs), len(texts))) + return vecs + + def probe(self, want_dim): + vec = self.embed([PROBE_TEXT])[0] + if len(vec) != want_dim: + raise EmbedError("model %s serves %d-dim vectors; manifest pins %d — refusing to write" + % (self.model, len(vec), want_dim)) + return len(vec) + + def describe(self): + return "%s@%s(%s)" % (self.model, self.url, self.api or "auto") + + +def build_endpoint(manifest, env): + url = env.get("EMBED_URL", DEFAULT_EMBED_URL).strip() or DEFAULT_EMBED_URL + api = env.get("EMBED_API", "").strip().lower() or None + if api not in (None, "ollama", "openai"): + raise EmbedError("EMBED_API=%r is not ollama|openai" % api) + return Endpoint(url, manifest.embed.model, api=api, + timeout=float(env.get("EMBED_TIMEOUT", EMBED_TIMEOUT))) + + +# ── embed stage ───────────────────────────────────────────────────────────── + +def load_chunks(out_dir): + path = os.path.join(out_dir, "chunks.jsonl") + if not os.path.exists(path): + raise FileNotFoundError("no chunks at %s — run the chunk stage first" % path) + with open(path, encoding="utf-8") as fh: + return [json.loads(l) for l in fh] + + +def embed(manifest, out_dir, env=None, endpoint=None, now=None, batch=None): + cfg = manifest.embed + if cfg is None: + raise EmbedError("manifest has no embed section / stage") + env = dict(os.environ) if env is None else env + ep = endpoint or build_endpoint(manifest, env) + batch = batch or int(env.get("EMBED_BATCH", EMBED_BATCH)) + chunks = load_chunks(out_dir) + ep.probe(cfg.dim) + + conn = open_store(out_dir) + counts = {"chunks": len(chunks), "unique": 0, "embedded": 0, "reused": 0, "docs": 0} + try: + have = {r[0] for r in conn.execute("SELECT chunk_id FROM vectors WHERE model=?", (cfg.model,))} + # chunk texts come from the sanitized store, sliced by recorded offsets + texts = {} + docs = {} + for c in chunks: + docs.setdefault(c["doc"], []).append(c) + counts["docs"] = len(docs) + for doc, cs in docs.items(): + with open(sanitized_store_path(out_dir, doc), encoding="utf-8") as fh: + text = fh.read() + for c in cs: + texts[c["chunk_id"]] = text[c["start"]:c["end"]] + # `chunks` is derived from THIS sweep (rebuilt each run); `vectors` + # is the durable store and is only ever added to. + with conn: + conn.execute("DELETE FROM chunks") + conn.executemany( + "INSERT OR REPLACE INTO chunks(chunk_id, doc, idx, start, end, text) VALUES (?,?,?,?,?,?)", + [(c["chunk_id"], c["doc"], c["idx"], c["start"], c["end"], texts[c["chunk_id"]]) + for c in chunks]) + # identical windows (a repeated paragraph) share a chunk id: embed once + unique = {} + for c in chunks: + unique.setdefault(c["chunk_id"], c) + counts["unique"] = len(unique) + todo = [c for c in unique.values() if c["chunk_id"] not in have] + counts["reused"] = len(unique) - len(todo) + stamp = now or _dt.datetime.now(_dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + for i in range(0, len(todo), batch): + group = todo[i:i + batch] + vecs = ep.embed([texts[c["chunk_id"]] for c in group]) + rows = [] + for c, v in zip(group, vecs): + if len(v) != cfg.dim: + raise EmbedError("vector of dim %d for chunk %s; manifest pins %d" + % (len(v), c["chunk_id"][:12], cfg.dim)) + rows.append((c["chunk_id"], cfg.model, cfg.dim, pack(v), stamp)) + with conn: + conn.executemany( + "INSERT OR REPLACE INTO vectors(chunk_id, model, dim, vec, created) VALUES (?,?,?,?,?)", + rows) + counts["embedded"] += len(rows) + # vectors for chunks that no longer exist in this sweep are kept + # (never dump); report how many so the host can see the drift + stale = conn.execute( + "SELECT COUNT(*) FROM vectors v WHERE v.model=? AND NOT EXISTS " + "(SELECT 1 FROM chunks c WHERE c.chunk_id=v.chunk_id)", (cfg.model,)).fetchone()[0] + models = [dict(model=m, dim=d, vectors=n) for m, d, n in conn.execute( + "SELECT model, dim, COUNT(*) FROM vectors GROUP BY model, dim ORDER BY model")] + finally: + conn.close() + summary = { + "freeagent": 1, "stage": "embed", + "manifest_id": manifest.id, + "manifest_digest": "sha256:" + signing_digest(manifest.raw), + "embedded_at": now or _dt.datetime.now(_dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "endpoint": ep.describe(), "model": cfg.model, "dim": cfg.dim, + "counts": counts, "stale_vectors": stale, "models_in_store": models, + "store": vectors_path(out_dir), + } + _atomic_write_text(os.path.join(out_dir, "embed.summary.json"), + json.dumps(summary, indent=2, sort_keys=True) + "\n") + return summary + + +# ── search ────────────────────────────────────────────────────────────────── + +def cosine(a, b): + dot = na = nb = 0.0 + for x, y in zip(a, b): + dot += x * y + na += x * x + nb += y * y + if na == 0.0 or nb == 0.0: + return 0.0 + return dot / math.sqrt(na * nb) + + +def search(manifest, out_dir, query, k=5, env=None, endpoint=None): + cfg = manifest.embed + if cfg is None: + raise EmbedError("manifest has no embed section") + env = dict(os.environ) if env is None else env + ep = endpoint or build_endpoint(manifest, env) + qv = ep.embed([query])[0] + if len(qv) != cfg.dim: + raise EmbedError("query vector dim %d != manifest dim %d" % (len(qv), cfg.dim)) + conn = open_store(out_dir) + try: + scored = [] + for chunk_id_, blob in conn.execute("SELECT chunk_id, vec FROM vectors WHERE model=?", (cfg.model,)): + scored.append((cosine(qv, unpack(blob)), chunk_id_)) + scored.sort(reverse=True) + out = [] + for score, cid in scored[:k]: + row = conn.execute("SELECT doc, idx, start, end, text FROM chunks WHERE chunk_id=?", (cid,)).fetchone() + if row is None: + continue + out.append({"score": round(score, 6), "chunk_id": cid, "doc": row[0], "idx": row[1], + "start": row[2], "end": row[3], "text": row[4]}) + return out + finally: + conn.close() diff --git a/bastion_agent/extract.py b/bastion_agent/extract.py new file mode 100644 index 0000000..29b9e6e --- /dev/null +++ b/bastion_agent/extract.py @@ -0,0 +1,768 @@ +"""bastion agent `extract` stage — text out of the files the walk found. + +Input is the walk map (walk.jsonl); output is a content-addressed text +store plus an index: + + /extract//.txt the extracted text, one file per + distinct content hash + /extract.jsonl one record per walked file: + status, extractor, chars, error + /extract.summary.json + +Content-addressing is the cache: a file whose hash already has a text +entry is not opened again, on this sweep or the next, and two copies of +the same document are extracted once. + +Portability rules are the walker's: stdlib only, Python 3.8 syntax floor, +runs on Mac/Windows/Linux. Office formats are zip + XML and need no +library. Two OUTSIDE tools are used when — and only when — they are on +the machine's PATH: `pdftotext` (poppler) for PDFs and `tesseract` for +images. They are sensors, resolved at run time and named in every record +they produced; nothing pins them and nothing fails for their absence — +the file is recorded `unsupported` with the tool that would handle it, +so the host can see exactly what it cannot read. + +The built-in PDF path handles simple PDFs (uncompressed or Flate content +streams with simple fonts). It refuses to emit text that does not look +like text (CID/Identity-encoded fonts come out as byte salad), recording +`unsupported` instead — garbage in the catalog is worse than a gap. +""" + +import codecs +import csv +import datetime as _dt +import email +import email.policy +import html +import html.parser +import io +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +import time +import unicodedata +import zipfile +import zlib + +from . import pdftext as _pdftext +import xml.etree.ElementTree as ET + +from .manifest import signing_digest + +STATUS_OK, STATUS_UNSUPPORTED, STATUS_ERROR, STATUS_SKIPPED, STATUS_EMPTY = ( + "ok", "unsupported", "error", "skipped", "empty") + +# Bump when an extractor's output changes: every cached verdict and text +# carries the version it was produced by and is redone under a new one. +EXTRACT_VERSION = 2 # 2: the built-in PDF reader (pdftext) reads real PDFs + +ZIP_MEMBER_CAP = 64 * 1024 * 1024 # decompressed bytes we will read from one member +TOOL_TIMEOUT = 300 # seconds per external-tool call +PDF_TEXT_MIN_RATIO = 0.9 # printable share below which builtin PDF text is refused + +TEXT_MIMES_PREFIX = ("text/",) +TEXT_MIMES = { + "application/json", "application/x-ndjson", "application/yaml", "application/toml", + "application/xml", "application/sql", "image/svg+xml", "application/x-pem-file", +} +DOCX = "application/vnd.openxmlformats-officedocument.wordprocessingml.document" +XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" +PPTX = "application/vnd.openxmlformats-officedocument.presentationml.presentation" +ODT, ODS, ODP = ("application/vnd.oasis.opendocument.text", + "application/vnd.oasis.opendocument.spreadsheet", + "application/vnd.oasis.opendocument.presentation") +IMAGE_OCR_MIMES = ("image/png", "image/jpeg", "image/tiff", "image/bmp", "image/gif", "image/webp") + + +# ── text normalisation ────────────────────────────────────────────────────── + +def _normalise(text): + text = text.replace("\r\n", "\n").replace("\r", "\n").replace("\x00", "") + text = unicodedata.normalize("NFC", text) + return text.strip("\n") + ("\n" if text.strip() else "") + + +def decode_bytes(data): + """Best-effort decode with the encoding recorded. BOMs first, strict + UTF-8 next, UTF-16 by NUL pattern, then cp1252 with replacement.""" + if data.startswith(b"\xef\xbb\xbf"): + return data[3:].decode("utf-8", "replace"), "utf-8-sig" + if data.startswith(b"\xff\xfe") or data.startswith(b"\xfe\xff"): + return data.decode("utf-16", "replace"), "utf-16" + # BOM-less UTF-16 is byte-valid UTF-8 (NUL is a code point), so the + # NUL-density test has to come before the strict UTF-8 attempt. + sample = data[:4096] + if len(sample) >= 4 and sample.count(b"\x00") > len(sample) // 4: + enc = "utf-16-le" if sample[1:2] == b"\x00" else "utf-16-be" + return data.decode(enc, "replace"), enc + try: + return data.decode("utf-8"), "utf-8" + except UnicodeDecodeError: + pass + return data.decode("cp1252", "replace"), "cp1252" + + +# ── html ──────────────────────────────────────────────────────────────────── + +_BLOCK_TAGS = {"p", "div", "br", "li", "tr", "h1", "h2", "h3", "h4", "h5", "h6", + "section", "article", "header", "footer", "table", "ul", "ol", + "blockquote", "pre", "hr", "td", "th", "dd", "dt", "title"} + + +class _HtmlText(html.parser.HTMLParser): + def __init__(self): + super().__init__(convert_charrefs=True) + self.parts = [] + self._skip = 0 + + def handle_starttag(self, tag, attrs): + if tag in ("script", "style", "noscript", "template"): + self._skip += 1 + elif tag in _BLOCK_TAGS: + self.parts.append("\n") + + def handle_endtag(self, tag): + if tag in ("script", "style", "noscript", "template"): + self._skip = max(0, self._skip - 1) + elif tag in _BLOCK_TAGS: + self.parts.append("\n") + + def handle_data(self, data): + if not self._skip: + self.parts.append(data) + + def text(self): + out = "".join(self.parts) + out = re.sub(r"[ \t\f\v]+", " ", out) + out = re.sub(r" *\n *", "\n", out) + return re.sub(r"\n{3,}", "\n\n", out) + + +def html_to_text(markup): + p = _HtmlText() + p.feed(markup) + p.close() + return p.text() + + +# ── office (zip + xml) ────────────────────────────────────────────────────── +# +# STREAMED, ON A BUDGET. A 200 KB spreadsheet can hold a gigabyte of XML, and +# parsing a member whole (ElementTree.fromstring) built the full tree in +# memory — two of them at once across sheets (measured 2.2 GB peak for a +# 1.3 MB xlsx). Members are fed to a pull parser in chunks; each wanted element +# (a paragraph, a row) is read and then dropped from its parent, so memory is +# one element, not one document. A document's decompressed XML, its text and +# its wall time are capped; reaching a cap stops the read with the text so far +# and says so on the last line. A DTD is refused outright: Office XML never +# carries one, and one is how entity expansion is smuggled in. + +OFFICE_DOC_CAP = 256 * 1024 * 1024 # decompressed XML bytes, one document +OFFICE_TEXT_CAP = 16 * 1024 * 1024 # characters of text, one document +OFFICE_SECONDS = 60.0 # wall time, one document +# Parts one paragraph / row may hold while it is being read. A real paragraph +# has tens of runs and an Excel row at most 16,384 cells; a crafted one with +# millions is refused rather than held in memory. +OFFICE_PARTS_PER_ITEM = 200_000 +_OFFICE_CHUNK = 64 * 1024 +_DTD = re.compile(rb"]*?encoding\s*=\s*["'])[^"']*(["'])""") + + +class _OfficeStop(Exception): + """A per-document cap was reached; the text so far stands.""" + + +class _OfficeBudget: + def __init__(self): + self.bytes = self.chars = 0 + self.deadline = time.monotonic() + OFFICE_SECONDS + self.cut_items = 0 # paragraphs/rows cut at OFFICE_PARTS_PER_ITEM + + def take_bytes(self, n): + self.bytes += n + if self.bytes > OFFICE_DOC_CAP: + raise _OfficeStop("the document expands past %d MB of XML" % (OFFICE_DOC_CAP >> 20)) + if time.monotonic() > self.deadline: + raise _OfficeStop("the document took longer than %ds to read" % OFFICE_SECONDS) + + def take_text(self, out, line): + self.chars += len(line) + 1 + if self.chars > OFFICE_TEXT_CAP: + raise _OfficeStop("the document holds more than %d MB of text" % (OFFICE_TEXT_CAP >> 20)) + out.append(line) + + +def _zip_elements(zf, name, budget, wanted, skip_within=()): + """Yield each `wanted` element of one member as it closes, then drop it. + A wanted tag inside a `skip_within` ancestor is left in place for that + ancestor (an ODF table row reads its own cells' paragraphs). Every other + finished element that no open wanted element needs is dropped as it + closes too, so siblings never pile up — memory is one item, and removal + from a parent stays O(1).""" + info = zf.getinfo(name) + if info.file_size > ZIP_MEMBER_CAP: + raise ValueError("zip member %s too large (%d bytes)" % (name, info.file_size)) + keep = set(wanted) | set(skip_within) + parser = ET.XMLPullParser(("start", "end")) + stack = [] + state = {"skipping": 0, "keeping": 0, "parts": 0} + + # Finished elements are cleared at once (their subtree is freed) and + # detached from their parents in one batch per fed chunk. The pull parser + # reads the whole chunk ahead, so a closing element is NOT its parent's + # last child when its event is handled, and remove() scans every kept + # sibling from the start: per element that was quadratic. + detach = {} + + def drop(el): + el.clear() + # Until flush() detaches it, a dropped element is still a child of + # its parent; renamed so no reader's tag match sees it (a cut row + # would otherwise end in empty cells). + el.tag = _DROPPED + if stack: + parent = stack[-1] + detach.setdefault(id(parent), (parent, set()))[1].add(id(el)) + + def flush(): + for parent, gone in detach.values(): + parent[:] = [c for c in parent if id(c) not in gone] + detach.clear() + + def drain(): + for ev, el in parser.read_events(): + if ev == "start": + stack.append(el) + if el.tag in skip_within: + state["skipping"] += 1 + if el.tag in keep: + state["keeping"] += 1 + elif state["keeping"]: + state["parts"] += 1 + if state["parts"] == OFFICE_PARTS_PER_ITEM + 1: + budget.cut_items += 1 + continue + stack.pop() + if el.tag in skip_within: + state["skipping"] -= 1 + if el.tag in keep: + state["keeping"] -= 1 + if not state["keeping"]: + state["parts"] = 0 + if el.tag in wanted and not (state["skipping"] and el.tag not in skip_within): + # How many wanted items are still open around this one: a + # docx text box is a paragraph inside a paragraph. + yield el, state["keeping"] + drop(el) + elif not state["keeping"] or state["parts"] > OFFICE_PARTS_PER_ITEM: + # Past the cap, an item's further parts are dropped as they + # close: that one paragraph or row is cut, the rest of the + # document still reads. + drop(el) + + seen, tail, utf16 = 0, b"", None + with zf.open(name) as fh: + while True: + chunk = fh.read(_OFFICE_CHUNK) + if not chunk: + break + if not seen: + # OOXML allows UTF-8 or UTF-16. UTF-16 is decoded to UTF-8 + # here, BEFORE the DTD check, so a DTD cannot ride past it in + # a wide encoding; the declaration is rewritten to match. + # Anything else wide (UTF-32, no BOM) is refused. + if chunk[:2] in (b"\xff\xfe", b"\xfe\xff"): + utf16 = codecs.getincrementaldecoder("utf-16")("strict") + elif b"\x00" in chunk[:4]: + raise ValueError("zip member %s is not UTF-8 or UTF-16 XML" % name) + if utf16 is not None: + text = utf16.decode(chunk) + if not seen: + text = _XML_ENCODING.sub(r"\1UTF-8\2", text, count=1) + raw = chunk + chunk = text.encode("utf-8") + seen += len(raw) + else: + seen += len(chunk) + if seen > ZIP_MEMBER_CAP: + raise ValueError("zip member %s too large" % name) + if _DTD.search(tail + chunk): + raise ValueError("zip member %s declares a DTD; refused" % name) + tail = chunk[-16:] + budget.take_bytes(len(chunk)) + parser.feed(chunk) + for item in drain(): + yield item + flush() + parser.close() + for item in drain(): + yield item + flush() + + +def _office(read): + """Run one reader with a fresh budget; a cap reached keeps the text so + far and names the cap on the last line.""" + budget, out = _OfficeBudget(), [] + try: + read(budget, out) + except _OfficeStop as e: + out.append("[stopped reading here: %s]" % e) + if budget.cut_items: + out.append("[%d paragraph(s) or row(s) held more than %d parts; each was cut there]" + % (budget.cut_items, OFFICE_PARTS_PER_ITEM)) + return "\n".join(out) + + +def _local(tag): + return tag.rsplit("}", 1)[-1] + + +W = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}" +A = "{http://schemas.openxmlformats.org/drawingml/2006/main}" +S = "{http://schemas.openxmlformats.org/spreadsheetml/2006/main}" +TEXT_NS = "{urn:oasis:names:tc:opendocument:xmlns:text:1.0}" +TABLE_NS = "{urn:oasis:names:tc:opendocument:xmlns:table:1.0}" + + +def _docx_paragraph(p): + parts = [] + for node in p.iter(): + t = _local(node.tag) + if t == "t" and node.text: + parts.append(node.text) + elif t == "tab": + parts.append("\t") + elif t in ("br", "cr"): + parts.append("\n") + return "".join(parts) + + +def extract_docx(data): + def read(budget, out): + with zipfile.ZipFile(io.BytesIO(data)) as zf: + names = set(zf.namelist()) + order = ["word/document.xml"] + sorted( + n for n in names if re.match(r"word/(header|footer)\d*\.xml$", n)) + for name in order: + if name not in names: + continue + boxed = [] # text-box paragraphs, read after the paragraph holding them + for p, depth in _zip_elements(zf, name, budget, {W + "p"}): + if depth: + boxed.append(_docx_paragraph(p)) + continue + budget.take_text(out, _docx_paragraph(p)) + for line in boxed: + budget.take_text(out, line) + boxed = [] + for line in boxed: + budget.take_text(out, line) + out.append("") + return _office(read) + + +def extract_xlsx(data): + def read(budget, out): + with zipfile.ZipFile(io.BytesIO(data)) as zf: + names = set(zf.namelist()) + shared = [] + if "xl/sharedStrings.xml" in names: + for si, _ in _zip_elements(zf, "xl/sharedStrings.xml", budget, {S + "si"}): + text = "".join(t.text or "" for t in si.iter(S + "t")) + budget.take_text([], text) # shared strings count against the cap too + shared.append(text) + sheets = sorted(n for n in names if re.match(r"xl/worksheets/sheet\d+\.xml$", n)) + for name in sheets: + out.append("# " + name.rsplit("/", 1)[-1][:-4]) + for row, _ in _zip_elements(zf, name, budget, {S + "row"}): + cells = [] + for c in row.iter(S + "c"): + v = c.find(S + "v") + is_node = c.find(S + "is") + val = "" + if c.get("t") == "s" and v is not None and v.text is not None: + try: + val = shared[int(v.text)] + except (ValueError, IndexError): + val = "" + elif c.get("t") == "inlineStr" and is_node is not None: + val = "".join(t.text or "" for t in is_node.iter(S + "t")) + elif v is not None and v.text is not None: + val = v.text + cells.append(val) + if any(cells): + budget.take_text(out, "\t".join(cells)) + out.append("") + return _office(read) + + +def extract_pptx(data): + def read(budget, out): + with zipfile.ZipFile(io.BytesIO(data)) as zf: + names = zf.namelist() + slides = sorted((n for n in names if re.match(r"ppt/slides/slide\d+\.xml$", n)), + key=lambda n: int(re.search(r"(\d+)\.xml$", n).group(1))) + for name in slides: + out.append("# " + name.rsplit("/", 1)[-1][:-4]) + for p, _ in _zip_elements(zf, name, budget, {A + "p"}): + line = "".join(t.text or "" for t in p.iter(A + "t")) + if line: + budget.take_text(out, line) + out.append("") + return _office(read) + + +def extract_odf(data): + row_tag, cell_tag = TABLE_NS + "table-row", TABLE_NS + "table-cell" + + def read(budget, out): + with zipfile.ZipFile(io.BytesIO(data)) as zf: + wanted = {TEXT_NS + "p", TEXT_NS + "h", row_tag} + for el, _ in _zip_elements(zf, "content.xml", budget, wanted, skip_within={row_tag}): + if el.tag == row_tag: + # A row's cells hold paragraphs; the row is one line. + cells = ["".join(c.itertext()) for c in el if c.tag == cell_tag] + if any(cells): + budget.take_text(out, "\t".join(cells)) + else: + budget.take_text(out, "".join(el.itertext())) + return _office(read) + + +# ── email, rtf ────────────────────────────────────────────────────────────── + +def extract_eml(data): + msg = email.message_from_bytes(data, policy=email.policy.default) + head = [] + for h in ("From", "To", "Cc", "Date", "Subject"): + if msg[h]: + head.append("%s: %s" % (h, msg[h])) + body = msg.get_body(preferencelist=("plain", "html")) + text = "" + if body is not None: + content = body.get_content() + text = html_to_text(content) if body.get_content_type() == "text/html" else content + return "\n".join(head) + "\n\n" + text + + +_RTF_SKIP_GROUPS = {"fonttbl", "colortbl", "stylesheet", "info", "pict", "object", + "header", "footer", "xmlnstbl", "themedata", "datastore", "generator"} +_RTF_TOKEN = re.compile(r"\\([a-zA-Z]+)(-?\d+)? ?|\\'([0-9a-fA-F]{2})|\\([^a-zA-Z])|([{}])|([^\\{}]+)") + + +def extract_rtf(data): + text = data.decode("latin-1", "replace") + out = [] + stack = [] + skip_depth = None + uc_skip = 0 + for m in _RTF_TOKEN.finditer(text): + word, arg, hexch, esc, brace, plain = m.groups() + if brace == "{": + stack.append(len(out)) + continue + if brace == "}": + if stack: + stack.pop() + # the skipped group closes when the stack drops below the depth + # it was opened at + if skip_depth is not None and len(stack) < skip_depth: + skip_depth = None + continue + if skip_depth is not None: + continue + if word: + if word in _RTF_SKIP_GROUPS: + skip_depth = len(stack) + elif word in ("par", "line", "sect", "page"): + out.append("\n") + elif word == "tab": + out.append("\t") + elif word == "u" and arg is not None: + cp = int(arg) + out.append(chr(cp + 65536 if cp < 0 else cp)) + uc_skip = 1 + continue + if hexch: + if uc_skip: + uc_skip -= 1 + continue + out.append(bytes([int(hexch, 16)]).decode("cp1252", "replace")) + continue + if esc: + out.append({"~": "\u00a0", "-": "", "_": "-", "\n": "\n", "\r": ""}.get(esc, esc)) + continue + if plain: + if uc_skip: + plain = plain[uc_skip:] + uc_skip = 0 + out.append(plain.replace("\r", "").replace("\n", "")) + return "".join(out) + + +# ── pdf (built-in best effort) ────────────────────────────────────────────── + +def extract_pdf_builtin(data): + """Text from the document's pages with the standard library (pdftext): + fonts' /ToUnicode maps, compressed streams and object streams, so a + browser-printed PDF reads the same as an old literal-string one. Returns + None when the result does not look like text (encrypted files, pages that + are only pictures). pdftext never raises and reads on a per-document + budget, so a hostile file costs bounded time and is at worst no text.""" + text = _pdftext.extract_text(data) + if not text or not text.strip(): + return None + printable = sum(1 for ch in text if ch.isprintable() or ch in "\n\t\f") + if printable / max(1, len(text)) < PDF_TEXT_MIN_RATIO: + return None + return text + + +# ── external tools (sensors) ──────────────────────────────────────────────── + +def tool_path(name): + return shutil.which(name) + + +def run_tool(argv, timeout=TOOL_TIMEOUT): + """Run an external tool; returns stdout bytes or raises RuntimeError.""" + try: + proc = subprocess.run(argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + timeout=timeout, check=False) + except (OSError, subprocess.TimeoutExpired) as exc: + raise RuntimeError("%s: %s" % (argv[0], exc)) + if proc.returncode != 0: + err = proc.stderr.decode("utf-8", "replace").strip().splitlines() + raise RuntimeError("%s exited %d: %s" % (argv[0], proc.returncode, err[-1] if err else "")) + return proc.stdout + + +def extract_pdf_tool(path, pdftotext): + return decode_bytes(run_tool([pdftotext, "-layout", "-enc", "UTF-8", path, "-"]))[0] + + +def extract_image_tool(path, tesseract): + return decode_bytes(run_tool([tesseract, path, "stdout"]))[0] + + +# ── dispatch ──────────────────────────────────────────────────────────────── + +class Extractor: + """Maps a walk record to (status, extractor_name, text_or_None, error).""" + + def __init__(self, use_external=True): + self.pdftotext = tool_path("pdftotext") if use_external else None + self.tesseract = tool_path("tesseract") if use_external else None + + def tools(self): + return {"pdftotext": self.pdftotext, "tesseract": self.tesseract} + + def _read(self, path): + with open(path, "rb") as fh: + return fh.read() + + def extract(self, path, mime): + try: + return self._extract(path, mime) + except (OSError, ValueError, zipfile.BadZipFile, ET.ParseError, zlib.error, + RuntimeError, UnicodeError, KeyError, csv.Error) as exc: + # csv.Error is NOT a ValueError: one >128 KiB CSV field escaped this + # tuple and crash-looped the whole sweep + return STATUS_ERROR, None, None, "%s: %s" % (type(exc).__name__, exc) + + def _extract(self, path, mime): + if mime is None: + return STATUS_UNSUPPORTED, None, None, "no mime" + if mime == "inode/x-empty": + return STATUS_OK, "empty", "", None # recorded as `empty` by the stage + if mime.startswith(TEXT_MIMES_PREFIX) or mime in TEXT_MIMES: + text, enc = decode_bytes(self._read(path)) + if mime == "text/html": + return STATUS_OK, "html", html_to_text(text), None + if mime == "message/rfc822": + return STATUS_OK, "eml", extract_eml(self._read(path)), None + if mime in ("text/csv", "text/tab-separated-values"): + rows = csv.reader(io.StringIO(text), delimiter="\t" if "tab" in mime else ",") + return STATUS_OK, "csv", "\n".join("\t".join(r) for r in rows), None + return STATUS_OK, "text:" + enc, text, None + if mime == "message/rfc822": + return STATUS_OK, "eml", extract_eml(self._read(path)), None + if mime == "application/rtf": + return STATUS_OK, "rtf", extract_rtf(self._read(path)), None + if mime == DOCX: + return STATUS_OK, "docx", extract_docx(self._read(path)), None + if mime == XLSX: + return STATUS_OK, "xlsx", extract_xlsx(self._read(path)), None + if mime == PPTX: + return STATUS_OK, "pptx", extract_pptx(self._read(path)), None + if mime in (ODT, ODS, ODP): + return STATUS_OK, "odf", extract_odf(self._read(path)), None + if mime == "application/pdf": + if self.pdftotext: + return STATUS_OK, "pdftotext", extract_pdf_tool(path, self.pdftotext), None + text = extract_pdf_builtin(self._read(path)) + if text is None: + return (STATUS_UNSUPPORTED, "pdf-builtin", None, + "needs pdftotext (poppler) on this machine") + return STATUS_OK, "pdf-builtin", text, None + if mime in IMAGE_OCR_MIMES: + if self.tesseract: + return STATUS_OK, "tesseract", extract_image_tool(path, self.tesseract), None + return STATUS_UNSUPPORTED, None, None, "needs tesseract on this machine" + return STATUS_UNSUPPORTED, None, None, "no extractor for %s" % mime + + +# ── the stage ─────────────────────────────────────────────────────────────── + +def _atomic_write_text(path, text): + d = os.path.dirname(path) + os.makedirs(d, exist_ok=True) + fd, tmp = tempfile.mkstemp(prefix=".tmp-", dir=d) + try: + with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as fh: + fh.write(text) + os.replace(tmp, path) + except BaseException: + try: + os.unlink(tmp) + except OSError: + pass + raise + + +def store_path(out_dir, sha): + return os.path.join(out_dir, "extract", sha[:2], sha + ".txt") + + +def load_walk(out_dir): + path = os.path.join(out_dir, "walk.jsonl") + if not os.path.exists(path): + raise FileNotFoundError("no walk map at %s — run the walk stage first" % path) + summary_path = os.path.join(out_dir, "walk.summary.json") + with open(summary_path, encoding="utf-8") as fh: + summary = json.load(fh) + roots = {r["index"]: r["resolved"] for r in summary["roots"]} + records = [] + with open(path, encoding="utf-8") as fh: + for line in fh: + rec = json.loads(line) + if rec.get("kind") == "file": + records.append(rec) + return roots, records, summary + + +def load_index(out_dir): + """Previous extract.jsonl: sha256 -> record, so unchanged content is + not re-extracted and prior unsupported/error verdicts are re-tried + only when the machine's tools changed.""" + idx = {} + path = os.path.join(out_dir, "extract.jsonl") + try: + with open(path, encoding="utf-8") as fh: + for line in fh: + try: + rec = json.loads(line) + except ValueError: + continue + if rec.get("sha256"): + idx[rec["sha256"]] = rec + except OSError: + pass + return idx + + +def extract(manifest, out_dir, use_external=True, now=None): + roots, files, walk_summary = load_walk(out_dir) + ex = Extractor(use_external=use_external) + tools = ex.tools() + prev = load_index(out_dir) + counts = {k: 0 for k in ("files", "ok", "reused", "unsupported", "error", "skipped", "empty")} + chars = 0 + index = [] + done_sha = {} + + for rec in files: + counts["files"] += 1 + sha = rec.get("sha256") + base = {"root": rec["root"], "path": rec["path"], "sha256": sha, "mime": rec.get("mime"), + "v": EXTRACT_VERSION} + if rec.get("skip") or rec.get("error") or not sha: + counts["skipped"] += 1 + index.append(dict(base, status=STATUS_SKIPPED, + error=rec.get("skip") or rec.get("error") or "no hash")) + continue + if sha in done_sha: # duplicate content on this sweep + dup = done_sha[sha] + index.append(dict(dup, root=rec["root"], path=rec["path"])) + counts["reused" if dup.get("reused") else dup["status"]] += 1 + if dup.get("reused") is None and dup["status"] == STATUS_OK: + chars += dup.get("chars", 0) + continue + spath = store_path(out_dir, sha) + old = prev.get(sha) + if old and old.get("v") != EXTRACT_VERSION: + old = None # produced by an older extractor: redo + if old and old.get("status") in (STATUS_OK, STATUS_EMPTY) and ( + old["status"] == STATUS_EMPTY or os.path.exists(spath)): + out = dict(base, status=old["status"], extractor=old.get("extractor"), + chars=old.get("chars", 0), reused=True) + counts["reused"] += 1 + chars += out["chars"] + index.append(out) + done_sha[sha] = out + continue + if old and old.get("status") == STATUS_UNSUPPORTED and old.get("tools") == tools: + out = dict(base, status=STATUS_UNSUPPORTED, extractor=old.get("extractor"), + error=old.get("error"), tools=tools, reused=True) + counts["unsupported"] += 1 + index.append(out) + done_sha[sha] = out + continue + + full = os.path.join(roots[rec["root"]], *rec["path"].split("/")) + status, name, text, err = ex.extract(full, rec.get("mime")) + out = dict(base, status=status, extractor=name) + if status == STATUS_OK: + text = _normalise(text or "") + if not text.strip(): + out["status"] = STATUS_EMPTY + out["chars"] = 0 + else: + _atomic_write_text(spath, text) + out["chars"] = len(text) + chars += len(text) + else: + out["error"] = err + if status == STATUS_UNSUPPORTED: + out["tools"] = tools + counts[out["status"]] += 1 + index.append(out) + done_sha[sha] = out + + index.sort(key=lambda r: (r["root"], r["path"])) + idx_path = os.path.join(out_dir, "extract.jsonl") + _atomic_write_text(idx_path, "".join( + json.dumps(r, ensure_ascii=False, sort_keys=True) + "\n" for r in index)) + summary = { + "freeagent": 1, + "stage": "extract", + "extractor_version": EXTRACT_VERSION, + "manifest_id": manifest.id, + "manifest_digest": "sha256:" + signing_digest(manifest.raw), + "walk_digest": walk_summary.get("manifest_digest"), + "extracted_at": now or _dt.datetime.now(_dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "tools": tools, + "counts": counts, + "chars": chars, + "entries": len(index), + } + _atomic_write_text(os.path.join(out_dir, "extract.summary.json"), + json.dumps(summary, ensure_ascii=False, indent=2, sort_keys=True) + "\n") + return summary diff --git a/bastion_agent/install.ps1 b/bastion_agent/install.ps1 new file mode 100644 index 0000000..98f667b --- /dev/null +++ b/bastion_agent/install.ps1 @@ -0,0 +1,32 @@ +# bastion agent installer — Windows (PowerShell 5+). No admin, no downloads. +# powershell -ExecutionPolicy Bypass -File install.ps1 dist\bastion-agent-0.1.0.pyz +# Puts bastion-agent.cmd in %LOCALAPPDATA%\bastion-agent and runs the preflight. +param([Parameter(Mandatory = $true)][string]$Pyz) +$ErrorActionPreference = "Stop" +$py = Get-Command python -ErrorAction SilentlyContinue +if (-not $py) { $py = Get-Command py -ErrorAction SilentlyContinue } +if (-not $py) { Write-Error "Python 3.8+ is required (python.org or the Store); none on PATH"; exit 1 } +& $py.Source -c "import sys; sys.exit(0 if sys.version_info >= (3, 8) else 1)" +if ($LASTEXITCODE -ne 0) { Write-Error "Python is older than 3.8"; exit 1 } +if (Test-Path "$Pyz.sha256") { + $want = (Get-Content "$Pyz.sha256").Split(" ")[0].ToLower() + $have = (Get-FileHash -Algorithm SHA256 $Pyz).Hash.ToLower() + if ($want -ne $have) { Write-Error "checksum MISMATCH - refusing to install"; exit 1 } + Write-Host "checksum ok" +} +$dir = Join-Path $env:LOCALAPPDATA "bastion-agent" +New-Item -ItemType Directory -Force -Path $dir | Out-Null +Copy-Item $Pyz (Join-Path $dir "bastion-agent.pyz") -Force +$cmd = Join-Path $dir "bastion-agent.cmd" +"@echo off`r`n`"$($py.Source)`" `"$dir\bastion-agent.pyz`" %*`r`n" | Set-Content -Path $cmd -Encoding ASCII +Write-Host "installed $cmd" +$userPath = [Environment]::GetEnvironmentVariable("Path", "User") +if ($userPath -notlike "*$dir*") { + [Environment]::SetEnvironmentVariable("Path", "$userPath;$dir", "User") + Write-Host "added $dir to your user PATH (open a new terminal)" +} +& $cmd --version +& $cmd doctor +Write-Host "" +Write-Host "next: put your manifest at %APPDATA%\bastion-agent\manifest.yaml, set FREEAGENT_OWNER_RECIPIENT" +Write-Host " (age-keygen), then: bastion-agent sweep %APPDATA%\bastion-agent\manifest.yaml --once" diff --git a/bastion_agent/install.sh b/bastion_agent/install.sh new file mode 100644 index 0000000..9bcb7d5 --- /dev/null +++ b/bastion_agent/install.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env sh +# bastion agent installer — Linux and macOS. POSIX sh, no root, no downloads. +# sh install.sh dist/bastion-agent-0.1.7.pyz +# Puts `bastion-agent` on your PATH (~/.local/bin) and runs the preflight. +# +# This is the HAND install. The managed path — the bastion deploying the agent +# from a grant — does not use this script. +set -eu +PYZ="${1:?usage: install.sh }" +# FREEAGENT_BIN_DIR keeps its name deliberately: the six FREEAGENT_* variables +# are the contract the bastion sets when it spawns the agent, and renaming them +# here would break every bastion that still sets the old ones. They move in +# their own compatibility window, not this one. +BIN="${BASTION_AGENT_BIN_DIR:-${FREEAGENT_BIN_DIR:-$HOME/.local/bin}}" +PY="$(command -v python3 || true)" +[ -n "$PY" ] || { echo "python3 is required (3.8 or newer); none on PATH" >&2; exit 1; } +"$PY" -c 'import sys; sys.exit(0 if sys.version_info >= (3, 8) else 1)' \ + || { echo "python3 is older than 3.8: $("$PY" --version)" >&2; exit 1; } +if [ -f "${PYZ}.sha256" ]; then + (cd "$(dirname "$PYZ")" && { command -v sha256sum >/dev/null && sha256sum -c "$(basename "$PYZ").sha256" \ + || shasum -a 256 -c "$(basename "$PYZ").sha256"; }) >/dev/null \ + && echo "checksum ok" || { echo "checksum MISMATCH — refusing to install" >&2; exit 1; } +fi +mkdir -p "$BIN" +cp "$PYZ" "$BIN/bastion-agent.pyz" +printf '#!/usr/bin/env sh\nexec python3 "%s/bastion-agent.pyz" "$@"\n' "$BIN" > "$BIN/bastion-agent" +chmod 755 "$BIN/bastion-agent" "$BIN/bastion-agent.pyz" +# An earlier hand install left a `freeagent` wrapper pointing at a file this +# install replaces. Repoint it rather than leaving it broken on someone's PATH; +# it is the same program under its old name. +if [ -f "$BIN/freeagent" ]; then + printf '#!/usr/bin/env sh\n# renamed to bastion-agent; this wrapper kept so an old habit still works\nexec python3 "%s/bastion-agent.pyz" "$@"\n' "$BIN" > "$BIN/freeagent" + chmod 755 "$BIN/freeagent" + rm -f "$BIN/freeagent.pyz" "$BIN/bastion_agent.pyz" + echo "note: kept $BIN/freeagent working — it now runs bastion-agent" +fi +echo "installed $BIN/bastion-agent ($("$BIN/bastion-agent" --version))" +case ":$PATH:" in *":$BIN:"*) ;; *) echo "note: add $BIN to PATH";; esac +echo +"$BIN/bastion-agent" doctor || true +echo +echo "next: put your manifest at ~/.config/bastion-agent/manifest.yaml, set FREEAGENT_OWNER_RECIPIENT" +echo " (age-keygen), then: bastion-agent sweep ~/.config/bastion-agent/manifest.yaml --once" diff --git a/bastion_agent/manifest.py b/bastion_agent/manifest.py new file mode 100644 index 0000000..c4b0eab --- /dev/null +++ b/bastion_agent/manifest.py @@ -0,0 +1,576 @@ +"""bastion agent job-manifest loader — the only way tools read a manifest. + +Validation collects every problem before raising, so one run reports the whole +file. See docs/manifest.md for the format contract. The loader fixes +the *shape* and the signing payload; cryptographic verification of the +signature belongs to the runtime (run.py, x509gate.py), never here. +""" + +import base64 +import datetime as _dt +import hashlib +import json +import re +from dataclasses import dataclass + +import yaml + +SCHEMA_VERSION = 1 + +# The agent lives WITH its folder: /.agent/ holds the manifest and every +# prepared output (catalog, sanitized index, vectors, run log). The walker +# always excludes it, so an agent never indexes its own output. Secrets (the +# PII vault, the identity key) are NOT kept there — a Documents folder is +# often cloud-synced — see FREEAGENT_VAULT_DIR and the bastion's state dir. +AGENT_DIR = ".agent" + +SLUG_RE = re.compile(r"^[a-z0-9-]+$") +DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$") +# a root must mean the same place wherever the manifest lands: POSIX absolute, +# home-relative, Windows drive-absolute (C:\ or C:/), or UNC (\\server\share) +ROOT_RE = re.compile(r"^(/|~$|~/|[A-Za-z]:[\\/]|\\\\[^\\/]+[\\/][^\\/]+)") + +TOP_KEYS_REQUIRED = ("freeagent", "id", "owner", "scope", "mask", "stages", "sweep", "sink") +TOP_KEYS_OPTIONAL = ("sanitize", "embed", "summarize", "grants", "signature") + +MASK_OPS = ("read", "search", "index", "summarize") +MASK_MANDATORY = "read" + +# canonical stage order; a manifest's stages must be a subset in this order +STAGES = ("walk", "extract", "sanitize", "chunk", "embed", "catalog", "seal") +# chunk and catalog read the SANITIZED store, so they need sanitize, not +# merely extract (the runtime failed closed, the schema should +# have refused first) +STAGE_NEEDS_STAGE = { + "extract": "walk", + "sanitize": "extract", + "chunk": "sanitize", + "embed": "chunk", + "catalog": "sanitize", + "seal": "catalog", +} +STAGE_NEEDS_OP = {"embed": "index", "catalog": "index"} + +SANITIZE_MODES = ("sterilize", "flag") +SUMMARIZE_PER = ("file", "directory") +SWEEPS = ("15m", "hourly", "daily", "manual") # same vocabulary as the platform context store +SINK_CATALOG = ("local", "host") +SINK_VECTORS = ("local",) +SINK_PLAINTEXT = "never" # a constant, not a setting +GRANT_OPS = ("search", "read_range") +GRANT_OP_NEEDS_MASK = {"search": "search", "read_range": "read"} +SIGNATURE_ALGS = ("ecdsa-p256-sha256",) + +DEFAULT_MAX_FILE_BYTES = 256 * 1024 * 1024 + + +class ManifestError(Exception): + """Raised with every validation error joined into one message.""" + + def __init__(self, errors): + self.errors = list(errors) + super().__init__( + "manifest validation failed:\n" + "\n".join(f" - {e}" for e in self.errors) + ) + + +@dataclass(frozen=True) +class Scope: + roots: tuple + exclude: tuple = () + follow_symlinks: bool = False + max_file_bytes: int = DEFAULT_MAX_FILE_BYTES + + +@dataclass(frozen=True) +class Sanitize: + mode: str + scope: str + verify: bool = False + + +@dataclass(frozen=True) +class Embed: + model: str + dim: int + chunk_size_tokens: int + chunk_overlap_tokens: int + + +@dataclass(frozen=True) +class Summarize: + model: str + per: tuple + + +@dataclass(frozen=True) +class Sink: + catalog: str + vectors: str = "local" + plaintext: str = SINK_PLAINTEXT + seal_to: tuple = () + + +@dataclass(frozen=True) +class Grant: + host: str + ops: tuple + until: _dt.date + + +@dataclass(frozen=True) +class Signature: + alg: str + cert: str + value: str + + +@dataclass(frozen=True) +class Manifest: + id: str + owner_host: str + scope: Scope + mask: tuple + stages: tuple + sweep: str + sink: Sink + sanitize: Sanitize = None + embed: Embed = None + summarize: Summarize = None + grants: tuple = () + signature: Signature = None + raw: dict = None # the parsed document, for signing_payload() + + @property + def signed(self): + return self.signature is not None + + def has_stage(self, name): + return name in self.stages + + +# ── signing payload ───────────────────────────────────────────────────────── + +def signing_payload(doc): + """Bytes the signature covers: the document minus `signature`, as + canonical JSON (sorted keys, no whitespace, UTF-8). Signer and runtime + must agree byte-for-byte, so this is the single definition.""" + body = {k: v for k, v in doc.items() if k != "signature"} + return json.dumps( + body, sort_keys=True, separators=(",", ":"), ensure_ascii=False, + default=_canonical_scalar, + ).encode("utf-8") + + +def _canonical_scalar(obj): + # YAML parses a bare 2026-12-31 as a date; a quoted one stays a string. + # Both must sign to the same bytes, so dates canonicalise to ISO text. + if isinstance(obj, _dt.date) and not isinstance(obj, _dt.datetime): + return obj.isoformat() + raise TypeError(f"signing payload cannot contain {type(obj).__name__}") + + +def signing_digest(doc): + return hashlib.sha256(signing_payload(doc)).hexdigest() + + +# ── helpers ───────────────────────────────────────────────────────────────── + +def _is_int(v): + return isinstance(v, int) and not isinstance(v, bool) + + +def _is_str(v, lo=1, hi=None): + return isinstance(v, str) and len(v) >= lo and (hi is None or len(v) <= hi) + + +def _check_keys(where, raw, required, optional, errors): + """Unknown/missing key check. Returns True when the key set is clean.""" + if not isinstance(raw, dict): + errors.append(f"{where}: must be a map") + return False + unknown = set(raw) - set(required) - set(optional) + for key in sorted(unknown, key=str): + errors.append(f"{where}: unknown key {key!r}") + missing = [k for k in required if k not in raw] + if missing: + errors.append(f"{where}: missing required key(s): {', '.join(missing)}") + return not unknown and not missing + + +def _str_list(where, key, val, errors, allowed=None, nonempty=True, hi=None): + """Validate a list of strings (optionally from a closed vocabulary, no + duplicates). Returns the tuple, or None when invalid.""" + if not isinstance(val, list) or not all(isinstance(s, str) for s in val): + errors.append(f"{where}: {key} must be a list of strings") + return None + if nonempty and not val: + errors.append(f"{where}: {key} must not be empty") + return None + ok = True + if allowed is not None: + bad = [s for s in val if s not in allowed] + if bad: + errors.append(f"{where}: {key} has value(s) not in {allowed}: {bad}") + ok = False + if hi is not None: + long = [s for s in val if not _is_str(s, 1, hi)] + if long: + errors.append(f"{where}: {key} entries must be 1-{hi} chars: {long}") + ok = False + dupes = sorted({s for s in val if val.count(s) > 1}) + if dupes: + errors.append(f"{where}: {key} has duplicate(s): {dupes}") + ok = False + return tuple(val) if ok else None + + +# ── sections ──────────────────────────────────────────────────────────────── + +def _validate_scope(raw, errors): + where = "scope" + if not _check_keys(where, raw, ("roots",), ("exclude", "follow_symlinks", "max_file_bytes"), errors): + return None + ok = True + roots = _str_list(where, "roots", raw.get("roots"), errors) + if roots is not None: + bad = [r for r in roots if not ROOT_RE.match(r)] + if bad: + errors.append( + f"{where}: roots must be absolute (/, C:\\, \\\\server\\share) or start with '~/': {bad}" + ) + ok = False + else: + ok = False + exclude = _str_list(where, "exclude", raw.get("exclude", []), errors, nonempty=False) + if exclude is None: + ok = False + follow = raw.get("follow_symlinks", False) + if not isinstance(follow, bool): + errors.append(f"{where}: follow_symlinks must be a boolean") + ok = False + max_bytes = raw.get("max_file_bytes", DEFAULT_MAX_FILE_BYTES) + if not _is_int(max_bytes) or max_bytes < 1: + errors.append(f"{where}: max_file_bytes must be a positive integer") + ok = False + if not ok: + return None + return Scope(roots=roots, exclude=exclude, follow_symlinks=follow, max_file_bytes=max_bytes) + + +def _validate_stages(raw, errors): + stages = _str_list("stages", "stages", raw, errors, allowed=STAGES) + if stages is None: + return None + ok = True + if stages[0] != "walk": + errors.append("stages: 'walk' must be present and first") + ok = False + order = [STAGES.index(s) for s in stages] + if order != sorted(order): + errors.append(f"stages: must follow the canonical order {STAGES}") + ok = False + for s in stages: + need = STAGE_NEEDS_STAGE.get(s) + if need and need not in stages: + errors.append(f"stages: {s!r} requires {need!r}") + ok = False + return stages if ok else None + + +def _validate_sanitize(raw, manifest_id, errors): + where = "sanitize" + if not _check_keys(where, raw, ("mode",), ("scope", "verify"), errors): + return None + ok = True + mode = raw.get("mode") + if mode not in SANITIZE_MODES: + errors.append(f"{where}: mode {mode!r} is not one of {SANITIZE_MODES}") + ok = False + scope = raw.get("scope", manifest_id) + if not _is_str(scope, 1, 63): + errors.append(f"{where}: scope must be a string of 1-63 chars") + ok = False + verify = raw.get("verify", False) + if not isinstance(verify, bool): + errors.append(f"{where}: verify must be a boolean") + ok = False + elif verify and mode == "flag": + errors.append(f"{where}: verify is only valid with mode 'sterilize' (flag is rules-only)") + ok = False + if not ok: + return None + return Sanitize(mode=mode, scope=scope, verify=verify) + + +def _validate_embed(raw, errors): + where = "embed" + if not _check_keys(where, raw, ("model", "dim", "chunk"), (), errors): + return None + ok = True + model = raw.get("model") + if not _is_str(model, 1, 128): + errors.append(f"{where}: model must be a string of 1-128 chars") + ok = False + dim = raw.get("dim") + if not _is_int(dim) or not 1 <= dim <= 8192: + errors.append(f"{where}: dim must be an integer 1-8192") + ok = False + chunk = raw.get("chunk") + size = overlap = None + if _check_keys(f"{where}.chunk", chunk, ("size_tokens", "overlap_tokens"), (), errors): + size = chunk.get("size_tokens") + overlap = chunk.get("overlap_tokens") + if not _is_int(size) or not 1 <= size <= 8192: + errors.append(f"{where}.chunk: size_tokens must be an integer 1-8192") + ok = False + if not _is_int(overlap) or overlap < 0: + errors.append(f"{where}.chunk: overlap_tokens must be a non-negative integer") + ok = False + elif _is_int(size) and overlap >= size: + errors.append(f"{where}.chunk: overlap_tokens must be smaller than size_tokens") + ok = False + else: + ok = False + if not ok: + return None + return Embed(model=model, dim=dim, chunk_size_tokens=size, chunk_overlap_tokens=overlap) + + +def _validate_summarize(raw, errors): + where = "summarize" + if not _check_keys(where, raw, ("model", "per"), (), errors): + return None + model = raw.get("model") + ok = True + if not _is_str(model, 1, 128): + errors.append(f"{where}: model must be a string of 1-128 chars") + ok = False + per = _str_list(where, "per", raw.get("per"), errors, allowed=SUMMARIZE_PER) + if per is None: + ok = False + return Summarize(model=model, per=per) if ok else None + + +def _validate_sink(raw, errors): + where = "sink" + if not _check_keys(where, raw, ("catalog", "vectors", "plaintext"), ("seal_to",), errors): + return None + ok = True + catalog = raw.get("catalog") + if catalog not in SINK_CATALOG: + errors.append(f"{where}: catalog {catalog!r} is not one of {SINK_CATALOG}") + ok = False + vectors = raw.get("vectors") + if vectors not in SINK_VECTORS: + errors.append(f"{where}: vectors {vectors!r} is not one of {SINK_VECTORS} (v1: local only)") + ok = False + plaintext = raw.get("plaintext") + if plaintext != SINK_PLAINTEXT: + errors.append( + f"{where}: plaintext must be {SINK_PLAINTEXT!r} — it is an invariant, not a setting" + ) + ok = False + seal_to = _str_list(where, "seal_to", raw.get("seal_to", []), errors, nonempty=False, hi=128) + if seal_to is None: + ok = False + if not ok: + return None + return Sink(catalog=catalog, vectors=vectors, plaintext=plaintext, seal_to=seal_to) + + +def _validate_grants(raw, owner_host, mask, errors): + if raw is None: + raw = [] + if not isinstance(raw, list): + errors.append("grants: must be a list") + return None + grants = [] + seen = set() + ok = True + for idx, g in enumerate(raw): + where = f"grants[{idx}]" + if not _check_keys(where, g, ("host", "ops", "until"), (), errors): + ok = False + continue + host = g.get("host") + if not _is_str(host, 1, 63) or not SLUG_RE.match(host): + errors.append(f"{where}: host must be a lowercase slug [a-z0-9-]+ of 1-63 chars") + ok = False + elif host == owner_host: + errors.append(f"{where}: host {host!r} is the owner — a grant to yourself is a mistake") + ok = False + elif host in seen: + errors.append(f"{where}: duplicate grant for host {host!r}") + ok = False + seen.add(host) + ops = _str_list(where, "ops", g.get("ops"), errors, allowed=GRANT_OPS) + if ops is None: + ok = False + elif mask is not None: + for op in ops: + need = GRANT_OP_NEEDS_MASK[op] + if need not in mask: + errors.append(f"{where}: op {op!r} requires {need!r} in mask") + ok = False + until = g.get("until") + until_date = None + if isinstance(until, _dt.date) and not isinstance(until, _dt.datetime): + until_date = until # PyYAML parses bare YYYY-MM-DD as a date + elif isinstance(until, str) and DATE_RE.match(until): + try: + until_date = _dt.date.fromisoformat(until) + except ValueError: + pass + if until_date is None: + errors.append(f"{where}: until {until!r} is not a real YYYY-MM-DD date") + ok = False + if ok and until_date is not None and ops is not None: + grants.append(Grant(host=host, ops=ops, until=until_date)) + return tuple(grants) if ok else None + + +def _validate_signature(raw, errors): + where = "signature" + if not _check_keys(where, raw, ("alg", "cert", "value"), (), errors): + return None + ok = True + alg = raw.get("alg") + if alg not in SIGNATURE_ALGS: + errors.append(f"{where}: alg {alg!r} is not one of {SIGNATURE_ALGS}") + ok = False + cert = raw.get("cert") + if not _is_str(cert, 1, 256): + errors.append(f"{where}: cert must be a path string of 1-256 chars") + ok = False + value = raw.get("value") + if not _is_str(value, 1): + errors.append(f"{where}: value must be a non-empty base64 string") + ok = False + else: + try: + base64.b64decode(value, validate=True) + except Exception: + errors.append(f"{where}: value is not valid base64") + ok = False + return Signature(alg=alg, cert=cert, value=value) if ok else None + + +# ── entry point ───────────────────────────────────────────────────────────── + +def validate_document(doc, require_signed=False): + """Validate a parsed manifest document. Raises ManifestError listing all + problems; returns a Manifest otherwise.""" + errors = [] + if not isinstance(doc, dict): + raise ManifestError(["top level must be a map"]) + _check_keys("top level", doc, TOP_KEYS_REQUIRED, TOP_KEYS_OPTIONAL, errors) + + ver = doc.get("freeagent") + if "freeagent" in doc and ver != SCHEMA_VERSION: + errors.append(f"freeagent: schema version {ver!r} is not {SCHEMA_VERSION}") + + manifest_id = doc.get("id") + if "id" in doc and (not _is_str(manifest_id, 1, 63) or not SLUG_RE.match(manifest_id)): + errors.append(f"id {manifest_id!r} is not a lowercase slug [a-z0-9-]+ of 1-63 chars") + manifest_id = None + + owner_host = None + if "owner" in doc and _check_keys("owner", doc.get("owner"), ("host",), (), errors): + owner_host = doc["owner"].get("host") + if not _is_str(owner_host, 1, 63) or not SLUG_RE.match(owner_host): + errors.append(f"owner.host {owner_host!r} is not a lowercase slug [a-z0-9-]+ of 1-63 chars") + owner_host = None + + scope = _validate_scope(doc.get("scope"), errors) if "scope" in doc else None + + mask = None + if "mask" in doc: + mask = _str_list("mask", "mask", doc.get("mask"), errors, allowed=MASK_OPS) + if mask is not None and MASK_MANDATORY not in mask: + errors.append(f"mask: {MASK_MANDATORY!r} is mandatory") + mask = None + + stages = _validate_stages(doc.get("stages"), errors) if "stages" in doc else None + + sweep = doc.get("sweep") + if "sweep" in doc and sweep not in SWEEPS: + errors.append(f"sweep {sweep!r} is not one of {SWEEPS}") + sweep = None + + sink = _validate_sink(doc.get("sink"), errors) if "sink" in doc else None + + # conditional sections — present iff their stage is + sanitize = embed = summarize = None + if stages is not None: + if "sanitize" in stages and "sanitize" not in doc: + errors.append("sanitize: section required when 'sanitize' is in stages") + if "sanitize" not in stages and "sanitize" in doc: + errors.append("sanitize: section present but 'sanitize' is not in stages") + if "embed" in stages and "embed" not in doc: + errors.append("embed: section required when 'embed' is in stages") + if "embed" not in stages and "embed" in doc: + errors.append("embed: section present but 'embed' is not in stages") + if "sanitize" in doc: + sanitize = _validate_sanitize(doc["sanitize"], manifest_id or "", errors) + if "embed" in doc: + embed = _validate_embed(doc["embed"], errors) + if "summarize" in doc: + summarize = _validate_summarize(doc["summarize"], errors) + + grants = _validate_grants(doc.get("grants"), owner_host, mask, errors) + + signature = None + if "signature" in doc: + signature = _validate_signature(doc["signature"], errors) + elif require_signed: + errors.append("signature: required (--require-signed) but absent") + + # cross-section rules — the doctrine + if stages is not None and mask is not None: + for s in stages: + need = STAGE_NEEDS_OP.get(s) + if need and need not in mask: + errors.append(f"stages: {s!r} requires {need!r} in mask") + if "summarize" in doc: + if mask is not None and "summarize" not in mask: + errors.append("summarize: section present but 'summarize' is not in mask") + if stages is not None and "catalog" not in stages: + errors.append("summarize: requires the 'catalog' stage (summaries live in the catalog)") + if stages is not None: + if sink is not None and sink.catalog == "host" and "seal" not in stages: + errors.append("sink: catalog 'host' requires the 'seal' stage — nothing leaves unsealed") + if grants and "seal" not in stages: + errors.append("grants: any grant requires the 'seal' stage — nothing is served unsealed") + if sink is not None and sink.seal_to and "seal" not in stages: + errors.append("sink: seal_to given but 'seal' is not in stages") + + if errors: + raise ManifestError(errors) + return Manifest( + id=manifest_id, + owner_host=owner_host, + scope=scope, + mask=mask, + stages=stages, + sweep=sweep, + sink=sink, + sanitize=sanitize, + embed=embed, + summarize=summarize, + grants=grants, + signature=signature, + raw=doc, + ) + + +def load_manifest(path, require_signed=False): + """Load and validate a manifest file. Raises ManifestError listing all problems.""" + try: + with open(path) as fh: + doc = yaml.safe_load(fh) + except FileNotFoundError: + raise ManifestError([f"manifest file not found: {path}"]) + except yaml.YAMLError as exc: + raise ManifestError([f"not valid YAML: {exc}"]) + return validate_document(doc, require_signed=require_signed) diff --git a/bastion_agent/pdftext.py b/bastion_agent/pdftext.py new file mode 100644 index 0000000..6576359 --- /dev/null +++ b/bastion_agent/pdftext.py @@ -0,0 +1,1002 @@ +"""Text out of a PDF with the standard library — the agent's built-in reader. + +The regex reader it replaces found only literal `(...) Tj` strings, which is +how old PDFs set text. A browser's "Print to PDF" (and most modern writers) +sets text as glyph codes in hex strings through Type0 / Identity-H fonts, +mapped back to Unicode by each font's /ToUnicode CMap — the regex saw none of +it and returned nothing for a browser-printed page. A machine without +poppler (every Windows install) therefore got no text from most real PDFs. + +This reader walks the document the way a viewer does, far enough for text: + + objects `N G obj … endobj`, indexed in one linear pass (a later + definition — an incremental update — wins), plus compressed + object streams (/ObjStm) + streams raw or FlateDecode, each decoded once + pages the /Pages tree in order, with inherited /Resources + fonts /ToUnicode CMaps (bfchar, bfrange, byte widths from the + codespace ranges); a simple font without one by its encoding + (WinAnsi, MacRoman); glyph widths (/Widths, /W), so a word + placed by position alone still gets its space + content BT/ET, Tf, TL, Tj, TJ, ', ", T*, Td/TD/Tm, and form XObjects + (Do) with their own resources; the text state carries across + a page's content streams + +It reads arbitrary files, so it reads on a BUDGET (see Budget): wall time, +decoded bytes, content operators, form invocations and CMap entries are +capped per document, and reaching one stops the read with the text found so +far. Nothing raises out of extract_text; references never loop; encrypted +documents are refused (None). +""" + +import bisect +import re +import time +import zlib + +MAX_SECONDS = 60.0 # wall time, one document +MAX_DECODED = 256 * 1024 * 1024 # decoded stream bytes, one document +MAX_INFLATE = 64 * 1024 * 1024 # decoded bytes, one stream +MAX_OPS = 5_000_000 # content operators, one document +MAX_FORM_CALLS = 2_000 # form XObject invocations, one document +MAX_XOBJECT_DEPTH = 6 +MAX_CMAP_ENTRIES = 200_000 # ToUnicode entries, one document +MAX_NESTING = 200 # dict/array nesting inside one object +MAX_TOKENS = 20_000_000 # tokens lexed, one document +MAX_LEXED = 1024 * 1024 * 1024 # bytes lexed (re-reads included), one document +MAX_ARGS = 64 # operands kept before an operator + +# A gap wider than this share of the font size, on the same line, is a space +# (TJ kerning or an explicit move). With no real glyph widths the pen position +# is a guess, so only a gap past GUESS_GAP_EM is trusted to be a word break. +# A vertical move wider than NEWLINE_EM of the font size (and NEWLINE_Y at +# least) is a new line — a superscript's small rise is not. +GAP_EM = 0.18 +GUESS_GAP_EM = 1.0 +NEWLINE_EM = 0.5 +NEWLINE_Y = 0.5 + +_HDR = re.compile(rb"(\d{1,10})\s+(\d{1,5})\s+obj\b") +_ENDOBJ = re.compile(rb"\bendobj\b") +_ENDSTREAM = re.compile(rb"endstream") +_STREAM_AFTER = re.compile(rb"[ \t\r\n\x0c\x00]*stream(?:\r\n|\n|\r)") +_NUM = re.compile(rb"[+-]?(?:\d+\.?\d*|\.\d+)") +_WS = b" \t\r\n\x0c\x00" +# Runs the lexer skips or takes whole are matched in C: a 64 MB run of +# spaces, a comment, a name or a string body is one regex call, not a +# Python loop per byte. +_WS_RUN = re.compile(rb"[ \t\r\n\x0c\x00]+") +_COMMENT_RUN = re.compile(rb"%[^\r\n]*") +_REGULAR_RUN = re.compile(rb"[^ \t\r\n\x0c\x00()<>\[\]{}/%]*") +_LITERAL_RUN = re.compile(rb"[^()\\]+") +# Everything that is not a hex digit, deleted in C by bytes.translate: a +# regex substitution made one match object per stripped byte (~90 bytes of +# memory each — 3 GB for a 32 MB spaced hex string). +_NON_HEX = bytes(b for b in range(256) if b not in b"0123456789abcdefABCDEF") +_DELIM = b"()<>[]{}/%" + + +def _first_at_or_after(positions, x): + """Index of the first sorted integer position >= x (bisect_left on ints, + spelled so the 3.8 syntax-floor check has nothing to flag).""" + return bisect.bisect_right(positions, x - 1) + + +class _Ref(tuple): + """An indirect reference (num, gen).""" + + +class _Spent(Exception): + """A per-document limit was reached.""" + + +class Budget: + """What one document may cost. Every limit is per document, so a file + built to multiply work (a form drawn by a form drawn by a form…, a CMap + range per line, one huge stream referenced a thousand times) runs out of + budget instead of out of memory or time.""" + + def __init__(self, seconds=MAX_SECONDS): + self.deadline = time.monotonic() + seconds + self.decoded = self.ops = self.forms = self.cmap = self.tokens = 0 + self.lexed = self._lexed_at_check = 0 + self.reason = None + + def _stop(self, reason): + self.reason = reason + raise _Spent(reason) + + def check_time(self): + if time.monotonic() > self.deadline: + self._stop("time") + + def spend_decoded(self, n): + self.decoded += n + if self.decoded > MAX_DECODED: + self._stop("decoded bytes") + self.check_time() + + def spend_op(self): + self.ops += 1 + if self.ops > MAX_OPS: + self._stop("operators") + if not self.ops & 0x3FFF: + self.check_time() + + def spend_token(self, nbytes): + """One token, and the bytes it covered: a single token can be a + 64 MB run of spaces, and a shared stream is re-read per page.""" + self.tokens += 1 + self.lexed += nbytes + if self.tokens > MAX_TOKENS: + self._stop("tokens") + if self.lexed > MAX_LEXED: + self._stop("bytes lexed") + if not self.tokens & 0xFFF or self.lexed - self._lexed_at_check > (1 << 20): + self._lexed_at_check = self.lexed + self.check_time() + + def spend_form(self): + self.forms += 1 + if self.forms > MAX_FORM_CALLS: + self._stop("form invocations") + + def spend_cmap(self, n): + self.cmap += n + if self.cmap > MAX_CMAP_ENTRIES: + self._stop("cmap entries") + + +def _number(word): + """A PDF number: plain decimal only — PDF has no exponents, so `1.e999` + is not a number, and an absurd digit string is 0, never an error.""" + if not _NUM.fullmatch(word): + return None + if len(word) > 20: # no real coordinate or count is this long + return 0 + try: + return float(word) if b"." in word else int(word) + except (ValueError, OverflowError): + return 0 + + +def _is_int(v): + return isinstance(v, int) and not isinstance(v, bool) + + +def _is_num(v): + return isinstance(v, (int, float)) and not isinstance(v, bool) + + +# ── tokenizer / object parser ─────────────────────────────────────────────── + +class _Lexer: + """Tokens of `data[pos:end]`. With a budget, every token is charged to it: + one decoded stream may be lexed once per page, form or font that uses it, + so the cost that has to be bounded is the lexing, not the decode.""" + + def __init__(self, data, pos=0, end=None, budget=None): + self.d = data + self.i = pos + self.n = len(data) if end is None else min(end, len(data)) + self.budget = budget + + def _skip(self): + # Whitespace and comments alternate; each run is one C regex call and + # the loop holds no state per pass (a single repeated-group regex kept + # state per comment line: 5.8 GB for 32 MB of "%\\n"). The clock is + # checked as the loop goes, so one huge run cannot outlast the budget. + d, n, passes = self.d, self.n, 0 + while self.i < n: + m = _WS_RUN.match(d, self.i, n) or _COMMENT_RUN.match(d, self.i, n) + if not m or m.end() == self.i: + break + self.i = m.end() + passes += 1 + if self.budget is not None and not passes & 0xFFF: + self.budget.check_time() + + def token(self): + """Next token: ('num', v) ('name', b) ('str', b) ('op', b) ('[',) + (']',) ('<<',) ('>>',) or None at end.""" + start = self.i + t = self._token() + if self.budget is not None: + self.budget.spend_token(self.i - start) + return t + + def _token(self): + self._skip() + d, n = self.d, self.n + if self.i >= n: + return None + c = d[self.i] + if c == 0x28: # ( + return ("str", self._literal()) + if c == 0x3c: # < + if d[self.i + 1:self.i + 2] == b"<": + self.i += 2 + return ("<<",) + return ("str", self._hex()) + if c == 0x3e and d[self.i + 1:self.i + 2] == b">": + self.i += 2 + return (">>",) + if c == 0x5b: + self.i += 1 + return ("[",) + if c == 0x5d: + self.i += 1 + return ("]",) + if c in b"{}": + self.i += 1 + return ("op", bytes([c])) + if c == 0x2f: # /name + j = _REGULAR_RUN.match(d, self.i + 1, n).end() + name = d[self.i + 1:j] + self.i = j + return ("name", re.sub(rb"#([0-9A-Fa-f]{2})", lambda m: bytes([int(m.group(1), 16)]), name)) + j = _REGULAR_RUN.match(d, self.i, n).end() + if j == self.i: # a stray delimiter: step over it + self.i += 1 + return ("op", bytes([c])) + word = d[self.i:j] + self.i = j + v = _number(word) + return ("op", word) if v is None else ("num", v) + + def _literal(self): + d, n = self.d, self.n + self.i += 1 + depth, out, passes = 1, bytearray(), 0 + while self.i < n: + passes += 1 + if self.budget is not None and not passes & 0xFFFF: + self.budget.check_time() # escapes and parens go byte by byte + m = _LITERAL_RUN.match(d, self.i, n) + if m: + out += m.group(0) + self.i = m.end() + if self.i >= n: + break + c = d[self.i] + if c == 0x5c: # backslash + self.i += 1 + if self.i >= n: + break + e = d[self.i] + if e in b"nrtbf": + out += {0x6e: b"\n", 0x72: b"\r", 0x74: b"\t", 0x62: b"\b", 0x66: b"\f"}[e] + self.i += 1 + elif 0x30 <= e <= 0x37: + m = re.match(rb"[0-7]{1,3}", d[self.i:self.i + 3]) + out.append(int(m.group(0), 8) & 0xFF) + self.i += len(m.group(0)) + elif e in b"\r\n": # line continuation + self.i += 1 + if e == 0x0d and d[self.i:self.i + 1] == b"\n": + self.i += 1 + else: + out.append(e) + self.i += 1 + continue + if c == 0x28: + depth += 1 + elif c == 0x29: + depth -= 1 + if depth == 0: + self.i += 1 + break + out.append(c) + self.i += 1 + return bytes(out) + + def _hex(self): + d = self.d + j = d.find(b">", self.i + 1, self.n) + if j < 0: + j = self.n + hx = d[self.i + 1:j].translate(None, _NON_HEX) + self.i = j + 1 + if len(hx) % 2: + hx += b"0" + return bytes.fromhex(hx.decode("ascii")) + + def obj(self, depth=0): + """One PDF object at the cursor (dicts, arrays, refs as _Ref). + Nesting deeper than MAX_NESTING is a ValueError, never a + RecursionError.""" + if depth > MAX_NESTING: + raise ValueError("nesting too deep") + return self._from(self.token(), depth) + + def _from(self, t, depth): + if t is None: + return None + k = t[0] + if k == "<<": + dct = {} + while True: + kt = self.token() + if kt is None or kt[0] == ">>": + return dct + if kt[0] != "name": + continue + dct[kt[1]] = self.obj(depth + 1) + if k == "[": + arr = [] + while True: + save = self.i + it = self.token() + if it is None or it[0] == "]": + return arr + if it[0] == "op" and it[1] == b"R" and len(arr) >= 2 \ + and _is_int(arr[-1]) and _is_int(arr[-2]): + g, nnum = arr.pop(), arr.pop() + arr.append(_Ref((nnum, g))) + continue + self.i = save + arr.append(self.obj(depth + 1)) + if k == "num": + # `N G R` → a reference + save = self.i + t2 = self.token() + if t2 and t2[0] == "num" and _is_int(t[1]): + t3 = self.token() + if t3 and t3[0] == "op" and t3[1] == b"R": + return _Ref((t[1], t2[1])) + self.i = save + return t[1] + if k in ("name", "str"): + return t[1] + if k == "op": + return {b"true": True, b"false": False, b"null": None}.get(t[1], t[1]) + return None + + +# ── the document ──────────────────────────────────────────────────────────── + +class _Doc: + def __init__(self, data, budget): + self.data = data + self.budget = budget + self.objs = {} # num -> (value, raw stream bytes or None) + self._at = {} # num -> file offset of the definition kept + self._decoded = {} # num -> decoded stream bytes (or None) + self.cmaps = {} # ToUnicode stream num -> (map, code widths) + self._index(data) + for num, (val, raw) in list(self.objs.items()): + if isinstance(val, dict) and val.get(b"Type") == b"ObjStm" and raw is not None: + self._unpack_objstm(num, val) + + def _index(self, data): + # One linear pass: object headers, `endobj`s and `endstream`s are + # each found once and paired by bisection, so a file with a missing + # `endobj` (or thousands of them) costs the same as a clean one. + hdrs = list(_HDR.finditer(data)) + ends = [m.start() for m in _ENDOBJ.finditer(data)] + estreams = [m.start() for m in _ENDSTREAM.finditer(data)] + skip = 0 + for idx, m in enumerate(hdrs): + if not idx & 0x3FF: + self.budget.check_time() + if m.start() < skip: # inside the previous stream's bytes + continue + k = _first_at_or_after(ends, m.end()) + stop = ends[k] if k < len(ends) else len(data) + nxt = hdrs[idx + 1].start() if idx + 1 < len(hdrs) else len(data) + # The object first, bounded by its own endobj (or the next header): + # only a DICTIONARY followed directly by the `stream` keyword owns + # stream bytes — the word inside a string or a name does not. + lx = _Lexer(data, m.end(), min(stop, nxt), budget=self.budget) + try: + val = lx.obj() + except ValueError: # one bad object is skipped, not fatal + continue + raw = None + sm = _STREAM_AFTER.match(data, lx.i) if isinstance(val, dict) else None + if sm: + start = sm.end() + end = None + ln = val.get(b"Length") + if _is_int(ln) and ln >= 0 and \ + data[start + ln:start + ln + 12].lstrip(_WS).startswith(b"endstream"): + end = start + ln + if end is None: # indirect or wrong /Length: the keyword + j = _first_at_or_after(estreams, start) + end = estreams[j] if j < len(estreams) else len(data) + if data[end - 2:end] == b"\r\n": + end -= 2 + elif data[end - 1:end] in (b"\n", b"\r"): + end -= 1 + raw = data[start:end] + skip = end + self._put(int(m.group(1)), val, raw, m.start()) + + def _put(self, num, val, raw, at): + # Incremental updates APPEND a new definition: the later one wins. + if num in self._at and self._at[num] > at: + return + self.objs[num] = (val, raw) + self._at[num] = at + self._decoded.pop(num, None) + + def _unpack_objstm(self, snum, dct): + data = self.stream(_Ref((snum, 0))) + if data is None: + return + n, first = self.get(dct.get(b"N")), self.get(dct.get(b"First")) + if not (_is_int(n) and _is_int(first)) or n <= 0 or not 0 <= first <= len(data): + return + lx = _Lexer(data, budget=self.budget) + pairs = [] + for _ in range(min(n, len(data))): + a, b = lx.token(), lx.token() + if not a or not b or a[0] != "num" or b[0] != "num" \ + or not _is_int(a[1]) or not _is_int(b[1]): + break + pairs.append((a[1], b[1])) + at = self._at.get(snum, 0) + for num, off in pairs: + if not 0 <= off < len(data) - first: + continue + try: + val = _Lexer(data, first + off, budget=self.budget).obj() + except ValueError: + continue + self._put(num, val, None, at) + + def get(self, v): + """Resolve references (cycle-safe).""" + seen = set() + while isinstance(v, _Ref): + if v[0] in seen: + return None + seen.add(v[0]) + v = self.objs.get(v[0], (None, None))[0] + return v + + def stream(self, v): + """The decoded bytes of a stream object (by ref), or None. Decoded + once per document; the cost counts against the budget.""" + if not isinstance(v, _Ref): + return None + num = v[0] + if num in self._decoded: + return self._decoded[num] + dct, raw = self.objs.get(num, (None, None)) + out = None + if raw is not None and isinstance(dct, dict): + out = self._decode(dct, raw) + if out is not None: + self.budget.spend_decoded(len(out)) + self._decoded[num] = out + return out + + def _decode(self, dct, raw): + f = self.get(dct.get(b"Filter")) + filters = f if isinstance(f, list) else ([f] if f else []) + data = raw + for flt in filters: + flt = self.get(flt) + if flt in (b"FlateDecode", b"Fl"): + try: + data = zlib.decompressobj().decompress(data, MAX_INFLATE) + except zlib.error: + return None + else: + return None # a filter we do not read: skip the stream + return data + + def pages(self): + """(page dict, resources) in document order, resources inherited.""" + root = None + for val, _ in self.objs.values(): + if isinstance(val, dict) and val.get(b"Type") == b"Catalog": + root = self.get(val.get(b"Pages")) + break + out = [] + if isinstance(root, dict): + self._walk_pages(root, None, out, set(), 0) + if not out: # no usable tree: every page object, in file order + for val, _ in self.objs.values(): + if isinstance(val, dict) and val.get(b"Type") == b"Page": + res = self.get(val.get(b"Resources")) + out.append((val, res if isinstance(res, dict) else None)) + return out + + def _walk_pages(self, node, res, out, seen, depth): + if id(node) in seen or depth > 64: + return + seen.add(id(node)) + r = self.get(node.get(b"Resources")) + if isinstance(r, dict): + res = r + kids = self.get(node.get(b"Kids")) + if node.get(b"Type") == b"Page" or not isinstance(kids, list): + out.append((node, res)) + return + for k in kids: + kid = self.get(k) + if isinstance(kid, dict): + self._walk_pages(kid, res, out, seen, depth + 1) + + +# ── fonts ─────────────────────────────────────────────────────────────────── + +_SIMPLE_CODECS = {b"WinAnsiEncoding": "cp1252", b"MacRomanEncoding": "mac_roman"} + + +def _widths(text): + return [int(x) for x in text.split()] + + +# THE STANDARD FONTS' OWN WIDTHS, for a simple font that names one and ships +# no /Widths (legal before PDF 1.5, and common in old files). Codes 32-126 in +# StandardEncoding, from the metric-compatible URW base-35 AFMs (NimbusSans = +# Helvetica, NimbusRoman = Times); Courier is 600 for every glyph. Real widths +# are what let a gap between words placed by position read as a space without +# inventing one inside a word of wide letters. +_STD_WIDTHS = { + "Helvetica": _widths(""" + 278 278 355 556 556 889 667 222 333 333 389 584 278 333 278 278 556 556 556 + 556 556 556 556 556 556 556 278 278 584 584 584 556 1015 667 667 722 722 667 + 611 778 722 278 500 667 556 833 722 778 667 778 722 667 611 722 667 944 667 + 667 611 278 278 278 469 556 222 556 556 500 556 556 278 556 556 222 222 500 + 222 833 556 556 556 556 333 500 278 556 500 722 500 500 500 334 260 334 584"""), + "Helvetica-Bold": _widths(""" + 278 333 474 556 556 889 722 278 333 333 389 584 278 333 278 278 556 556 556 + 556 556 556 556 556 556 556 333 333 584 584 584 611 975 722 722 722 722 667 + 611 778 722 278 556 722 611 833 722 778 667 778 722 667 611 722 667 944 667 + 667 611 333 278 333 584 556 278 556 611 556 611 556 333 611 611 278 278 556 + 278 889 611 611 611 611 389 556 333 611 556 778 556 556 500 389 280 389 584"""), + "Times": _widths(""" + 250 333 408 500 500 833 778 333 333 333 500 564 250 333 250 278 500 500 500 + 500 500 500 500 500 500 500 278 278 564 564 564 444 921 722 667 667 722 611 + 556 722 722 333 389 722 611 889 722 722 556 722 667 556 611 722 722 944 722 + 722 611 333 278 333 469 500 333 444 500 444 500 444 333 500 500 278 278 500 + 278 778 500 500 500 500 333 389 278 500 500 722 500 500 444 480 200 480 541"""), + "Times-Bold": _widths(""" + 250 333 555 500 500 1000 833 333 333 333 500 570 250 333 250 278 500 500 500 + 500 500 500 500 500 500 500 333 333 570 570 570 500 930 722 667 722 722 667 + 611 778 778 389 500 778 667 944 722 778 611 778 722 556 667 722 722 1000 722 + 722 667 333 278 333 581 500 333 500 556 444 556 444 333 500 556 278 333 556 + 278 833 556 500 556 556 444 389 333 556 500 722 500 500 444 394 220 394 520"""), +} + + +def _standard_widths(base): + """(first code, widths) for a standard-14 BaseFont name, or None.""" + name = base.split(b"+", 1)[-1].decode("latin-1", "replace").replace(",", "-") + if name.startswith("Courier"): + return 32, [600] * 95 + family = ("Helvetica" if name.startswith(("Helvetica", "Arial")) else + "Times" if name.startswith("Times") else None) + if family is None: + return None + bold = "Bold" in name or "Black" in name + return 32, _STD_WIDTHS[family + ("-Bold" if bold else "")] + + +class _Font: + def __init__(self, doc, fdict): + self.map = {} # code bytes -> str + self.widths = [1] # byte lengths codes come in + self.cid = fdict.get(b"Subtype") == b"Type0" + self.codec = "cp1252" # a simple font with no map (StandardEncoding ≈ ASCII) + self.adv = {} # code -> advance, 1/1000 of the font size + self.default_adv = 1000 if self.cid else 500 + enc = doc.get(fdict.get(b"Encoding")) + if isinstance(enc, dict): + enc = doc.get(enc.get(b"BaseEncoding")) + if isinstance(enc, bytes) and enc in _SIMPLE_CODECS: + self.codec = _SIMPLE_CODECS[enc] + if self.cid: + self.widths = [2] + self._cid_widths(doc, fdict) + else: + self._simple_widths(doc, fdict) + if not self.adv: + # No widths and not a standard font (a spec violation, but + # seen): guess wide, so the guessed pen never falls short of + # a real glyph and invents a space inside a word. + self.default_adv = 1000 + tu = fdict.get(b"ToUnicode") + if isinstance(tu, _Ref): + # One CMap is often shared by many fonts: parse it once. + cached = doc.cmaps.get(tu[0]) + if cached is None: + cmap = doc.stream(tu) + if cmap: + self._parse(cmap, doc.budget) + cached = doc.cmaps[tu[0]] = (self.map, self.widths) + self.map, self.widths = cached + + def _simple_widths(self, doc, fdict): + first = doc.get(fdict.get(b"FirstChar")) + ws = doc.get(fdict.get(b"Widths")) + if not isinstance(ws, list): + base = doc.get(fdict.get(b"BaseFont")) + std = _standard_widths(base) if isinstance(base, bytes) else None + if std: + first, ws = std + if _is_int(first) and isinstance(ws, list): + for i, w in enumerate(ws[:256]): + w = doc.get(w) + if _is_num(w): + self.adv[first + i] = w + + def _cid_widths(self, doc, fdict): + ds = doc.get(fdict.get(b"DescendantFonts")) + d0 = doc.get(ds[0]) if isinstance(ds, list) and ds else None + if not isinstance(d0, dict): + return + dw = doc.get(d0.get(b"DW")) + if _is_num(dw): + self.default_adv = dw + w = doc.get(d0.get(b"W")) + if not isinstance(w, list): + return + # [c [w1 w2 …]] or [c_first c_last w], repeated. + i = 0 + while i < len(w) and len(self.adv) < 65536: + c = doc.get(w[i]) + nxt = doc.get(w[i + 1]) if i + 1 < len(w) else None + if _is_int(c) and isinstance(nxt, list): + for k, val in enumerate(nxt[:65536]): + val = doc.get(val) + if _is_num(val): + self.adv[c + k] = val + i += 2 + elif _is_int(c) and _is_int(nxt) and i + 2 < len(w): + val = doc.get(w[i + 2]) + if _is_num(val) and 0 <= nxt - c < 65536: + for code in range(c, nxt + 1): + self.adv[code] = val + i += 3 + else: + break + + def _parse(self, cmap, budget): + lx = _Lexer(cmap, budget=budget) + widths = set() + while True: + t = lx.token() + if t is None: + break + if not (t[0] == "op" and t[1] in (b"begincodespacerange", b"beginbfchar", b"beginbfrange")): + continue + mode, stack = t[1], [] + while True: + u = lx.token() + if u is None or (u[0] == "op" and u[1].startswith(b"end")): + break + if u[0] == "[": + arr = [] + while True: + v = lx.token() + if v is None or v[0] == "]": + break + if v[0] == "str": + arr.append(v[1]) + stack.append(arr) + elif u[0] == "str": + stack.append(u[1]) + if mode == b"begincodespacerange": + for lo in stack[0::2]: + if isinstance(lo, bytes) and 0 < len(lo) <= 4: + widths.add(len(lo)) + elif mode == b"beginbfchar": + for src, dst in zip(stack[0::2], stack[1::2]): + if isinstance(src, bytes) and isinstance(dst, bytes) and 0 < len(src) <= 4: + budget.spend_cmap(1) + self.map[src] = _utf16(dst) + widths.add(len(src)) + else: + for i in range(0, len(stack) - 2, 3): + lo, hi, dst = stack[i], stack[i + 1], stack[i + 2] + if not (isinstance(lo, bytes) and isinstance(hi, bytes)) \ + or len(lo) != len(hi) or not 0 < len(lo) <= 4: + continue + a, b = int.from_bytes(lo, "big"), int.from_bytes(hi, "big") + if not 0 <= b - a < 65536: + continue + budget.spend_cmap(b - a + 1) + widths.add(len(lo)) + for k, code in enumerate(range(a, b + 1)): + key = code.to_bytes(len(lo), "big") + if isinstance(dst, list): + if k >= len(dst): + break + self.map[key] = _utf16(dst[k]) if isinstance(dst[k], bytes) else "" + elif isinstance(dst, bytes) and 0 < len(dst) <= 64: + base = int.from_bytes(dst, "big") + k + if base >= 1 << (8 * len(dst)): # the range runs past its width + break + self.map[key] = _utf16(base.to_bytes(len(dst), "big")) + if widths: + self.widths = sorted(widths, reverse=True) + + def codes(self, s): + """[(code, text)] for one shown string, code for the glyph width.""" + if not self.map: + if self.cid: # CID codes with no map: widths, no honest text + return [(int.from_bytes(s[i:i + 2], "big"), "") for i in range(0, len(s) - 1, 2)] + return [(b, bytes([b]).decode(self.codec, "replace")) for b in s] + out, i = [], 0 + while i < len(s): + for w in self.widths: + key = s[i:i + w] + if len(key) == w and key in self.map: + out.append((int.from_bytes(key, "big"), self.map[key])) + i += w + break + else: + w = self.widths[-1] + out.append((int.from_bytes(s[i:i + w], "big"), "")) + i += w + return out + + def advance(self, code): + return self.adv.get(code, self.default_adv) + + +def _utf16(b): + try: + return b.decode("utf-16-be") + except UnicodeDecodeError: + return "" + + +# ── content ───────────────────────────────────────────────────────────────── + +class _Text: + """The text state, carried across one page's content streams (a writer + may set the font in one stream and show text in the next). + + Positions are in text space scaled by the text matrix (a, d); that is + enough to tell a new line (y moved) from a new run on the same line, and + a run placed after a visible gap (a word set by position, not by a + space) from a run that continues the word.""" + + def __init__(self): + self.font = None + self.size = 0.0 + self.leading = 0.0 + self.a = self.d = 1.0 + self.lx = self.ly = 0.0 # start of the current line (Td/T*/Tm) + self.x = 0.0 # pen: where the next glyph goes + self.end_x = None # where the last shown glyph ended… + self.end_y = None # …and on which line + self.tail = "" # last character written + + def begin(self): # BT: the text matrix is the identity again + self.a = self.d = 1.0 + self.lx = self.ly = self.x = 0.0 + + def move(self, tx, ty): # Td + self.lx += tx * self.a + self.ly += ty * self.d + self.x = self.lx + + def next_line(self): # T* + self.move(0, -(self.leading or self.size * 1.2 or 1.0)) + + def matrix(self, a, d, e, f): # Tm + self.a = a or 1.0 + self.d = d or 1.0 + self.lx = self.x = e + self.ly = f + + def em(self): + return abs(self.size * self.a) + + +class _Reader: + def __init__(self, doc): + self.doc = doc + self.budget = doc.budget + self.fonts = {} # id(font dict) -> _Font + + def _font(self, res, name): + fonts = self.doc.get(res.get(b"Font")) if isinstance(res, dict) else None + fd = self.doc.get(fonts.get(name)) if isinstance(fonts, dict) else None + if not isinstance(fd, dict): + return None + key = id(fd) + if key not in self.fonts: + try: + self.fonts[key] = _Font(self.doc, fd) + except _Spent: + raise + except Exception: # noqa: BLE001 — a malformed font reads as no font + self.fonts[key] = None + return self.fonts[key] + + @staticmethod + def _emit(st, out, text): + if text: + out.append(text) + st.tail = text[-1] + + def _show(self, st, s, out): + font = st.font + if st.end_y is not None: + if abs(st.ly - st.end_y) > max(NEWLINE_Y, NEWLINE_EM * st.em()): + if st.tail != "\n": + self._emit(st, out, "\n") + elif not st.tail.isspace(): + # Same line, placed after a gap. With real widths a small gap + # is a word break; with guessed widths only a wide one is, so + # the guess never invents a space inside a word. + real = font is not None and font.adv + if st.x - st.end_x > (GAP_EM if real else GUESS_GAP_EM) * st.em(): + self._emit(st, out, " ") + if font is None: + text = s.decode("cp1252", "replace") + self._emit(st, out, text) + st.x += len(s) * 0.5 * st.em() + else: + adv = 0.0 + for code, text in font.codes(s): + self._emit(st, out, text) + adv += font.advance(code) + st.x += adv / 1000.0 * st.em() + st.end_x, st.end_y = st.x, st.ly + + def content(self, data, res, out, st, depth=0): + budget = self.budget + lx = _Lexer(data, budget=budget) + args = [] + while True: + t = lx.token() + if t is None: + break + k = t[0] + if k == "[": + arr = [] + while True: + u = lx.token() + if u is None or u[0] == "]": + break + if u[0] in ("str", "num"): + arr.append(u[1]) + args.append(arr) + continue + if k == "<<": # inline dictionaries (marked content): skip + depth_d = 1 + while depth_d: + u = lx.token() + if u is None: + break + depth_d += {"<<": 1, ">>": -1}.get(u[0], 0) + continue + if k != "op": + args.append(t[1] if len(t) > 1 else None) + if len(args) > MAX_ARGS: # operands with no operator: keep the tail + del args[:-MAX_ARGS] + continue + budget.spend_op() + op = t[1] + if op == b"Tf" and len(args) >= 2 and isinstance(args[-2], bytes): + st.font = self._font(res, args[-2]) + if _is_num(args[-1]): + st.size = float(args[-1]) + elif op == b"TL" and args and _is_num(args[-1]): + st.leading = float(args[-1]) + elif op in (b"Tj", b"'", b'"') and args and isinstance(args[-1], bytes): + if op != b"Tj": + st.next_line() + self._show(st, args[-1], out) + elif op == b"TJ" and args and isinstance(args[-1], list): + for el in args[-1]: + if isinstance(el, bytes): + self._show(st, el, out) + elif _is_num(el): + st.x -= el / 1000.0 * st.em() + if el < -GAP_EM * 1000 and not st.tail.isspace() and st.tail: + self._emit(st, out, " ") + elif op == b"T*": + st.next_line() + elif op in (b"Td", b"TD") and len(args) >= 2 and _is_num(args[-2]) and _is_num(args[-1]): + if op == b"TD": + st.leading = -float(args[-1]) + st.move(float(args[-2]), float(args[-1])) + elif op == b"Tm" and len(args) >= 6 and all(_is_num(v) for v in args[-6:]): + a, _b, _c, d, e, f = (float(v) for v in args[-6:]) + st.matrix(a, d, e, f) + elif op == b"BT": + st.begin() + elif op == b"Do" and args and isinstance(args[-1], bytes) and depth < MAX_XOBJECT_DEPTH: + xos = self.doc.get(res.get(b"XObject")) if isinstance(res, dict) else None + ref = xos.get(args[-1]) if isinstance(xos, dict) else None + xo = self.doc.get(ref) + if isinstance(xo, dict) and xo.get(b"Subtype") == b"Form": + budget.spend_form() + sub = self.doc.stream(ref) + if sub: + r = self.doc.get(xo.get(b"Resources")) + self.content(sub, r if isinstance(r, dict) else res, out, st, depth + 1) + elif op == b"BI": # inline image: skip its data + end = data.find(b"EI", lx.i) + lx.i = len(data) if end < 0 else end + 2 + args = [] + + +def extract_text(data, budget=None): + """The document's text, pages separated by form feeds, or None when it is + encrypted or holds no text a reader could recover. + + Never raises: the input is whatever file a user had. A spent budget + returns the pages read so far; any other failure is None.""" + budget = budget or Budget() + pages_out = [] # finished pages, then the page being read + try: + return _extract(data, budget, pages_out) + except _Spent: + text = "\f".join(p if isinstance(p, str) else _tidy("".join(p)) for p in pages_out) + return text if text.strip() else None + except Exception: # noqa: BLE001 — a malformed file is "no text", never a crash + return None + + +def _read_isolated(rd, stream, res, buf, st): + """One content stream. A malformed value in it ends THIS stream's read + with the text found so far; the page's other streams and the document's + other pages still read. A spent budget still stops everything.""" + try: + rd.content(stream, res, buf, st) + except _Spent: + raise + except Exception: # noqa: BLE001 + pass + + +def _extract(data, budget, pages_out): + if re.search(rb"/Encrypt(?![A-Za-z])", data): + return None + doc = _Doc(data, budget) + rd = _Reader(doc) + read_any = False + for page, res in doc.pages(): + # /Contents is one stream reference, or an array of them (possibly + # itself behind a reference). + contents = page.get(b"Contents") + resolved = doc.get(contents) + refs = resolved if isinstance(resolved, list) else [contents] + buf = [] + pages_out.append(buf) + st = _Text() + for r in refs: + s = doc.stream(r) + if s: + read_any = True + _read_isolated(rd, s, res, buf, st) + pages_out[-1] = _tidy("".join(buf)) + text = "\f".join(pages_out) + if not read_any: + # No page tree a reader could follow (a damaged or hand-made file): + # read every stream that carries text operators, in file order, the + # way the old regex reader did — fonts unknown. Only then: pages that + # WERE read and hold no text (a CID font with no map, a scan) must not + # be re-read without their fonts, which turns glyph codes into junk. + found = [] + for num in sorted(doc.objs): + s = doc.stream(_Ref((num, 0))) + if s and b"BT" in s and (b"Tj" in s or b"TJ" in s): + buf = [] + _read_isolated(rd, s, None, buf, _Text()) + found.append(_tidy("".join(buf))) + text = "\n".join(t for t in found if t) + return text if text.strip() else None + + +_LIGATURES = {"ff": "ff", "fi": "fi", "fl": "fl", "ffi": "ffi", + "ffl": "ffl", "ſt": "st", "st": "st", " ": " "} + + +def _tidy(t): + for a, b in _LIGATURES.items(): + t = t.replace(a, b) + t = re.sub(r"[ \t]+", " ", t) + t = re.sub(r" *\n *", "\n", t) + return re.sub(r"\n{3,}", "\n\n", t).strip() diff --git a/bastion_agent/publish.sh b/bastion_agent/publish.sh new file mode 100755 index 0000000..6583935 --- /dev/null +++ b/bastion_agent/publish.sh @@ -0,0 +1,299 @@ +#!/usr/bin/env bash +# Publish the bastion agent distributable to the public edges. It is served at +# BOTH addresses, because one of them is compiled into every bastion already +# installed: +# +# new https://privatae.ai/bastion-agent/version.json +# legacy https://privatae.ai/freeagent/version.json +# +# Those are two Caddy paths over ONE directory on each edge (WEBROOT below). +# There is a single copy of every file; the new address is an alias, so the two +# addresses can never drift or fall out of step at release time. +# +# THE LEGACY NAME, and when it ends. A bastion older than 0.20.0 knows only the +# legacy address, builds the artifact's filename itself and asks the manifest +# for `freeagent-.pyz`. If that address answers with a manifest that lists +# only the new name, the bastion gets "no entry" and loses the agent; if the +# address does not answer at all, the bastion keeps the agent it has. So: +# +# - while the legacy address is served, every publish writes the SAME BYTES +# under BOTH filenames and lists both, each with the same sha256; +# - once it is not served (it answers 404), a publish writes the new name only. +# +# The script reads which of the two is true on every run, on every edge, so it +# cannot publish the one combination that strands a machine. Retiring the legacy address is a +# step on the edges, taken once the published bastion is 0.42.0 or later (the +# first release that never reads it); docs/packaging.md has it. +# +# The bastion's own deploy/publish.sh is the model: build +# reproducibly, write a manifest with a sha256 per file, upload to EVERY edge, +# then verify each edge's bytes against the manifest IT serves. Same key, same +# edges, same webroot walls (Caddy refuses backups and dotfiles under both paths). +# +# Ratchet: refuse to publish a version that is not strictly newer than the one +# already published (a rebuilt older version would regress installers that +# compare versions). Override deliberately with ALLOW_REPUBLISH=1. +set -euo pipefail +# Repo root. This script sits in bastion_agent/, which is a top-level package +# here, hence one `..`. +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# WHERE THIS PUBLISHES IS NOT BAKED IN. This repository is read by everyone who +# runs the agent, so it names no host, no key path and no webroot of ours: the +# three below are required, and an unset one stops the script with a message +# rather than publishing somewhere wrong. Whoever cuts a release supplies them. +: "${KEY:?set KEY to the ssh key that reaches the edges}" +# EDGE (one box) satisfies the requirement on its own -- it is the documented +# single-box override below, and checking EDGES first would kill it with an error +# naming a variable the operator was never told to set. +if [ -n "${EDGE:-}" ]; then EDGES="$EDGE"; fi +: "${EDGES:?set EDGES to the space-separated edge hosts to publish to (or EDGE= for a single box)}" +: "${WEBROOT:?set WEBROOT to the directory on each edge that both URLs serve}" +: "${BACKUP_DIR:?set BACKUP_DIR to a directory OUTSIDE the webroot for rotated installers}" +# OUTSIDE is load-bearing: a backup inside the webroot is served at a stable public +# URL, which is how a superseded installer stays reachable forever. Assert it. +case "$BACKUP_DIR" in + "$WEBROOT"|"$WEBROOT"/*) + echo "BACKUP_DIR ($BACKUP_DIR) is inside WEBROOT ($WEBROOT) -- rotated installers would be served publicly" >&2 + exit 1;; +esac +# The manifest is signed with the bastion's release key, which never comes to +# this machine: it stays on the key holder's own computer. So a publish runs +# twice. The first run builds, writes the manifest and prints the one +# release-sign command to run there; the second, with SIGNATURE=, rebuilds (the build is reproducible, so the manifest is the same +# bytes), checks the signature against the keys the bastion pins, and +# publishes. BASTION_TREE is a checkout of privatae-bastion at the release the +# fleet is moving to: its deploy/release-verify.sh and pinned keys are what a +# bastion will check this manifest against. +: "${BASTION_TREE:?set BASTION_TREE to a privatae-bastion checkout (its deploy/release-verify.sh checks the signature)}" +[ -x "$BASTION_TREE/deploy/release-verify.sh" ] || { echo "$BASTION_TREE has no deploy/release-verify.sh" >&2; exit 1; } +# The bastions that will read this manifest run the published bastion release, +# and check it with the keys THAT release pins. So the signature is held to +# those keys too, not only to BASTION_TREE's. +FLEET_FROM="$(curl -fsS --max-time 15 https://privatae.ai/bastion/version.json | python3 -c 'import sys,json;print(json.load(sys.stdin).get("built_from",""))' 2>/dev/null || true)" +git -C "$BASTION_TREE" fetch -q origin 2>/dev/null || true +if [ -n "$FLEET_FROM" ] && ! git -C "$BASTION_TREE" cat-file -e "$FLEET_FROM^{commit}" 2>/dev/null; then + echo "BASTION_TREE does not have the published bastion commit $FLEET_FROM; fetch it first" >&2 + exit 1 +fi +BASE_NEW="https://privatae.ai/bastion-agent" +BASE_OLD="https://privatae.ai/freeagent" + +VER="$(tr -d '[:space:]' < "$HERE/bastion_agent/VERSION")" +# Is the legacy address still served? It is asked of EVERY edge, not only of +# the one the public name reaches now: each edge serves from its own disk and +# its own Caddy config, and an edge that still has the route would serve a +# manifest without the legacy name the day traffic fails over to it. Each edge +# is asked for the route in the config Caddy has loaded (its local admin API). +# Anything but a clear "no route" (an answer, a timeout, an error) counts as +# served, because writing the legacy name when it is not needed costs one extra +# file, and leaving it out when it is needed costs every old bastion its agent. +# The public address is asked too, and a 404 there is also required. +# Counts the routes whose path matcher would take /freeagent/version.json, +# however the matcher is spelled (/freeagent/*, /freeagent*, a regexp), which +# covers handle, handle_path and redir alike. Anything unreadable is "unknown". +ROUTES_PY=' +import fnmatch, json, re, sys +try: + cfg = json.load(sys.stdin) +except Exception: + print("unknown"); sys.exit() +if not isinstance(cfg, dict) or "apps" not in cfg: + print("unknown"); sys.exit() +target = "/freeagent/version.json" +n = 0 +def walk(o): + global n + if isinstance(o, dict): + for k, v in o.items(): + if k == "path" and isinstance(v, list) and any(isinstance(p, str) and fnmatch.fnmatchcase(target, p) and p not in ("*", "/*") for p in v): + n += 1 + elif k == "path_regexp" and isinstance(v, dict) and isinstance(v.get("pattern"), str): + try: + if re.search(v["pattern"], target) and "freeagent" in v["pattern"].lower(): + n += 1 + except re.error: + n += 1 + walk(v) + elif isinstance(o, list): + for x in o: + walk(x) +walk(cfg) +print(n) +' +LEGACY=0 +for E in $EDGES; do + n="$(ssh -n -i "$KEY" "$E" 'curl -fsS --max-time 10 http://localhost:2019/config/' 2>/dev/null \ + | python3 -c "$ROUTES_PY" 2>/dev/null || true)" + if [ "$n" = 0 ]; then echo " $E: no /freeagent route" + else LEGACY=1; echo " $E: /freeagent route ${n:-unknown} (served)"; fi +done +LEGACY_CODE="$(curl -sS -o /dev/null -w '%{http_code}' --max-time 15 "$BASE_OLD/version.json" 2>/dev/null || true)" +[ "$LEGACY_CODE" = 404 ] || LEGACY=1 +if [ "$LEGACY" = 0 ]; then echo "legacy address retired on every edge (public: 404): publishing the new name only" +else echo "legacy address still served somewhere (public: HTTP ${LEGACY_CODE:-none}): publishing both names"; fi +GITSHA="$(git -C "$HERE" rev-parse --short HEAD 2>/dev/null || echo unknown)" +echo "publishing bastion agent v$VER ($GITSHA)" + +# Both addresses serve one directory, so the new one is the record of what is +# published, and it stays served after the legacy one is retired. +CUR="$(curl -fsS --max-time 15 "$BASE_NEW/version.json" 2>/dev/null | python3 -c 'import sys,json;print(json.load(sys.stdin).get("version",""))' 2>/dev/null || true)" +if [ -n "$CUR" ]; then + newer="$(python3 -c 'import sys +a=[int(x) for x in sys.argv[1].split(".")]; b=[int(x) for x in sys.argv[2].split(".")] +print("yes" if a>b else "no")' "$VER" "$CUR")" + if [ "$newer" = yes ]; then echo "ratchet ok: $CUR -> $VER"; else + echo "RATCHET REFUSED: v$VER is not newer than published v$CUR (ALLOW_REPUBLISH=1 to override)" >&2 + [ "${ALLOW_REPUBLISH:-}" = "1" ] || exit 1 + fi +fi + +DIST="$HERE/dist" +(cd "$HERE" && python3 -m bastion_agent.build --out dist >/dev/null) +PYZ="$DIST/bastion-agent-$VER.pyz" +[ -f "$PYZ" ] || { echo "build did not produce $PYZ" >&2; exit 1; } +python3 "$PYZ" --version >/dev/null + +UPLOAD=("$PYZ" "$PYZ.sha256") +if [ "$LEGACY" = 1 ]; then + # The same bytes under the legacy filename, for bastions that ask for it by name. + LEGACY_PYZ="$DIST/freeagent-$VER.pyz" + cp -f "$PYZ" "$LEGACY_PYZ" + sed "s|bastion-agent-$VER.pyz|freeagent-$VER.pyz|" "$PYZ.sha256" > "$LEGACY_PYZ.sha256" + # Prove they are the same artifact rather than assuming the copy worked. + a="$(python3 -c 'import hashlib,sys;print(hashlib.sha256(open(sys.argv[1],"rb").read()).hexdigest())' "$PYZ")" + b="$(python3 -c 'import hashlib,sys;print(hashlib.sha256(open(sys.argv[1],"rb").read()).hexdigest())' "$LEGACY_PYZ")" + [ "$a" = "$b" ] || { echo "the two filenames do not carry the same bytes" >&2; exit 1; } + echo "artifact: $a (published under both names)" + UPLOAD+=("$LEGACY_PYZ" "$LEGACY_PYZ.sha256") +fi + +python3 - "$VER" "$GITSHA" "$DIST" "$HERE/bastion_agent" "$BASE_NEW" "$BASE_OLD" "$LEGACY" > "$DIST/version.json" <<'PY' +import sys, os, hashlib, json +ver, sha, dist, pkg, base_new, base_old, legacy = sys.argv[1:8] +def h(p): return hashlib.sha256(open(p, "rb").read()).hexdigest() +def entry(name, base): + p = os.path.join(dist, name) + return name, {"url": "%s/dl/%s" % (base, name), "sha256": h(p), "bytes": os.path.getsize(p)} + +files = {} +names = [("bastion-agent-%s.pyz" % ver, base_new), + ("bastion-agent-%s.pyz.sha256" % ver, base_new)] +# While the legacy address is served, the legacy pair is what a bastion older +# than 0.20.0 asks for by name; leave it out and that bastion loses the agent. +if legacy == "1": + names += [("freeagent-%s.pyz" % ver, base_old), + ("freeagent-%s.pyz.sha256" % ver, base_old)] +for name, base in names: + k, v = entry(name, base) + files[k] = v + +installers = {n: {"url": "%s/%s" % (base_new, n), "sha256": h(os.path.join(pkg, n))} + for n in ("install.sh", "install.ps1")} +json.dump({"version": ver, "built_from": sha, "python_floor": "3.8", + "files": files, "installers": installers, + "notes": "https://git.privatae.ai/privatae/bastion-agent/releases/tag/v%s" % ver}, + sys.stdout, indent=2) +PY + +MAN="$DIST/version.json" +MAN_SHA="$(python3 -c 'import hashlib,sys;print(hashlib.sha256(open(sys.argv[1],"rb").read()).hexdigest())' "$MAN")" +if [ -z "${SIGNATURE:-}" ]; then + cat < "$DIST/$SIG_NAME" +SIGNED_BY="$("$BASTION_TREE/deploy/release-verify.sh" bastion-agent "$MAN" "$DIST/$SIG_NAME" ${FLEET_FROM:+"$FLEET_FROM"})" \ + || { echo "SIGNATURE does not verify for bastion agent v$VER (manifest $MAN_SHA); nothing was published." >&2 + echo "If the key holder signed the command the first run printed, compare the 'legacy address' lines of the two runs: a change on the edges between them changes the manifest." >&2 + exit 1; } +echo "manifest signature ok (key $SIGNED_BY)" + +for E in $EDGES; do + echo "--- publishing to $E ---" + ssh -i "$KEY" "$E" 'rm -rf /tmp/ba && mkdir -p /tmp/ba' + scp -q -i "$KEY" "${UPLOAD[@]}" "$DIST/version.json" "$DIST/$SIG_NAME" \ + "$HERE/bastion_agent/install.sh" "$HERE/bastion_agent/install.ps1" "$E":/tmp/ba/ + ssh -i "$KEY" "$E" " + set -e + install -d -m755 \"$WEBROOT/dl\" + install -d -m700 \"$BACKUP_DIR\" + ts=\$(date +%s) + for f in install.sh install.ps1 version.json; do cp -a \"$WEBROOT/\$f\" \"$BACKUP_DIR/\$f.bak-\$ts\" 2>/dev/null || true; done + for f in /tmp/ba/bastion-agent-* /tmp/ba/freeagent-*; do [ -e \"\$f\" ] || continue; b=\$(basename \"\$f\"); mv \"\$f\" \"$WEBROOT/dl/\$b\"; chmod 644 \"$WEBROOT/dl/\$b\"; done + # The signature lands before the manifest it signs. + for f in /tmp/ba/version.json.*.sig; do b=\$(basename \"\$f\"); mv \"\$f\" \"$WEBROOT/\$b\"; chmod 644 \"$WEBROOT/\$b\"; done + for f in version.json install.sh install.ps1; do mv /tmp/ba/\$f \"$WEBROOT/\$f\"; chmod 644 \"$WEBROOT/\$f\"; done + rm -rf /tmp/ba" +done + +# Verify each edge against the manifest IT serves, and, while the legacy address +# is served, assert the legacy entries are present: their absence is the failure +# that strands old bastions. +for E in $EDGES; do + printf ' %s: ' "$E" + ssh -i "$KEY" "$E" "cd $WEBROOT && LEGACY=$LEGACY python3 - <.sig), checked against +# the manifest that address serves and the keys the bastion pins. +SIGCHECK="$(mktemp -d)"; trap 'rm -rf "$SIGCHECK"' EXIT +# The legacy address is checked from outside only when it answers there: while +# edges disagree, the public name may reach one that no longer has the route, +# and the per-edge check above has already held every edge's files to the +# manifest. +ADDRS=("$BASE_NEW"); [ "$LEGACY" = 1 ] && [ "$LEGACY_CODE" != 404 ] && ADDRS+=("$BASE_OLD") +for B in "${ADDRS[@]}"; do + curl -fsS --max-time 30 "$B/version.json" -o "$SIGCHECK/m" + H="$(python3 -c 'import hashlib,sys;print(hashlib.sha256(open(sys.argv[1],"rb").read()).hexdigest()[:16])' "$SIGCHECK/m")" + curl -fsS --max-time 30 "$B/version.json.$H.sig" -o "$SIGCHECK/s" \ + || { echo " VERIFY FAILED: $B/version.json.$H.sig is not served publicly" >&2; exit 1; } + if "$BASTION_TREE/deploy/release-verify.sh" bastion-agent "$SIGCHECK/m" "$SIGCHECK/s" ${FLEET_FROM:+"$FLEET_FROM"} >/dev/null; then + echo " $B/version.json.$H.sig ok" + else + echo " VERIFY FAILED: $B/version.json.$H.sig does not verify against the manifest served there" >&2 + exit 1 + fi +done + +echo "published bastion agent v$VER to: $EDGES" +echo " new: $BASE_NEW/version.json · $BASE_NEW/dl/bastion-agent-$VER.pyz" +if [ "$LEGACY" = 1 ]; then + echo " legacy: $BASE_OLD/version.json · $BASE_OLD/dl/freeagent-$VER.pyz" +fi diff --git a/bastion_agent/run.py b/bastion_agent/run.py new file mode 100644 index 0000000..dd1d98c --- /dev/null +++ b/bastion_agent/run.py @@ -0,0 +1,447 @@ +"""bastion agent runtime: the `sweep` scheduler and the manifest gate. + +This is what turns seven commands into an agent. Each cycle: + + 1. re-read the manifest (a re-signed manifest takes effect next sweep) + 2. GATE: verify its signature — ECDSA P-256 over the canonical payload, + the leaf cert's CN must equal owner.host, and the cert must chain to + FREEAGENT_CA_ROOT, which is required. Unsigned or unverifiable → the + sweep does nothing and says so. (--allow-unsigned exists for + development and is recorded in every run it touches.) + 3. run the stages in manifest order, each with the module already + proven alone; the first failure stops the run (fail closed — a + later stage never consumes a half-finished earlier one) + 4. append one line to run.jsonl, rewrite run.state.json, sleep until + the next due time (manifest `sweep`), or exit for `manual`. + +A lock file (atomic create, pid + time inside) keeps two sweeps off one +output directory; a stale lock (dead pid on POSIX, or older than +LOCK_STALE_HOURS anywhere) is reclaimed and logged. + +Stdlib only. The gate's certificate and signature checks are in x509gate +(the stdlib has no ECDSA, so it carries its own P-256 verifier); openssl is +needed only by the developer helper that SIGNS a manifest. Python 3.8 floor. +""" + +import datetime as _dt +import json +import os +import shutil +import signal +import subprocess +import sys +import tempfile +import time + +from . import catalog as _catalog +from . import embed as _embed +from . import extract as _extract +from . import sanitize as _sanitize +from . import seal as _seal +from . import walk as _walk +from . import x509gate as _x509 +from .manifest import AGENT_DIR, ManifestError, load_manifest, signing_payload + +RUN_VERSION = 1 +SWEEP_SECONDS = {"15m": 15 * 60, "hourly": 60 * 60, "daily": 24 * 60 * 60, "manual": None} +LOCK_STALE_HOURS = 12 +MIN_RETRY_SECONDS = 60 + + +class RunError(Exception): + pass + + +class GateError(RunError): + """The manifest did not pass the gate; nothing ran.""" + + +# ── the gate ──────────────────────────────────────────────────────────────── + +def _openssl(): + p = shutil.which("openssl") + if not p: + raise GateError("openssl is not on PATH — cannot verify the manifest signature") + return p + + +def _run(argv, data=None): + try: + proc = subprocess.run(argv, input=data, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + timeout=60, check=False) + except (OSError, subprocess.TimeoutExpired) as exc: + raise GateError("%s: %s" % (argv[0], exc)) + return proc.returncode, proc.stdout, proc.stderr.decode("utf-8", "replace") + + +def cert_cn(cert_path): + try: + return _x509.load_cert_file(cert_path).cn() + except (OSError, _x509.CertError, ValueError, TypeError, IndexError, OverflowError) as exc: + raise GateError("cannot read signing cert %s: %s" % (cert_path, exc)) + + +def verify_signature(manifest, env, manifest_dir="."): + """Raise GateError unless the manifest carries a valid signature. + + Standard library only (x509gate): the gate used to shell out to openssl, + which Windows does not ship, so on a Windows machine every manifest was + refused and the agent never ran. The checks + are the ones openssl made: exactly one CN, equal to the owner; the cert + chains to FREEAGENT_CA_ROOT and is inside its validity window; the + ECDSA P-256 / SHA-256 signature verifies over the canonical payload.""" + sig = manifest.signature + if sig is None: + raise GateError("manifest is unsigned — the runtime refuses to execute it") + cert = sig.cert if os.path.isabs(sig.cert) else os.path.join(manifest_dir, sig.cert) + if not os.path.exists(cert): + raise GateError("signing cert %s not found" % cert) + try: + leaf = _x509.load_cert_file(cert) + cn = leaf.cn() + except (OSError, _x509.CertError, ValueError, TypeError, IndexError, OverflowError) as exc: + # Anything a malformed cert raises is a refusal, never a crash. + raise GateError("cannot read signing cert %s: %s" % (cert, exc)) + if cn != manifest.owner_host: + raise GateError("signing cert CN %r is not the manifest owner %r" % (cn, manifest.owner_host)) + ca_root = env.get("FREEAGENT_CA_ROOT", "").strip() + if not ca_root: + # without a root the gate would only prove "signed by the key in the + # cert next to the manifest" — that authenticates nothing + raise GateError("FREEAGENT_CA_ROOT is not set — the gate cannot verify who signed this manifest") + try: + with open(ca_root, "rb") as fh: + roots = _x509.load_root_file(fh.read()) + _x509.verify_chain(leaf, roots) + except OSError as exc: + raise GateError("cannot read FREEAGENT_CA_ROOT %s: %s" % (ca_root, exc)) + except (_x509.CertError, ValueError, TypeError, IndexError, OverflowError) as exc: + raise GateError("signing cert does not chain to FREEAGENT_CA_ROOT: %s" % exc) + import base64 + try: + raw = base64.b64decode(sig.value, validate=True) + except Exception: + raise GateError("signature value is not base64") + if not _x509.ecdsa_p256_sha256_verify(leaf.public_key, signing_payload(manifest.raw), raw): + raise GateError("manifest signature does not verify (payload changed or wrong key)") + return {"cn": cn, "cert": cert, "chained": True} + + +def manifest_payload(path): + """The exact bytes a manifest's signature covers (the canonical payload). + A signer that holds the key elsewhere — the Go bastion, which mints the + managed agent's identity with its own crypto — signs these and hands the + result back through write_signature.""" + return signing_payload(load_manifest(path).raw) + + +def write_signature(path, der_signature, cert_path): + """Write the signature block (a DER ECDSA signature over manifest_payload) + into the manifest, replacing any earlier one. Checked before it is kept: + the manifest must still load as signed.""" + import base64 + with open(path, encoding="utf-8") as fh: + text = fh.read() + # simpler and safe: drop an existing top-level signature block, then append + body = _strip_signature_block(text) + block = "signature:\n alg: ecdsa-p256-sha256\n cert: %s\n value: %s\n" % ( + cert_path, base64.b64encode(der_signature).decode("ascii")) + with open(path, "w", encoding="utf-8", newline="\n") as fh: + fh.write(body.rstrip("\n") + "\n" + block) + load_manifest(path, require_signed=True) # must still validate + return block + + +def sign_manifest_file(path, key_path, cert_path): + """Developer/host helper: sign the canonical payload with a PEM key via + openssl and write the signature block. The managed install does not use + this — the bastion signs with its own key and calls write_signature — so + no machine needs openssl to run the agent.""" + rc, out, err = _run([_openssl(), "dgst", "-sha256", "-sign", key_path], + data=manifest_payload(path)) + if rc != 0: + raise GateError("signing failed: %s" % err.strip()) + return write_signature(path, out, cert_path) + + +def _strip_signature_block(text): + out, skipping = [], False + for line in text.splitlines(keepends=True): + if line.startswith("signature:"): + skipping = True + continue + if skipping and (line.startswith(" ") or line.startswith("\t")): + continue + skipping = False + out.append(line) + return "".join(out) + + +# ── the lock ──────────────────────────────────────────────────────────────── + +def _pid_alive(pid): + if os.name == "nt": + return True # cannot cheaply tell; the age rule decides + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + return True + + +class Lock: + def __init__(self, out_dir): + os.makedirs(out_dir, exist_ok=True) + self.path = os.path.join(out_dir, "sweep.lock") + self.reclaimed = None + + def __enter__(self): + for attempt in range(2): + try: + fd = os.open(self.path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600) + except FileExistsError: + if attempt == 0 and self._stale(): + self.reclaimed = self._read() + try: + os.unlink(self.path) + except OSError: + pass + continue + raise RunError("another sweep holds %s (%s)" % (self.path, self._read())) + with os.fdopen(fd, "w") as fh: + json.dump({"pid": os.getpid(), "since": time.time()}, fh) + return self + raise RunError("could not acquire %s" % self.path) + + def _read(self): + try: + with open(self.path) as fh: + return json.load(fh) + except (OSError, ValueError): + return {} + + def _stale(self): + """A lock is stale when its holder is provably dead. On POSIX that is + the pid check and nothing else — a live pid keeps its lock however + long the sweep runs (age alone stole live locks). The + age rule applies only where the pid cannot be checked (Windows) or the + file is unreadable, and then by the FILE's mtime, so a truncated lock + is not instantly stale.""" + info = self._read() + pid = info.get("pid") + if os.name != "nt" and isinstance(pid, int): + return not _pid_alive(pid) + try: + age = time.time() - os.stat(self.path).st_mtime + except OSError: + return False + return age > LOCK_STALE_HOURS * 3600 + + def __exit__(self, *exc): + try: + os.unlink(self.path) + except OSError: + pass + + +# ── one run ───────────────────────────────────────────────────────────────── + +STAGES = { + "walk": lambda m, out, env: _walk.walk(m, out), + "extract": lambda m, out, env: _extract.extract(m, out, use_external=env.get("FREEAGENT_NO_EXTERNAL") != "1"), + "sanitize": lambda m, out, env: _sanitize.sanitize(m, out, env=env), + "chunk": lambda m, out, env: _embed.chunk(m, out), + "embed": lambda m, out, env: _embed.embed(m, out, env=env), + "catalog": lambda m, out, env: _catalog.catalog(m, out, env=env), + "seal": lambda m, out, env: _seal.seal(m, out, env=env), +} +STAGE_ERRORS = (FileNotFoundError, _sanitize.SanitizeError, _embed.EmbedError, + _catalog.CatalogError, _seal.SealError, OSError) + + +def _stage_failed(name, summary): + """A stage that completed but reports it stopped is a failure too.""" + if summary.get("stopped"): + return summary["stopped"] + return None + + +def _brief(name, summary): + c = summary.get("counts", {}) + keys = {"walk": ("files", "errors"), "extract": ("ok", "reused", "error"), + "sanitize": ("ok", "reused", "not_run"), "chunk": ("chunks",), + "embed": ("embedded", "reused"), "catalog": ("files", "dirs", "summaries"), + "seal": ()}.get(name, ()) + out = {k: c[k] for k in keys if k in c} + if name == "seal": + out = {"bytes_sealed": summary.get("bytes_sealed"), "recipients": len(summary.get("recipients", []))} + return out + + +def default_out(m): + """Where an agent keeps itself: /.agent for a single-root manifest + (files-and-folders style: copy the folder, the agent comes with it); + build/freeagent/ when there are several roots and no obvious home. + Every command uses this default, so `sweep` and `serve` run without + --out share one folder. + + The directory keeps the old name deliberately: it is on-disk output a user + may already have, and renaming it would orphan it.""" + roots = m.scope.roots + if len(roots) == 1: + return os.path.join(os.path.abspath(os.path.expanduser(roots[0])), AGENT_DIR) + return os.path.join("build", "freeagent", m.id) + + +def run_once(manifest_path, out_dir=None, env=None, allow_unsigned=False, now=None, clock=time.time): + env = dict(os.environ) if env is None else env + started = now or _dt.datetime.now(_dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + t0 = clock() + record = {"v": RUN_VERSION, "started": started, "manifest": manifest_path, "stages": [], + "ok": False, "allow_unsigned": bool(allow_unsigned)} + try: + m = load_manifest(manifest_path) + except ManifestError as exc: + record.update(error="manifest: %s" % "; ".join(exc.errors), finished=_now(now), seconds=round(clock() - t0, 3)) + # a refused manifest is a run that happened; the append-only log must + # say so. With no id to derive a directory from, the + # caller's --out or a fixed 'refused' directory holds the record. + _append_run(out_dir or os.path.join("build", "freeagent", "refused"), record) + return record, None + out_dir = out_dir or default_out(m) + record["out"] = out_dir + record["manifest_id"] = m.id + from .manifest import signing_digest + record["manifest_digest"] = "sha256:" + signing_digest(m.raw) + try: + gate = verify_signature(m, env, manifest_dir=os.path.dirname(os.path.abspath(manifest_path))) + record["gate"] = gate + except GateError as exc: + if not allow_unsigned: + record.update(error="gate: %s" % exc, finished=_now(now), seconds=round(clock() - t0, 3)) + _append_run(out_dir, record) # a gate refusal is on the record too + return record, m + record["gate"] = {"bypassed": str(exc)} + try: + with Lock(out_dir) as lock: + if lock.reclaimed: + record["lock_reclaimed"] = lock.reclaimed + for name in m.stages: + s0 = clock() + entry = {"stage": name} + try: + summary = STAGES[name](m, out_dir, env) + err = _stage_failed(name, summary) + entry.update(status="failed" if err else "ok", seconds=round(clock() - s0, 3), + **_brief(name, summary)) + if err: + entry["error"] = err + except STAGE_ERRORS as exc: + entry.update(status="failed", seconds=round(clock() - s0, 3), + error="%s: %s" % (type(exc).__name__, exc)) + record["stages"].append(entry) + if entry["status"] != "ok": + record["error"] = "stage %s: %s" % (name, entry["error"]) + break + else: + record["ok"] = True + except RunError as exc: + record["error"] = str(exc) + record["finished"] = _now(now) + record["seconds"] = round(clock() - t0, 3) + _append_run(out_dir, record) + return record, m + + +def _now(fixed=None): + return fixed or _dt.datetime.now(_dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _append_run(out_dir, record): + os.makedirs(out_dir, exist_ok=True) + with open(os.path.join(out_dir, "run.jsonl"), "a", encoding="utf-8") as fh: + fh.write(json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n") + + +def _write_state(out_dir, state): + path = os.path.join(out_dir, "run.state.json") + fd, tmp = tempfile.mkstemp(prefix=".tmp-", dir=out_dir) + with os.fdopen(fd, "w", encoding="utf-8") as fh: + json.dump(state, fh, indent=2, sort_keys=True) + os.replace(tmp, path) + + +def load_state(out_dir): + try: + with open(os.path.join(out_dir, "run.state.json"), encoding="utf-8") as fh: + return json.load(fh) + except (OSError, ValueError): + return {} + + +def next_due(sweep, last_started_ts, ok, failures): + """Seconds-since-epoch of the next run. Failures retry with backoff + (1 min, 2, 4 … capped at the cadence); manual never schedules.""" + period = SWEEP_SECONDS.get(sweep) + if period is None: + return None + if ok: + return last_started_ts + period + backoff = min(period, MIN_RETRY_SECONDS * (2 ** min(failures - 1, 10))) + return last_started_ts + backoff + + +# ── the loop ──────────────────────────────────────────────────────────────── + +def sweep(manifest_path, out_dir=None, env=None, allow_unsigned=False, once=False, + sleep=time.sleep, clock=time.time, max_runs=None, log=None): + """Run on the manifest's cadence until stopped. Returns the last record.""" + env = dict(os.environ) if env is None else env + log = log or (lambda s: sys.stderr.write(s + "\n")) + stop = {"flag": False} + + def _stop(signum, frame): + stop["flag"] = True + log("sweep: stop requested (signal %d); finishing the current run" % signum) + + for sig in (getattr(signal, "SIGTERM", None), getattr(signal, "SIGINT", None)): + if sig is not None: + try: + signal.signal(sig, _stop) + except (ValueError, OSError): + pass # not the main thread (tests) — run without handlers + + runs = 0 + last = None + while not stop["flag"]: + t_start = clock() + record, m = run_once(manifest_path, out_dir, env, allow_unsigned=allow_unsigned, clock=clock) + runs += 1 + last = record + out = record.get("out") or out_dir or "." + state = load_state(out) if os.path.isdir(out) else {} + failures = 0 if record["ok"] else state.get("consecutive_failures", 0) + 1 + sweep_kind = m.sweep if m is not None else "manual" + due = next_due(sweep_kind, t_start, record["ok"], failures) + state = {"last_started": record["started"], "last_finished": record.get("finished"), + "last_ok": record["ok"], "last_error": record.get("error"), + "consecutive_failures": failures, "sweep": sweep_kind, + "next_due": _dt.datetime.fromtimestamp(due, _dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") if due else None, + "pid": os.getpid(), "runs_this_process": runs} + if os.path.isdir(out): + _write_state(out, state) + stages = ", ".join("%s:%s" % (s["stage"], s["status"]) for s in record["stages"]) or "-" + log("sweep: %s %s in %.1fs [%s]%s" % ( + record.get("manifest_id", "?"), "ok" if record["ok"] else "FAILED", + record.get("seconds", 0), stages, + "" if record["ok"] else " — " + str(record.get("error")))) + if once or due is None or (max_runs and runs >= max_runs): + break + while not stop["flag"]: + wait = due - clock() + if wait <= 0: + break + sleep(min(wait, 30)) + return last diff --git a/bastion_agent/sanitize.py b/bastion_agent/sanitize.py new file mode 100644 index 0000000..76bcf72 --- /dev/null +++ b/bastion_agent/sanitize.py @@ -0,0 +1,404 @@ +"""bastion agent `sanitize` stage — the salt. + +Runs the PII sterilizer (pii) over the extract store and writes a +SECOND store: + + /sanitize//.txt sanitized text — PII replaced by + stable [PII_TYPE_n] tokens + /sanitize/vault.db token <-> value map, 0600, local + /sanitize.jsonl one record per extracted file: + status, mode, stats (counts only — + never a value) + /sanitize.summary.json + +From here on, later stages read ONLY the sanitized store. The extract +store keeps the plaintext and stays on this machine; the vault is the +only way back (`/rehydrate` semantics, local). + +Two ways to reach the sterilizer, chosen from the environment, never the +manifest (no addresses in manifests — docs/pii-sterilization.md): + + in-process the default. the pii package runs inside this process: the rules + pass, the recall backend selected by PII_RECALL + (llm | gliner — the laptop path), the vault under . + service PII_ADDR (+ PII_API_KEY) set: POST /sterilize to the + running PII service, which owns its own vault. The + server path; the bastion advertises the capability. + +Modes (manifest `sanitize.mode`): + sterilize rules + recall pass (+ optional adversarial verify). + flag rules pass ONLY — deterministic, model-free, always + in-process. Redacts the precision anchors (emails, cards, + keys, IBANs, …); contextual PII (names, schools) is NOT + caught. The manifest doc says so; this docstring says so. + +Fail closed: the first recall failure (model unreachable, package +missing) stops the stage — remaining files are recorded `not_run` and +nothing half-sanitized is written. Long texts are split on paragraph +boundaries per call (SANITIZE_CHUNK_CHARS); tokens are stable per scope, +so a value seen in two chunks gets one token. +""" + +import datetime as _dt +import json +import os +import re +import tempfile +import urllib.error +import urllib.request + +from .extract import store_path as extract_store_path +from .manifest import signing_digest + +# Part of the reuse key: bump it whenever the pipeline would sterilize the +# same text differently, or a stored result from older code is kept as is. +SANITIZE_VERSION = 2 +SANITIZE_CHUNK_CHARS = 20000 # per sterilize call; under the service's 2 MiB body cap +SERVICE_TIMEOUT = 600.0 + +STATUS_OK, STATUS_ERROR, STATUS_SKIPPED, STATUS_NOT_RUN = "ok", "error", "skipped", "not_run" + + +class SanitizeError(Exception): + """Fail-closed stop: the recall pass is unavailable.""" + + +def pii_modules(): + """The sterilizer, a SIBLING top-level package here and inside the built + .pyz alike (build.py writes it as `pii/`). One import path: an ImportError + here means the sterilizer is genuinely absent, which is a fail-closed stop, + never a silent skip. + + pii.pipeline.sterilize(text, scope, vault, llm_extract, llm_verify=None) + returning (text, result), pii.recall.build_backend (which raises + RecallError rather than degrading) and pii.vault.Vault.""" + from pii import pipeline, recall + from pii.vault import Vault + return pipeline, recall, Vault + + +# Test seam: when set, used instead of recall.build_backend(env). +BUILD_BACKEND = None + + +# ── sterilizer front-ends ─────────────────────────────────────────────────── + +class InProcess: + """pii inside this process. Vault lives under the output dir.""" + + name = "in-process" + + def __init__(self, out_dir, mode, env, verify=False): + pipeline, recall, Vault = pii_modules() + self._pipeline = pipeline + self._recall_error = recall.RecallError + self.mode = mode + self.verify = verify + os.umask(0o077) + # FREEAGENT_VAULT_DIR keeps the vault (plaintext PII values) OUT of the + # output directory when that directory sits inside a folder that may + # be cloud-synced (/.agent). The bastion sets it to its own + # state dir; the default stays beside the output for a hand-run agent. + vault_dir = (env or {}).get("FREEAGENT_VAULT_DIR", "").strip() + vault_path = (os.path.join(vault_dir, "vault.db") if vault_dir + else os.path.join(out_dir, "sanitize", "vault.db")) + self.vault = Vault(vault_path) + try: + os.chmod(self.vault.path, 0o600) + except OSError: + pass + if mode == "flag": + self.backend = None + self.describe = "flag:rules-only" + else: + build = BUILD_BACKEND or recall.build_backend + try: + self.backend = build(env) + except recall.RecallError as e: + raise SanitizeError(str(e)) + self.describe = "sterilize:" + self.backend.describe() + + def sterilize(self, text, scope): + if self.backend is None: + extract_fn, verify_fn = (lambda t: []), None + else: + extract_fn = self.backend.extract + verify_fn = self.backend.verify if self.verify else None + try: + # documents are raw files, never this pipeline's output: every + # placeholder-shaped literal in one is escaped + return self._pipeline.sterilize(text, scope, self.vault, extract_fn, verify_fn, + escape_held_literals=True) + except self._recall_error as e: + raise SanitizeError("recall pass failed: %s" % e) + + +class Service: + """The running PII service over HTTP. It owns the vault.""" + + name = "service" + + def __init__(self, addr, api_key, verify=False, timeout=SERVICE_TIMEOUT): + if not api_key: + raise SanitizeError("PII_ADDR set but PII_API_KEY empty — the service is fail-closed") + self.url = ("http://" + addr if "://" not in addr else addr).rstrip("/") + self.api_key = api_key + self.verify = verify + self.timeout = timeout + self.describe = "service:" + self.url + + def sterilize(self, text, scope): + body = json.dumps({"text": text, "scope": scope, "verify": self.verify, + "escape_held_literals": True}).encode("utf-8") + req = urllib.request.Request( + self.url + "/sterilize", data=body, + headers={"Content-Type": "application/json", + "Authorization": "Bearer " + self.api_key}) + try: + with urllib.request.urlopen(req, timeout=self.timeout) as resp: + payload = json.load(resp) + except urllib.error.HTTPError as e: + if e.code == 502: + raise SanitizeError("pii service: recall pass unavailable (502)") + raise RuntimeError("pii service HTTP %d" % e.code) + except (OSError, ValueError) as e: + raise SanitizeError("pii service unreachable: %s" % e) + stats = dict(payload.get("stats", {})) + # a service that predates the field ignores it and applies the narrower + # rule silently; without its echo the document cannot be trusted + if stats.pop("escape_held_literals", None) is not True: + raise SanitizeError("the pii service did not honour escape_held_literals " + "(no echo in its stats); upgrade the service") + return payload["text"], {"findings": payload.get("findings", []), "stats": stats} + + +def build_front_end(out_dir, sanitize_cfg, env): + if sanitize_cfg.mode == "flag": + return InProcess(out_dir, "flag", env) + addr = env.get("PII_ADDR", "").strip() + if addr: + return Service(addr, env.get("PII_API_KEY", ""), verify=sanitize_cfg.verify) + return InProcess(out_dir, "sterilize", env, verify=sanitize_cfg.verify) + + +# ── helpers ───────────────────────────────────────────────────────────────── + +# Same shape as pii.pipeline.TOKEN_RE (a test keeps them equal). Kept here so +# splitting needs no pii import; the sterilizer is loaded fail-closed elsewhere. +_PLACEHOLDER_RE = re.compile(r"\[PII_([A-Z_]+)_(\d+)\]") + + +def _clear_of_placeholder(piece, limit): + """A hard cut at limit, moved back to before the '[' of a placeholder it + would split. Each half of a split placeholder would reach the sterilizer + as unrecognised text and be stored unescaped. A placeholder at the very + start is kept whole instead, so the chunk may run over the limit by the + rest of the placeholder. + + A placeholder has no '[' after its first character, so the only one that + can straddle the cut starts at the last '[' before it: exact at any + placeholder length, with no search window.""" + start = piece.rfind("[", 0, limit) + if start >= 0: + m = _PLACEHOLDER_RE.match(piece, start) + if m and m.end() > limit: + return start if start > 0 else m.end() + return limit + + +def split_chunks(text, limit=None): + """Split on paragraph, then line, boundaries; a single oversized line + is cut hard. Concatenation of the chunks is byte-identical to text. + The default limit is read when called, so a changed + SANITIZE_CHUNK_CHARS takes effect.""" + if limit is None: + limit = SANITIZE_CHUNK_CHARS + if not text: + return [] + if len(text) <= limit: + return [text] + parts = text.split("\n\n") + paras = [p + ("\n\n" if i < len(parts) - 1 else "") for i, p in enumerate(parts)] + out, buf = [], "" + for piece in paras: + if len(buf) + len(piece) <= limit: + buf += piece + continue + if buf: + out.append(buf) + buf = "" + while len(piece) > limit: + cut = piece.rfind("\n", 0, limit) + cut = _clear_of_placeholder(piece, limit) if cut <= 0 else cut + 1 + out.append(piece[:cut]) + piece = piece[cut:] + buf = piece + if buf: + out.append(buf) + return [c for c in out if c] + + +def _atomic_write_text(path, text): + d = os.path.dirname(path) + os.makedirs(d, exist_ok=True) + fd, tmp = tempfile.mkstemp(prefix=".tmp-", dir=d) + try: + with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as fh: + fh.write(text) + os.replace(tmp, path) + except BaseException: + try: + os.unlink(tmp) + except OSError: + pass + raise + + +def store_path(out_dir, sha): + return os.path.join(out_dir, "sanitize", sha[:2], sha + ".txt") + + +def load_extract_index(out_dir): + path = os.path.join(out_dir, "extract.jsonl") + if not os.path.exists(path): + raise FileNotFoundError("no extract index at %s — run the extract stage first" % path) + with open(path, encoding="utf-8") as fh: + return [json.loads(l) for l in fh] + + +def load_index(out_dir): + idx = {} + try: + with open(os.path.join(out_dir, "sanitize.jsonl"), encoding="utf-8") as fh: + for line in fh: + try: + rec = json.loads(line) + except ValueError: + continue + if rec.get("sha256"): + idx[rec["sha256"]] = rec + except OSError: + pass + return idx + + +def _cache_key(front, cfg): + return "%s|%s|%s|%s|v%d" % (front.name, cfg.mode, cfg.scope, cfg.verify, SANITIZE_VERSION) + + +def _merge_stats(acc, stats): + for k, v in stats.items(): + if isinstance(v, int): + acc[k] = acc.get(k, 0) + v + else: + acc[k] = v + + +# ── the stage ─────────────────────────────────────────────────────────────── + +def sanitize(manifest, out_dir, env=None, now=None, front_end=None): + cfg = manifest.sanitize + if cfg is None: + raise SanitizeError("manifest has no sanitize section / stage") + env = dict(os.environ) if env is None else env + front = front_end or build_front_end(out_dir, cfg, env) + key = _cache_key(front, cfg) + extracted = load_extract_index(out_dir) + prev = load_index(out_dir) + counts = {k: 0 for k in ("files", "ok", "reused", "skipped", "error", "not_run")} + type_counts = {} + index = [] + done = {} + stopped = None + + for rec in extracted: + counts["files"] += 1 + sha = rec.get("sha256") + base = {"root": rec["root"], "path": rec["path"], "sha256": sha, + "mode": cfg.mode, "v": SANITIZE_VERSION} + if rec.get("status") not in ("ok",): + counts["skipped"] += 1 + index.append(dict(base, status=STATUS_SKIPPED, error="extract: " + str(rec.get("status")))) + continue + if sha in done: + dup = done[sha] + index.append(dict(dup, root=rec["root"], path=rec["path"])) + counts["reused" if dup.get("reused") else dup["status"]] += 1 + continue + if stopped is not None: + out = dict(base, status=STATUS_NOT_RUN, error=stopped) + counts["not_run"] += 1 + index.append(out) + done[sha] = out + continue + spath = store_path(out_dir, sha) + old = prev.get(sha) + if old and old.get("status") == STATUS_OK and old.get("key") == key and os.path.exists(spath): + out = dict(base, status=STATUS_OK, key=key, stats=old.get("stats", {}), + types=old.get("types", {}), chars=old.get("chars", 0), reused=True) + counts["reused"] += 1 + index.append(out) + done[sha] = out + continue + try: + with open(extract_store_path(out_dir, sha), encoding="utf-8") as fh: + text = fh.read() + except OSError as e: + out = dict(base, status=STATUS_ERROR, error="extract store: %s" % e) + counts["error"] += 1 + index.append(out) + done[sha] = out + continue + pieces, stats, types = [], {}, {} + try: + for chunk in split_chunks(text): + clean, res = front.sterilize(chunk, cfg.scope) + pieces.append(clean) + _merge_stats(stats, res.get("stats", {})) + for f in res.get("findings", []): + t = f.get("type", "?") + types[t] = types.get(t, 0) + 1 + except SanitizeError as e: + stopped = str(e) + out = dict(base, status=STATUS_NOT_RUN, error=stopped) + counts["not_run"] += 1 + index.append(out) + done[sha] = out + continue + except (RuntimeError, ValueError, KeyError) as e: + out = dict(base, status=STATUS_ERROR, error="%s: %s" % (type(e).__name__, e)) + counts["error"] += 1 + index.append(out) + done[sha] = out + continue + clean = "".join(pieces) + _atomic_write_text(spath, clean) + out = dict(base, status=STATUS_OK, key=key, stats=stats, types=types, chars=len(clean)) + counts["ok"] += 1 + for t, n in types.items(): + type_counts[t] = type_counts.get(t, 0) + n + index.append(out) + done[sha] = out + + index.sort(key=lambda r: (r["root"], r["path"])) + _atomic_write_text(os.path.join(out_dir, "sanitize.jsonl"), "".join( + json.dumps(r, ensure_ascii=False, sort_keys=True) + "\n" for r in index)) + summary = { + "freeagent": 1, + "stage": "sanitize", + "sanitize_version": SANITIZE_VERSION, + "manifest_id": manifest.id, + "manifest_digest": "sha256:" + signing_digest(manifest.raw), + "sanitized_at": now or _dt.datetime.now(_dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "mode": cfg.mode, + "scope": cfg.scope, + "verify": cfg.verify, + "sterilizer": front.describe, + "counts": counts, + "types": type_counts, + "stopped": stopped, + "entries": len(index), + } + _atomic_write_text(os.path.join(out_dir, "sanitize.summary.json"), + json.dumps(summary, ensure_ascii=False, indent=2, sort_keys=True) + "\n") + return summary diff --git a/bastion_agent/seal.py b/bastion_agent/seal.py new file mode 100644 index 0000000..8d06892 --- /dev/null +++ b/bastion_agent/seal.py @@ -0,0 +1,238 @@ +"""bastion agent `seal` stage — encrypt the catalog to the owner (and named +recipients) so it can leave the machine. + +What is sealed — and ONLY this — is an allowlisted bundle: + + manifest.json the manifest as executed (minus signature) + catalog.jsonl file cards + catalog.dirs.jsonl directory cards + *.summary.json every stage's summary — counts, digests, + tool names; never a value + seal.manifest.json bundle inventory with sha256 per member + +Never in the bundle, by construction (the allowlist is the mechanism, a +test is the proof): the extract store (plaintext), the sanitized store, +the vault, vectors.db, chunks. `sink.plaintext: never` is enforced here +one more time by not having a code path that could add those files. + +The envelope is age (X25519 + ChaCha20-Poly1305): a small, widely reviewed +format with a single static binary on Mac/Windows/Linux. It is REQUIRED for this stage +— the stdlib cannot produce the envelope, and a catalog that cannot be +sealed does not leave. Recipients: + + owner FREEAGENT_OWNER_RECIPIENT (an age1… public key), from the + environment, never the manifest. Always present. + seal_to manifest sink.seal_to — each entry an age1… recipient (or + an ssh-ed25519/ssh-rsa public key line, which age accepts). + +Grantees (manifest `grants`) are NOT recipients: they query the container +over the mesh; they never receive the sealed catalog. + +Output: /sealed/-.tar.age, /sealed/latest.json (a +pointer, not a symlink — Windows), /seal.summary.json. Optional +detached signature with the host's key: SEAL_SIGN_KEY (PEM path) → +.sig via openssl, so a receiving host can check who sealed it once +the runtime's CA verification lands. +""" + +import datetime as _dt +import hashlib +import io +import json +import os +import re +import shutil +import subprocess +import tarfile +import tempfile + +from .manifest import signing_digest + +SEAL_VERSION = 1 +AGE_RECIPIENT_RE = re.compile(r"^(age1[a-z0-9]{58}|ssh-(ed25519|rsa) [A-Za-z0-9+/=]+( .*)?)$") +BUNDLE_ALLOW = ("catalog.jsonl", "catalog.dirs.jsonl") +SUMMARY_GLOB_RE = re.compile(r"^[a-z]+\.summary\.json$") +TOOL_TIMEOUT = 300 + + +class SealError(Exception): + """Fail closed: nothing leaves.""" + + +def tool_path(name): + return shutil.which(name) + + +def run(argv, data=None, timeout=TOOL_TIMEOUT): + try: + proc = subprocess.run(argv, input=data, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + timeout=timeout, check=False) + except (OSError, subprocess.TimeoutExpired) as exc: + raise SealError("%s: %s" % (argv[0], exc)) + if proc.returncode != 0: + err = proc.stderr.decode("utf-8", "replace").strip().splitlines() + raise SealError("%s exited %d: %s" % (argv[0], proc.returncode, err[-1] if err else "")) + return proc.stdout + + +def recipients_for(manifest, env): + owner = env.get("FREEAGENT_OWNER_RECIPIENT", "").strip() + if not owner: + raise SealError("FREEAGENT_OWNER_RECIPIENT is not set — the owner's age public key " + "is the mandatory recipient; refusing to seal to nobody") + recips = [owner] + list(manifest.sink.seal_to) + bad = [r for r in recips if not AGE_RECIPIENT_RE.match(r)] + if bad: + raise SealError("not an age recipient (age1… or ssh-ed25519/ssh-rsa public key): %s" + % ", ".join(r[:24] + "…" if len(r) > 24 else r for r in bad)) + seen = set() + out = [] + for r in recips: + if r not in seen: + seen.add(r) + out.append(r) + return out + + +def _sha256_file(path): + h = hashlib.sha256() + with open(path, "rb") as fh: + for buf in iter(lambda: fh.read(1024 * 1024), b""): + h.update(buf) + return h.hexdigest() + + +def build_bundle(manifest, out_dir, stamp): + """The plaintext tar, in memory. Returns (bytes, inventory).""" + members = [] + for name in BUNDLE_ALLOW: + path = os.path.join(out_dir, name) + if not os.path.exists(path): + raise SealError("no %s in %s — run the catalog stage first" % (name, out_dir)) + members.append(name) + for name in sorted(os.listdir(out_dir)): + if SUMMARY_GLOB_RE.match(name) and name != "seal.summary.json": + members.append(name) + inventory = {"freeagent": 1, "seal_version": SEAL_VERSION, "manifest_id": manifest.id, + "manifest_digest": "sha256:" + signing_digest(manifest.raw), + "sealed_at": stamp, "members": {}} + buf = io.BytesIO() + with tarfile.open(fileobj=buf, mode="w") as tar: + def add_bytes(name, data): + info = tarfile.TarInfo(name) + info.size = len(data) + info.mtime = 0 # reproducible: the content is what is signed, not the clock + info.mode = 0o600 + tar.addfile(info, io.BytesIO(data)) + inventory["members"][name] = {"sha256": hashlib.sha256(data).hexdigest(), "bytes": len(data)} + + body = {k: v for k, v in manifest.raw.items() if k != "signature"} + add_bytes("manifest.json", json.dumps(body, sort_keys=True, indent=2, default=str).encode("utf-8")) + for name in members: + with open(os.path.join(out_dir, name), "rb") as fh: + add_bytes(name, fh.read()) + add_bytes("seal.manifest.json", json.dumps(inventory, sort_keys=True, indent=2).encode("utf-8")) + return buf.getvalue(), inventory + + +def seal(manifest, out_dir, env=None, now=None): + if "seal" not in manifest.stages: + raise SealError("'seal' is not in this manifest's stages") + env = dict(os.environ) if env is None else env + age = tool_path("age") + if not age: + raise SealError("`age` is not on PATH — the envelope cannot be produced, nothing leaves " + "(install age: https://age-encryption.org, one static binary)") + recips = recipients_for(manifest, env) + stamp = now or _dt.datetime.now(_dt.timezone.utc).strftime("%Y%m%dT%H%M%SZ") + plain, inventory = build_bundle(manifest, out_dir, stamp) + + sealed_dir = os.path.join(out_dir, "sealed") + os.makedirs(sealed_dir, exist_ok=True) + fname = "%s-%s.tar.age" % (manifest.id, stamp) + target = os.path.join(sealed_dir, fname) + argv = [age, "--encrypt"] + for r in recips: + argv += ["-r", r] + argv += ["-o", target + ".tmp"] + run(argv, data=plain) + os.replace(target + ".tmp", target) + try: + os.chmod(target, 0o600) + except OSError: + pass + + signature = None + sign_key = env.get("SEAL_SIGN_KEY", "").strip() + if sign_key: + openssl = tool_path("openssl") + if not openssl: + raise SealError("SEAL_SIGN_KEY set but openssl is not on PATH") + sig = run([openssl, "dgst", "-sha256", "-sign", sign_key, target]) + with open(target + ".sig", "wb") as fh: + fh.write(sig) + signature = os.path.basename(target) + ".sig" + + summary = { + "freeagent": 1, "stage": "seal", "seal_version": SEAL_VERSION, + "manifest_id": manifest.id, + "manifest_digest": inventory["manifest_digest"], + "sealed_at": stamp, + "file": os.path.join("sealed", fname), + "bytes_plain": len(plain), + "bytes_sealed": os.path.getsize(target), + "sha256_plain": hashlib.sha256(plain).hexdigest(), + "sha256_sealed": _sha256_file(target), + "recipients": recips, # public keys — safe to record + "members": sorted(inventory["members"]), + "signature": signature, + "catalog_sink": manifest.sink.catalog, + } + pointer = {"file": summary["file"], "sha256_sealed": summary["sha256_sealed"], + "sealed_at": stamp, "manifest_digest": summary["manifest_digest"]} + _atomic_write(os.path.join(sealed_dir, "latest.json"), json.dumps(pointer, indent=2, sort_keys=True) + "\n") + _atomic_write(os.path.join(out_dir, "seal.summary.json"), json.dumps(summary, indent=2, sort_keys=True) + "\n") + return summary + + +def unseal(sealed_path, identity_path, dest_dir): + """Owner-side: open a sealed bundle with an age identity file into + dest_dir. Verifies every member against the inventory.""" + age = tool_path("age") + if not age: + raise SealError("`age` is not on PATH") + plain = run([age, "--decrypt", "-i", identity_path, sealed_path]) + os.makedirs(dest_dir, exist_ok=True) + with tarfile.open(fileobj=io.BytesIO(plain), mode="r") as tar: + names = tar.getnames() + if any(n.startswith("/") or ".." in n.split("/") for n in names): + raise SealError("bundle contains an unsafe path") + inv_member = tar.extractfile("seal.manifest.json") + if inv_member is None: + raise SealError("bundle has no seal.manifest.json") + inventory = json.loads(inv_member.read().decode("utf-8")) + for name in names: + data = tar.extractfile(name).read() + if name != "seal.manifest.json": + want = inventory["members"].get(name, {}).get("sha256") + if want != hashlib.sha256(data).hexdigest(): + raise SealError("member %s does not match the inventory" % name) + with open(os.path.join(dest_dir, name), "wb") as fh: + fh.write(data) + return inventory + + +def _atomic_write(path, text): + d = os.path.dirname(path) + os.makedirs(d, exist_ok=True) + fd, tmp = tempfile.mkstemp(prefix=".tmp-", dir=d) + try: + with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as fh: + fh.write(text) + os.replace(tmp, path) + except BaseException: + try: + os.unlink(tmp) + except OSError: + pass + raise diff --git a/bastion_agent/serve.py b/bastion_agent/serve.py new file mode 100644 index 0000000..f1feebd --- /dev/null +++ b/bastion_agent/serve.py @@ -0,0 +1,425 @@ +"""bastion agent query endpoint — the container on the mesh. + +The other half of "access moves, data does not". Serves the owner's host +and the manifest's grantees over MUTUAL TLS; nothing here ever reads the +extract store, the vault, or the vectors' raw table for a client — the +only bytes that leave are sanitized chunks and catalog cards. + +Identity is the presented client certificate: + - the TLS layer requires a client cert that chains to FREEAGENT_CA_ROOT + (ssl.CERT_REQUIRED); no cert or a foreign CA never reaches a handler + - the cert's CN is the caller. CN == manifest owner.host → owner. + CN in manifest.grants (and today <= until) → grantee with that entry's + ops. Anyone else who chained → 403, audited. + +Ops (JSON over HTTPS): + POST /search {query, k?} search (owner, grantee w/ op) + POST /read_range {doc, start, end} read_range (owner, grantee w/ op) + from the SANITIZED store only, + <= READ_RANGE_MAX bytes + GET /catalog[?prefix=&offset=&limit=] owner only: the file cards (paged) + GET /catalog/dirs[?...] owner only: the directory cards (paged) + GET /status owner only: run.state.json + GET /health any chained cert: {ok, id} + +The manifest is re-read and re-gated (signature verified) whenever its +file changes: revoke a grant = drop the entry, re-sign, and the next +request is refused. A manifest that no longer passes the gate makes the +server refuse everything but /health until it does. + +Every request is audited to /serve.audit.jsonl: CN, op, status, +bytes, a sha256 of the query (never the query), doc id for read_range. + +Bind: --bind or FREEAGENT_BIND (host:port, default [::]:8470). The bastion +binds 127.0.0.1:0 and reads the port from --port-file, so it is the only +client; bind another interface only for another machine to reach. Server +cert and key: FREEAGENT_TLS_CERT / FREEAGENT_TLS_KEY. Stdlib only; the embedding endpoint is the same sensor `search` +already uses. +""" + +import datetime as _dt +import hashlib +import json +import os +import ssl +import sys +import threading +import urllib.parse +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +from . import embed as _embed +from .manifest import ManifestError, load_manifest +from .run import GateError, verify_signature +from .sanitize import store_path as sanitized_store_path + +DEFAULT_BIND = "[::]:8470" +MAX_BODY = 64 * 1024 +READ_RANGE_MAX = 64 * 1024 +SEARCH_K_MAX = 50 +CATALOG_PAGE_DEFAULT = 500 +CATALOG_PAGE_MAX = 2000 +OWNER_OPS = ("search", "read_range", "catalog", "status") + + +class ServeError(Exception): + pass + + +def parse_bind(bind): + bind = bind.strip() + if bind.startswith("["): + host, _, port = bind[1:].partition("]") + port = port.lstrip(":") + else: + host, _, port = bind.rpartition(":") + if not host or not port.isdigit(): + raise ServeError("FREEAGENT_BIND must be host:port or [v6]:port, got %r" % bind) + return host, int(port) + + +def peer_cn(cert): + for rdn in (cert or {}).get("subject", ()): + for k, v in rdn: + if k == "commonName": + return v + return None + + +class Policy: + """Who may do what, from a gated manifest. Reloads on file change.""" + + def __init__(self, manifest_path, env, allow_unsigned=False, today=None): + self.path = manifest_path + self.env = env + self.allow_unsigned = allow_unsigned + self.today = today + self._mtime = None + self.manifest = None + self.gate_error = None + self._lock = threading.Lock() + self.reload() + + def reload(self): + with self._lock: + try: + mtime = os.stat(self.path).st_mtime_ns + except OSError as e: + self.manifest, self.gate_error = None, "manifest unreadable: %s" % e + return + if mtime == self._mtime and (self.manifest or self.gate_error): + return + self._mtime = mtime + try: + m = load_manifest(self.path) + except ManifestError as e: + self.manifest, self.gate_error = None, "manifest invalid: %s" % "; ".join(e.errors) + return + except Exception as e: # noqa: BLE001 + # FAIL CLOSED here too: a manifest that cannot be read at all + # (permissions, bad encoding) must not leave the previous one + # in force once _mtime has advanced. + self.manifest, self.gate_error = None, "manifest unreadable: %s" % e + return + try: + verify_signature(m, self.env, manifest_dir=os.path.dirname(os.path.abspath(self.path))) + except GateError as e: + if not self.allow_unsigned: + self.manifest, self.gate_error = None, "gate: %s" % e + return + except Exception as e: # noqa: BLE001 + # FAIL CLOSED: an unexpected error in the gate must never leave + # the previous manifest in force (_mtime is already advanced). + self.manifest, self.gate_error = None, "gate: %s" % e + return + self.manifest, self.gate_error = m, None + + def allowed_ops(self, cn): + """Ops for this CN, or an empty tuple. Owner gets everything the + mask permits; a grantee gets its entry's ops while unexpired.""" + self.reload() + m = self.manifest + if m is None or not cn: + return () + if cn == m.owner_host: + ops = list(OWNER_OPS) + if "search" not in m.mask: + ops.remove("search") + return tuple(ops) + today = self.today or _dt.datetime.now(_dt.timezone.utc).date() + for g in m.grants: + if g.host == cn: + if today > g.until: + return () + return tuple(g.ops) + return () + + +HANDSHAKE_TIMEOUT = 10 + + +class Handler(BaseHTTPRequestHandler): + server_version = "freeagent" + protocol_version = "HTTP/1.1" + # keep-alive is one thread per connection here; an idle client must not + # pin that thread forever (a bastion review measured exactly that) + timeout = 60 + + def setup(self): + # The TLS handshake happens HERE, in this connection's own thread — + # never in the accept loop. Wrapping the listening socket (the first + # cut) ran every handshake synchronously in accept(), so one idle + # pre-auth TCP connection stalled every client. + # The raw socket arrives with a short timeout from get_request, so a + # peer that connects and says nothing is dropped after + # HANDSHAKE_TIMEOUT, and only its own thread waited. + self.request = self.server.ssl_ctx.wrap_socket( + self.request, server_side=True, do_handshake_on_connect=False) + self.request.do_handshake() + self.connection = self.request + super().setup() + policy = None # injected by build_server + out_dir = None + env = None + embed_endpoint = None + audit_lock = threading.Lock() + + # ── plumbing ──────────────────────────────────────────────────────── + def _cn(self): + try: + return peer_cn(self.connection.getpeercert()) + except (AttributeError, ValueError): + return None + + def _send(self, code, obj, cn=None, op=None, extra=None): + body = json.dumps(obj, ensure_ascii=False).encode("utf-8") + self.send_response(code) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + self._audit(cn, op, code, len(body), extra) + + def _audit(self, cn, op, status, nbytes, extra): + rec = {"ts": _dt.datetime.now(_dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "cn": cn, "op": op, "path": self.path, "status": status, "bytes": nbytes} + if extra: + rec.update(extra) + with self.audit_lock: + with open(os.path.join(self.out_dir, "serve.audit.jsonl"), "a", encoding="utf-8") as fh: + fh.write(json.dumps(rec, sort_keys=True) + "\n") + + def _body(self): + n = int(self.headers.get("Content-Length", "0") or 0) + if n > MAX_BODY: + raise ValueError("body too large") + raw = self.rfile.read(n) if n else b"{}" + obj = json.loads(raw.decode("utf-8")) + if not isinstance(obj, dict): + raise ValueError("body must be a JSON object") + return obj + + def _authorize(self, op): + cn = self._cn() + allowed = self.policy.allowed_ops(cn) + if op in allowed: + return cn, None + if self.policy.gate_error: + return cn, (503, {"error": "manifest not serving: %s" % self.policy.gate_error}) + return cn, (403, {"error": "not permitted for %s: %s" % (cn or "no-cn", op)}) + + def log_message(self, fmt, *args): + pass # the audit file is the log; no request lines on stderr + + # ── routes ─────────────────────────────────────────────────────────── + def do_GET(self): + cn = self._cn() + if self.path == "/health": + m = self.policy.manifest + self.policy.reload() + self._send(200, {"ok": True, "id": m.id if m else None, + "serving": self.policy.gate_error is None}, cn, "health") + return + route, _, qs = self.path.partition("?") + if route in ("/catalog", "/catalog/dirs"): + # /catalog = file cards; /catalog/dirs = directory cards — the MAP + # (sizes, types, newest change, keywords, entity counts per folder) + # a host plans a cleanup or an archive from without reading a file. + # PAGED: ?prefix=&offset=&limit= — a big store's + # file cards are megabytes and a client's reply cap is finite. + cn, denied = self._authorize("catalog") + if denied: + self._send(denied[0], denied[1], cn, "catalog") + return + q = urllib.parse.parse_qs(qs) + prefix = (q.get("prefix") or [""])[0] + try: + offset = max(0, int((q.get("offset") or ["0"])[0])) + limit = max(1, min(CATALOG_PAGE_MAX, int((q.get("limit") or [str(CATALOG_PAGE_DEFAULT)])[0]))) + except ValueError: + self._send(400, {"error": "offset/limit must be integers"}, cn, "catalog") + return + name = "catalog.dirs.jsonl" if route.endswith("/dirs") else "catalog.jsonl" + path = os.path.join(self.out_dir, name) + if not os.path.exists(path): + self._send(404, {"error": "no catalog yet — the first sweep has not finished"}, cn, "catalog") + return + with open(path, encoding="utf-8") as fh: + cards = [json.loads(l) for l in fh] + if prefix: + cards = [c for c in cards if str(c.get("path", "")).startswith(prefix)] + total = len(cards) + page = cards[offset:offset + limit] + nxt = offset + limit if offset + limit < total else None + self._send(200, {"cards": page, "kind": "dirs" if name.endswith("dirs.jsonl") else "files", + "total": total, "offset": offset, "limit": limit, "next_offset": nxt, + "prefix": prefix}, + cn, "catalog", {"kind": name, "n": len(page), "total": total}) + return + if self.path == "/status": + cn, denied = self._authorize("status") + if denied: + self._send(denied[0], denied[1], cn, "status") + return + path = os.path.join(self.out_dir, "run.state.json") + state = {} + if os.path.exists(path): + with open(path, encoding="utf-8") as fh: + state = json.load(fh) + self._send(200, {"state": state}, cn, "status") + return + self._send(404, {"error": "not found"}, cn, None) + + def do_POST(self): + if self.path == "/search": + cn, denied = self._authorize("search") + if denied: + self._send(denied[0], denied[1], cn, "search") + return + try: + req = self._body() + query = req["query"] + k = int(req.get("k", 5)) + if not isinstance(query, str) or not query.strip(): + raise ValueError("query must be a non-empty string") + k = max(1, min(SEARCH_K_MAX, k)) + except (ValueError, KeyError, TypeError) as e: + self._send(400, {"error": "bad request: %s" % e}, cn, "search") + return + qhash = hashlib.sha256(query.encode("utf-8")).hexdigest()[:16] + try: + hits = _embed.search(self.policy.manifest, self.out_dir, query, k=k, + env=self.env, endpoint=self.embed_endpoint) + except (_embed.EmbedError, OSError) as e: + self._send(502, {"error": "search unavailable: %s" % e}, cn, "search", {"q": qhash}) + return + self._send(200, {"hits": hits, "k": k}, cn, "search", {"q": qhash, "hits": len(hits)}) + return + if self.path == "/read_range": + cn, denied = self._authorize("read_range") + if denied: + self._send(denied[0], denied[1], cn, "read_range") + return + try: + req = self._body() + doc = req["doc"] + start = int(req.get("start", 0)) + end = int(req["end"]) + if not isinstance(doc, str) or len(doc) != 64 or not all(c in "0123456789abcdef" for c in doc): + raise ValueError("doc must be a sanitized-document sha256") + if start < 0 or end <= start: + raise ValueError("start/end out of order") + end = min(end, start + READ_RANGE_MAX) + except (ValueError, KeyError, TypeError) as e: + self._send(400, {"error": "bad request: %s" % e}, cn, "read_range") + return + path = sanitized_store_path(self.out_dir, doc) # the ONLY store with a route + if not os.path.exists(path): + self._send(404, {"error": "no such document"}, cn, "read_range", {"doc": doc[:12]}) + return + with open(path, encoding="utf-8") as fh: + text = fh.read() + piece = text[start:end] + self._send(200, {"doc": doc, "start": start, "end": start + len(piece), + "total": len(text), "text": piece}, + cn, "read_range", {"doc": doc[:12], "chars": len(piece)}) + return + self._send(404, {"error": "not found"}, self._cn(), None) + + +class _Server(ThreadingHTTPServer): + daemon_threads = True + ssl_ctx = None # set by build_server; used by Handler.setup per connection + + def get_request(self): + # Plain TCP accept; the handshake is the handler thread's job (Handler.setup). + sock, addr = self.socket.accept() + sock.settimeout(HANDSHAKE_TIMEOUT) + return sock, addr + + def handle_error(self, request, client_address): + # a client that fails the TLS handshake (no cert, foreign CA) is the + # expected case, not a bug: one line, no traceback, no audit row + # (it never became a request) + exc = sys.exc_info()[1] + if isinstance(exc, (ssl.SSLError, ConnectionError, OSError)): + sys.stderr.write("freeagent serve: rejected %s: %s\n" % (client_address[0], exc)) + return + super().handle_error(request, client_address) + + +def build_ssl_context(cert, key, ca_root): + for label, p in (("FREEAGENT_TLS_CERT", cert), ("FREEAGENT_TLS_KEY", key), ("FREEAGENT_CA_ROOT", ca_root)): + if not p: + raise ServeError("%s is not set — the endpoint is mTLS-only; refusing to serve" % label) + if not os.path.exists(p): + raise ServeError("%s=%s does not exist" % (label, p)) + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + ctx.minimum_version = ssl.TLSVersion.TLSv1_2 + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(ca_root) + ctx.load_cert_chain(cert, key) + return ctx + + +def build_server(manifest_path, out_dir, env=None, allow_unsigned=False, bind=None, + embed_endpoint=None, today=None, port_file=None): + env = dict(os.environ) if env is None else env + ctx = build_ssl_context(env.get("FREEAGENT_TLS_CERT", ""), env.get("FREEAGENT_TLS_KEY", ""), + env.get("FREEAGENT_CA_ROOT", "")) + policy = Policy(manifest_path, env, allow_unsigned=allow_unsigned, today=today) + if policy.manifest is None and not allow_unsigned: + raise ServeError("refusing to start: %s" % policy.gate_error) + os.makedirs(out_dir, exist_ok=True) + handler = type("BoundHandler", (Handler,), { + "policy": policy, "out_dir": out_dir, "env": env, "embed_endpoint": embed_endpoint}) + host, port = parse_bind(bind or env.get("FREEAGENT_BIND", DEFAULT_BIND)) + srv = _Server((host, port), handler, bind_and_activate=False) + if ":" in host: + import socket + srv.address_family = socket.AF_INET6 + srv.socket = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + srv.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + srv.server_bind() + srv.server_activate() + srv.ssl_ctx = ctx # per-connection wrap in Handler.setup, not on the listener + srv.policy = policy + if port_file: + # a supervisor (the bastion) binds port 0 and reads the real port here + tmp = port_file + ".tmp" + with open(tmp, "w", encoding="utf-8") as fh: + fh.write("%d\n" % srv.server_address[1]) + os.replace(tmp, port_file) + return srv + + +def serve(manifest_path, out_dir, env=None, allow_unsigned=False, bind=None, log=None, port_file=None): + log = log or (lambda s: sys.stderr.write(s + "\n")) + srv = build_server(manifest_path, out_dir, env, allow_unsigned=allow_unsigned, bind=bind, + port_file=port_file) + m = srv.policy.manifest + log("freeagent serve: %s on %s (owner %s, %d grant(s), mTLS)" % ( + m.id if m else "?", srv.server_address[:2], m.owner_host if m else "?", len(m.grants) if m else 0)) + try: + srv.serve_forever() + finally: + srv.server_close() diff --git a/bastion_agent/test_build.py b/bastion_agent/test_build.py new file mode 100644 index 0000000..de9d589 --- /dev/null +++ b/bastion_agent/test_build.py @@ -0,0 +1,132 @@ +"""Tests for the zipapp build and the doctor. Run: python3 -m unittest discover .""" + +import io +import json +import os +import shutil +import subprocess +import sys +import tempfile +import unittest +import zipfile + +from . import build as bd +from . import doctor as dr + +HERE = os.path.dirname(os.path.abspath(__file__)) +# Repo root: bastion_agent/ is a top-level package. +REPO = os.path.dirname(HERE) + + +class BuildTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.tmp = tempfile.mkdtemp(prefix="fa-build-") + cls.path, cls.digest, cls.size = bd.build(cls.tmp) + + @classmethod + def tearDownClass(cls): + shutil.rmtree(cls.tmp, ignore_errors=True) + + def test_reproducible(self): + self.assertEqual(bd.build_bytes(), bd.build_bytes()) + p2, d2, _ = bd.build(os.path.join(self.tmp, "again")) + self.assertEqual(d2, self.digest) + with open(self.path + ".sha256") as fh: + self.assertTrue(fh.read().startswith(self.digest)) + + def test_contents(self): + with open(self.path, "rb") as fh: + data = fh.read() + self.assertTrue(data.startswith(b"#!/usr/bin/env python3\n")) + with zipfile.ZipFile(io.BytesIO(data)) as zf: + names = zf.namelist() + times = {zf.getinfo(n).date_time for n in names} + self.assertIn("__main__.py", names) + self.assertIn("bastion_agent/__main__.py", names) + self.assertIn("bastion_agent/serve.py", names) + self.assertIn("bastion_agent/VERSION", names) + self.assertIn("pii/pipeline.py", names) + self.assertIn("yaml/__init__.py", names) + self.assertIn("yaml/LICENSE-NOTICE.txt", names) + self.assertFalse(any("test_" in n for n in names)) + self.assertFalse(any(n.endswith((".so", ".pyc", ".pyd")) for n in names)) + self.assertNotIn("bastion_agent/build.py", names) + self.assertNotIn("bastion_agent/verify_release.py", names) + self.assertNotIn("pii/Dockerfile", names) + self.assertEqual(times, {bd.FIXED_TIME}) + self.assertLess(self.size, 1_000_000) # the ~10MB helper is the bar; this is well under + + def run_pyz(self, *args, env=None, cwd=None): + e = dict(os.environ) + e.update(env or {}) + return subprocess.run([sys.executable, self.path] + list(args), capture_output=True, + text=True, env=e, cwd=cwd or REPO) + + def test_pyz_runs_version_doctor_validate(self): + r = self.run_pyz("--version") + self.assertEqual(r.returncode, 0, r.stderr) + self.assertEqual(r.stdout.strip(), "bastion-agent " + bd.version()) + r = self.run_pyz("doctor", "--json") + self.assertIn(r.returncode, (0, 1), r.stderr) + results = json.loads(r.stdout) + byname = {x["check"]: x for x in results} + self.assertTrue(byname["python >= 3.8"]["ok"]) + self.assertTrue(byname["yaml (manifest loader)"]["ok"]) + r = self.run_pyz("validate", os.path.join(REPO, "manifests", "home-documents.yaml")) + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn("OK", r.stdout) + + def test_pyz_runs_a_full_pipeline_outside_the_repo(self): + # from a scratch directory, no repo on sys.path: walk -> ... -> catalog via the zipapp + work = tempfile.mkdtemp(prefix="fa-pyzrun-") + try: + corpus = os.path.join(work, "c") + os.makedirs(corpus) + with open(os.path.join(corpus, "a.txt"), "w") as fh: + fh.write("hello m@example.com\n") + man = os.path.join(work, "m.yaml") + with open(man, "w") as fh: + fh.write("freeagent: 1\nid: z\nowner: {host: h}\nscope: {roots: ['%s']}\n" + "mask: [read, index]\nstages: [walk, extract, sanitize, catalog]\n" + "sanitize: {mode: flag}\nsweep: manual\n" + "sink: {catalog: local, vectors: local, plaintext: never}\n" % corpus) + r = self.run_pyz("sweep", man, "--out", os.path.join(work, "out"), "--once", + "--allow-unsigned", cwd=work, env={"PYTHONPATH": ""}) + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn("ok", r.stderr) + with open(os.path.join(work, "out", "catalog.jsonl")) as fh: + card = json.loads(fh.readline()) + self.assertIn("[PII_EMAIL_1]", card["head"]) + finally: + shutil.rmtree(work, ignore_errors=True) + + +class DefaultOutTests(unittest.TestCase): + def test_single_root_lives_in_dot_agent(self): + from .__main__ import default_out + from .manifest import validate_document + m = validate_document({"freeagent": 1, "id": "d", "owner": {"host": "h"}, + "scope": {"roots": ["/srv/docs"]}, "mask": ["read"], "stages": ["walk"], + "sweep": "manual", "sink": {"catalog": "local", "vectors": "local", "plaintext": "never"}}) + self.assertEqual(default_out(m), os.path.join(os.path.abspath("/srv/docs"), ".agent")) + m2 = validate_document(dict(m.raw, scope={"roots": ["/a", "/b"]})) + self.assertEqual(default_out(m2), os.path.join("build", "freeagent", "d")) + + +class DoctorTests(unittest.TestCase): + def test_checks_and_render(self): + res = dr.checks(env={"EMBED_URL": "http://127.0.0.1:9", "FREEAGENT_CA_ROOT": "/nonexistent"}) + byname = {x["check"]: x for x in res} + self.assertTrue(byname["python >= 3.8"]["ok"]) + self.assertFalse(byname["embedding endpoint http://127.0.0.1:9"]["ok"]) + self.assertFalse(byname["embedding endpoint http://127.0.0.1:9"]["required"]) + self.assertFalse(byname["FREEAGENT_CA_ROOT"]["ok"]) + self.assertIn("missing file", byname["FREEAGENT_CA_ROOT"]["detail"]) + text = dr.render(res) + self.assertIn("python >= 3.8", text) + self.assertTrue(text.endswith("pass") or "required missing" in text) + + +if __name__ == "__main__": + unittest.main() diff --git a/bastion_agent/test_catalog.py b/bastion_agent/test_catalog.py new file mode 100644 index 0000000..14f3125 --- /dev/null +++ b/bastion_agent/test_catalog.py @@ -0,0 +1,246 @@ +"""Unit tests for the catalog stage. Run: python3 -m unittest discover . + +The summarizer is a loopback stub (same process, test only) speaking both +chat wire formats and echoing a deterministic summary built from the +prompt, so cache and fail-closed behaviour are checked without a model. +""" + +import json +import os +import shutil +import tempfile +import threading +import unittest +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +from . import catalog as cat +from .extract import extract +from .manifest import validate_document +from .sanitize import sanitize +from .walk import walk + + +def manifest(root, summarize=None): + doc = { + "freeagent": 1, "id": "t", "owner": {"host": "h"}, + "scope": {"roots": [root]}, + "mask": ["read", "search", "index"] + (["summarize"] if summarize else []), + "stages": ["walk", "extract", "sanitize", "catalog"], "sweep": "manual", + "sanitize": {"mode": "flag"}, + "sink": {"catalog": "local", "vectors": "local", "plaintext": "never"}, + } + if summarize: + doc["summarize"] = {"model": "stub-chat", "per": summarize} + return validate_document(doc) + + +def put(root, rel, text): + path = os.path.join(root, *rel.split("/")) + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8") as fh: + fh.write(text) + + +class _StubChat(BaseHTTPRequestHandler): + serve = ("ollama", "openai") + calls = [] + fail = False + + def do_POST(self): + n = int(self.headers.get("Content-Length", "0")) + req = json.loads(self.rfile.read(n)) + prompt = req["messages"][-1]["content"] + _StubChat.calls.append((self.path, req["model"], prompt)) + if _StubChat.fail or (self.path == "/api/chat" and "ollama" not in self.serve) \ + or (self.path == "/v1/chat/completions" and "openai" not in self.serve) \ + or self.path not in ("/api/chat", "/v1/chat/completions"): + self.send_response(500 if _StubChat.fail else 404) + self.send_header("Content-Length", "0") + self.end_headers() + return + body_text = prompt.split("\n\n", 1)[-1] + summary = 'hmm "SUMMARY of: %s"' % body_text[:40].replace("\n", " ") + if self.path == "/api/chat": + body = {"message": {"role": "assistant", "content": summary}} + else: + body = {"choices": [{"message": {"role": "assistant", "content": summary}}]} + data = json.dumps(body).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def log_message(self, *a): + pass + + +class UnitTests(unittest.TestCase): + def test_title_head_keywords(self): + text = "# Quarterly Budget Review\n\nSpending on routers and firewalls rose. Routers again.\n" + self.assertEqual(cat.title_of(text), "Quarterly Budget Review") + self.assertTrue(cat.head_of(text).startswith("# Quarterly Budget Review Spending")) + counts = cat.term_counts(text) + self.assertEqual(counts["routers"], 2) + self.assertNotIn("and", counts) + kws = cat.keywords(counts, {"routers": 1}, 1, 3) + self.assertEqual(kws[0], "routers") + self.assertEqual(kws, cat.keywords(counts, {"routers": 1}, 1, 3)) # deterministic + + def test_mask_name(self): + detect = cat._rules_detect() + # the email rule legitimately spans dots and hyphens, so a name that + # embeds an address is masked conservatively — whole span, not just + # the address; the card keeps mime/size so nothing is lost but the name + self.assertEqual(cat.mask_name("invoice-m@example.com.pdf", detect), "[PII_EMAIL]") + self.assertEqual(cat.mask_name("invoice m@example.com.pdf", detect), "invoice [PII_EMAIL]") + self.assertEqual(cat.mask_name("notes.txt", detect), "notes.txt") + self.assertEqual(cat.mask_name("call 415-555-0173 notes", detect), "call [PII_PHONE] notes") + + def test_clean_summary(self): + self.assertEqual(cat.clean_summary('x "A b" '), "A b") + self.assertEqual(len(cat.clean_summary("x" * 2000)), cat.SUMMARY_MAX_CHARS) + + +class Case(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.srv = ThreadingHTTPServer(("127.0.0.1", 0), _StubChat) + threading.Thread(target=cls.srv.serve_forever, daemon=True).start() + cls.url = "http://127.0.0.1:%d" % cls.srv.server_address[1] + + @classmethod + def tearDownClass(cls): + cls.srv.shutdown() + cls.srv.server_close() + + def setUp(self): + self.root = tempfile.mkdtemp(prefix="fa-cat-") + self.out = tempfile.mkdtemp(prefix="fa-catout-") + _StubChat.serve, _StubChat.calls, _StubChat.fail = ("ollama", "openai"), [], False + put(self.root, "finance/budget.md", "# Budget 2026\n\nRouters and firewalls. Contact m@example.com.\n") + put(self.root, "finance/invoice-m@example.com.txt", "invoice for routers 4111 1111 1111 1111\n") + put(self.root, "notes/orchard.txt", "apples pears plums harvest\n") + put(self.root, "blob.bin", "\x00\x01") + self.env = {"SUMMARIZE_URL": self.url} + + def tearDown(self): + shutil.rmtree(self.root, ignore_errors=True) + shutil.rmtree(self.out, ignore_errors=True) + + def run_stage(self, m): + walk(m, self.out) + extract(m, self.out, use_external=False) + sanitize(m, self.out, env={}) + s = cat.catalog(m, self.out, env=self.env) + with open(os.path.join(self.out, "catalog.jsonl"), encoding="utf-8") as fh: + files = {json.loads(l)["path"]: json.loads(l) for l in fh} + with open(os.path.join(self.out, "catalog.dirs.jsonl"), encoding="utf-8") as fh: + dirs = {json.loads(l)["path"]: json.loads(l) for l in fh} + return s, files, dirs + + +class StageTests(Case): + def test_cards_without_summarizer(self): + s, files, dirs = self.run_stage(manifest(self.root)) + self.assertIsNone(s["summarizer"]) + self.assertEqual(_StubChat.calls, []) + b = files["finance/budget.md"] + self.assertEqual(b["title"], "Budget 2026") + self.assertIn("[PII_EMAIL_1]", b["head"]) + self.assertNotIn("m@example.com", json.dumps(b)) + self.assertEqual(b["entities"], {"EMAIL": 1}) + self.assertIn("routers", b["keywords"]) + self.assertEqual(b["text"], "ok") + self.assertEqual(b["mime"], "text/markdown") + self.assertNotIn("summary", b) + # filename with an email is masked in the card + self.assertIn("finance/[PII_EMAIL]", files) # conservative whole-span mask + self.assertEqual(files["finance/[PII_EMAIL]"]["mime"], "text/plain") + self.assertEqual(s["counts"]["names_masked"], 1) + # placeholders never become keywords + self.assertNotIn("pii", b["keywords"]) + self.assertNotIn("email", b["keywords"]) + # a file with no text still gets a card + self.assertEqual(files["blob.bin"]["text"], "unsupported") + self.assertNotIn("head", files["blob.bin"]) + # directories: root "" aggregates everything + self.assertEqual(dirs[""]["files"], 4) + self.assertEqual(dirs["finance"]["files"], 2) + self.assertEqual(dirs["finance"]["with_text"], 2) + self.assertEqual(dirs["finance"]["entities"], {"EMAIL": 1, "CREDIT_CARD": 1}) + self.assertIn("routers", dirs["finance"]["keywords"]) + self.assertIn("apples", dirs["notes"]["keywords"]) + self.assertEqual(dirs["finance"]["types"]["text/markdown"], 1) + self.assertEqual(s["counts"], dict(files=4, with_text=3, dirs=3, summaries=0, + summaries_reused=0, summary_errors=0, names_masked=1)) + + def test_parent_directory_names_are_masked_too(self): + # a directory named with an address must not leak through + # a file card's path, which is what seal ships + put(self.root, "marisol.vega@example.com/notes.txt", "plain\n") + s, files, dirs = self.run_stage(manifest(self.root)) + self.assertIn("[PII_EMAIL]/notes.txt", files) + self.assertIn("[PII_EMAIL]", dirs) + blob = json.dumps(files) + json.dumps(dirs) + self.assertNotIn("marisol.vega", blob) + self.assertEqual(s["counts"]["names_masked"], 2) # the invoice file + the directory + + def test_catalog_carries_no_values(self): + _, files, dirs = self.run_stage(manifest(self.root)) + blob = json.dumps(files) + json.dumps(dirs) + for leak in ("m@example.com", "4111"): + self.assertNotIn(leak, blob) + + def test_summaries_file_and_directory_with_cache(self): + m = manifest(self.root, summarize=["file", "directory"]) + s, files, dirs = self.run_stage(m) + self.assertTrue(s["summarizer"].startswith("stub-chat@")) + self.assertTrue(s["summarizer"].endswith("(ollama)")) + self.assertTrue(files["finance/budget.md"]["summary"].startswith("SUMMARY of:")) + self.assertNotIn("", files["finance/budget.md"]["summary"]) + self.assertIsNone(files["blob.bin"]["summary"]) + self.assertTrue(dirs["finance"]["summary"].startswith("SUMMARY of:")) + self.assertEqual(s["counts"]["summaries"], 3 + 3) # 3 texts + 3 dirs + # the model only ever saw sanitized text + for _, _, prompt in _StubChat.calls: + self.assertNotIn("m@example.com", prompt) + self.assertNotIn("4111", prompt) + # second run: all summaries from cache, no calls + _StubChat.calls = [] + s2, _, _ = self.run_stage(m) + self.assertEqual((s2["counts"]["summaries"], s2["counts"]["summaries_reused"]), (0, 6)) + self.assertEqual(_StubChat.calls, []) + + def test_openai_autodetect(self): + _StubChat.serve = ("openai",) + s, files, _ = self.run_stage(manifest(self.root, summarize=["file"])) + self.assertTrue(s["summarizer"].endswith("(openai)")) + self.assertTrue(files["notes/orchard.txt"]["summary"]) + + def test_summarizer_failure_stops_but_cards_are_written(self): + _StubChat.fail = True + s, files, dirs = self.run_stage(manifest(self.root, summarize=["file", "directory"])) + self.assertIsNotNone(s["stopped"]) + self.assertEqual(s["counts"]["summary_errors"], 1) # stopped after the first + self.assertEqual(len(files), 4) + self.assertEqual(len(dirs), 3) + errs = [c for c in list(files.values()) + list(dirs.values()) if c.get("summary_error")] + self.assertEqual(len(errs), 1) + self.assertTrue(all(c.get("summary") is None for c in files.values())) + + def test_unreachable_summarizer(self): + m = manifest(self.root, summarize=["file"]) + walk(m, self.out) + extract(m, self.out, use_external=False) + sanitize(m, self.out, env={}) + s = cat.catalog(m, self.out, env={"SUMMARIZE_URL": "http://127.0.0.1:9"}) + self.assertIn("summarizer", s["stopped"]) + + def test_requires_earlier_stages(self): + with self.assertRaises(FileNotFoundError): + cat.catalog(manifest(self.root), self.out, env=self.env) + + +if __name__ == "__main__": + unittest.main() diff --git a/bastion_agent/test_embed.py b/bastion_agent/test_embed.py new file mode 100644 index 0000000..6ee255a --- /dev/null +++ b/bastion_agent/test_embed.py @@ -0,0 +1,262 @@ +"""Unit tests for chunk / embed / search. Run: python3 -m unittest discover . + +The embedding endpoint is a stub HTTP server on loopback (same process, +test only) speaking both wire formats with a deterministic bag-of-words +vector, so search ranking is meaningful without a model. +""" + +import hashlib +import json +import os +import re +import shutil +import sqlite3 +import tempfile +import threading +import unittest +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +from . import embed as em +from .extract import extract +from .manifest import validate_document +from .sanitize import sanitize +from .walk import walk + + +def manifest(root, model="stub-model", dim=64, size=8, overlap=2): + return validate_document({ + "freeagent": 1, "id": "t", "owner": {"host": "h"}, + "scope": {"roots": [root]}, + "mask": ["read", "search", "index"], + "stages": ["walk", "extract", "sanitize", "chunk", "embed"], "sweep": "manual", + "sanitize": {"mode": "flag"}, + "embed": {"model": model, "dim": dim, "chunk": {"size_tokens": size, "overlap_tokens": overlap}}, + "sink": {"catalog": "local", "vectors": "local", "plaintext": "never"}, + }) + + +def bow_vector(text, dim): + v = [0.0] * dim + for w in re.findall(r"\w+", text.lower()): + h = int(hashlib.md5(w.encode()).hexdigest(), 16) + v[h % dim] += 1.0 + return v + + +class _Stub(BaseHTTPRequestHandler): + dim = 64 + serve = ("ollama", "openai") + calls = [] + + def do_POST(self): + n = int(self.headers.get("Content-Length", "0")) + req = json.loads(self.rfile.read(n)) + _Stub.calls.append((self.path, req["model"], len(req["input"]))) + inputs = req["input"] if isinstance(req["input"], list) else [req["input"]] + if self.path == "/api/embed" and "ollama" in self.serve: + body = {"embeddings": [bow_vector(t, self.dim) for t in inputs]} + elif self.path == "/v1/embeddings" and "openai" in self.serve: + body = {"data": [{"index": i, "embedding": bow_vector(t, self.dim)} + for i, t in enumerate(inputs)]} + else: + self.send_response(404) + self.send_header("Content-Length", "0") + self.end_headers() + return + data = json.dumps(body).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def log_message(self, *a): + pass + + +class ChunkUnitTests(unittest.TestCase): + def test_windows_cover_text_and_overlap(self): + text = " ".join("w%d" % i for i in range(20)) + wins = list(em.chunk_text(text, 8, 2)) + self.assertEqual(wins[0][0], 0) + self.assertEqual(wins[-1][1], len(text)) + # 20 tokens, step 6: windows at 0, 6, 12 — the third reaches the end + self.assertEqual([em.count_tokens(w[2]) for w in wins], [8, 8, 8]) + # step = 6: window 2 starts at token 6 = "w6" + self.assertTrue(wins[1][2].startswith("w6 ")) + # every char is inside at least one window + covered = set() + for s, e, _ in wins: + covered.update(range(s, e)) + self.assertEqual(len(covered), len(text)) + + def test_short_and_empty(self): + self.assertEqual(list(em.chunk_text("", 8, 2)), []) + self.assertEqual(list(em.chunk_text(" \n ", 8, 2)), []) + wins = list(em.chunk_text("just three tokens", 8, 2)) + self.assertEqual(len(wins), 1) + self.assertEqual(wins[0][2], "just three tokens") + + def test_punctuation_is_a_token_and_ids_are_content_addressed(self): + self.assertEqual(em.count_tokens("Hi, there!"), 4) + self.assertEqual(em.chunk_id("d", "x"), em.chunk_id("d", "x")) + self.assertNotEqual(em.chunk_id("d", "x"), em.chunk_id("e", "x")) + + def test_pack_roundtrip(self): + v = [0.5, -1.25, 3.0] + self.assertEqual(list(em.unpack(em.pack(v))), v) + self.assertAlmostEqual(em.cosine([1, 0], [1, 0]), 1.0) + self.assertAlmostEqual(em.cosine([1, 0], [0, 1]), 0.0) + self.assertEqual(em.cosine([0, 0], [1, 1]), 0.0) + + +class Case(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.srv = ThreadingHTTPServer(("127.0.0.1", 0), _Stub) + threading.Thread(target=cls.srv.serve_forever, daemon=True).start() + cls.url = "http://127.0.0.1:%d" % cls.srv.server_address[1] + + @classmethod + def tearDownClass(cls): + cls.srv.shutdown() + cls.srv.server_close() + + def setUp(self): + self.root = tempfile.mkdtemp(prefix="fa-em-") + self.out = tempfile.mkdtemp(prefix="fa-emout-") + _Stub.dim, _Stub.serve, _Stub.calls = 64, ("ollama", "openai"), [] + self.put("apples.txt", "apples pears plums. " * 12 + "orchard harvest.") + self.put("routers.txt", "router switch firewall vlan. " * 12 + "network segment.") + self.put("twin.txt", "apples pears plums. " * 12 + "orchard harvest.") + self.env = {"EMBED_URL": self.url, "EMBED_BATCH": "4"} + + def tearDown(self): + shutil.rmtree(self.root, ignore_errors=True) + shutil.rmtree(self.out, ignore_errors=True) + + def put(self, name, text): + with open(os.path.join(self.root, name), "w", encoding="utf-8") as fh: + fh.write(text) + + def prep(self, m): + walk(m, self.out) + extract(m, self.out, use_external=False) + sanitize(m, self.out, env={}) + + def chunks(self): + with open(os.path.join(self.out, "chunks.jsonl"), encoding="utf-8") as fh: + return [json.loads(l) for l in fh] + + def db(self, sql, *params): + conn = sqlite3.connect(em.vectors_path(self.out)) + try: + return conn.execute(sql, params).fetchall() + finally: + conn.close() + + +class StageTests(Case): + def test_chunk_stage(self): + m = manifest(self.root) + self.prep(m) + s = em.chunk(m, self.out) + rows = self.chunks() + self.assertEqual(s["counts"]["docs"], 2) # twin shares content with apples + self.assertEqual(s["counts"]["chunks"], len(rows)) + self.assertGreater(len(rows), 4) + docs = {r["doc"] for r in rows} + self.assertEqual(len(docs), 2) + self.assertEqual(rows, sorted(rows, key=lambda r: (r["doc"], r["idx"]))) + # deterministic + em.chunk(m, self.out) + self.assertEqual(rows, self.chunks()) + + def test_chunk_requires_sanitize(self): + with self.assertRaises(FileNotFoundError): + em.chunk(manifest(self.root), self.out) + + def test_embed_probe_refuses_wrong_dim(self): + m = manifest(self.root, dim=32) # stub serves 64 + self.prep(m) + em.chunk(m, self.out) + with self.assertRaises(em.EmbedError) as cm: + em.embed(m, self.out, env=self.env) + self.assertIn("serves 64-dim", str(cm.exception)) + self.assertFalse(os.path.exists(em.vectors_path(self.out))) # refused before opening the store + + def test_embed_unreachable_fails_closed(self): + m = manifest(self.root) + self.prep(m) + em.chunk(m, self.out) + with self.assertRaises(em.EmbedError): + em.embed(m, self.out, env={"EMBED_URL": "http://127.0.0.1:9"}) + + def test_embed_writes_reuses_and_keeps_old_models(self): + m = manifest(self.root) + self.prep(m) + em.chunk(m, self.out) + s1 = em.embed(m, self.out, env=self.env) + n = s1["counts"]["unique"] # repeated paragraphs share a chunk id + self.assertLess(n, s1["counts"]["chunks"]) + self.assertEqual(s1["counts"]["embedded"], n) + self.assertEqual(s1["endpoint"], "stub-model@%s(ollama)" % self.url) + # batches of 4 + one probe call + embed_calls = [c for c in _Stub.calls if c[0] == "/api/embed"] + self.assertEqual(sum(c[2] for c in embed_calls), n + 1) + self.assertTrue(all(c[2] <= 4 for c in embed_calls)) + # second run: nothing re-embedded + s2 = em.embed(m, self.out, env=self.env) + self.assertEqual((s2["counts"]["embedded"], s2["counts"]["reused"]), (0, n)) + # a second model ADDS rows; the first model's rows survive + m2 = manifest(self.root, model="other-model") + s3 = em.embed(m2, self.out, env=self.env) + self.assertEqual(s3["counts"]["embedded"], n) + self.assertEqual(sorted(x["model"] for x in s3["models_in_store"]), ["other-model", "stub-model"]) + self.assertEqual(self.db("SELECT COUNT(*) FROM vectors WHERE model='stub-model'")[0][0], n) + + def test_stale_vectors_are_kept_and_counted(self): + m = manifest(self.root) + self.prep(m) + em.chunk(m, self.out) + em.embed(m, self.out, env=self.env) + before = self.db("SELECT COUNT(*) FROM vectors")[0][0] + self.put("routers.txt", "completely different content now.") + self.prep(m) + em.chunk(m, self.out) + s = em.embed(m, self.out, env=self.env) + self.assertGreater(s["stale_vectors"], 0) + self.assertGreaterEqual(self.db("SELECT COUNT(*) FROM vectors")[0][0], before) + # unchanged doc's chunks were reused, not re-embedded + self.assertGreater(s["counts"]["reused"], 0) + + def test_openai_autodetect_and_forced_api(self): + _Stub.serve = ("openai",) + m = manifest(self.root) + self.prep(m) + em.chunk(m, self.out) + s = em.embed(m, self.out, env=self.env) + self.assertTrue(s["endpoint"].endswith("(openai)")) + with self.assertRaises(em.EmbedError): + em.embed(m, self.out, env=dict(self.env, EMBED_API="ollama")) + with self.assertRaises(em.EmbedError): + em.build_endpoint(m, dict(self.env, EMBED_API="magic")) + + def test_search_ranks_the_right_document(self): + m = manifest(self.root) + self.prep(m) + em.chunk(m, self.out) + em.embed(m, self.out, env=self.env) + hits = em.search(m, self.out, "firewall vlan router", k=3, env=self.env) + self.assertEqual(len(hits), 3) + self.assertIn("router", hits[0]["text"]) + self.assertNotIn("apples", hits[0]["text"]) + hits = em.search(m, self.out, "pears and plums", k=1, env=self.env) + self.assertIn("apples", hits[0]["text"]) + self.assertTrue(0 < hits[0]["score"] <= 1.0) + # a model with no vectors yields nothing, not an error + self.assertEqual(em.search(manifest(self.root, model="nope"), self.out, "x", env=self.env), []) + + +if __name__ == "__main__": + unittest.main() diff --git a/bastion_agent/test_extract.py b/bastion_agent/test_extract.py new file mode 100644 index 0000000..628d6d0 --- /dev/null +++ b/bastion_agent/test_extract.py @@ -0,0 +1,438 @@ +"""Unit tests for the extract stage. Run: python3 -m unittest discover . + +Every fixture is generated here with the stdlib (zip + XML for office +formats, hand-built PDFs), so the suite needs no binary blobs and runs on +any OS. External tools are mocked; the real-tool path is exercised only +where the machine happens to have them. +""" + +import io +import json +import os +import shutil +import tempfile +import unittest +import zipfile +import zlib +from unittest import mock + +from . import extract as ex +from .manifest import validate_document +from .walk import walk + + +def manifest(root, **scope): + return validate_document({ + "freeagent": 1, "id": "t", "owner": {"host": "h"}, + "scope": dict({"roots": [root]}, **scope), + "mask": ["read"], "stages": ["walk", "extract"], "sweep": "manual", + "sink": {"catalog": "local", "vectors": "local", "plaintext": "never"}, + }) + + +def put(root, rel, data): + path = os.path.join(root, *rel.split("/")) + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "wb") as fh: + fh.write(data if isinstance(data, bytes) else data.encode("utf-8")) + + +def zipped(members): + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf: + for name, data in members.items(): + zf.writestr(name, data) + return buf.getvalue() + + +DOCX_XML = ('' + 'HelloWorld' + 'Second paragraph') +XLSX_SST = ('' + 'NameAda') +XLSX_SHEET = ('' + '042' + '1inline' + '') +PPTX_SLIDE = ('' + 'Slide ' + 'title') +ODT_CONTENT = ('' + 'HeadingBody span' + 'c1' + 'c2' + '') + + +def pdf_bytes(content, compress=False): + stream = zlib.compress(content) if compress else content + filt = b"/Filter /FlateDecode " if compress else b"" + return (b"%PDF-1.4\n1 0 obj << /Type /Catalog >> endobj\n" + b"2 0 obj << " + filt + b"/Length " + str(len(stream)).encode() + b" >>\nstream\n" + + stream + b"\nendstream\nendobj\n%%EOF\n") + + +class ExtractorUnitTests(unittest.TestCase): + def test_decode_bytes(self): + self.assertEqual(ex.decode_bytes("héllo".encode("utf-8")), ("héllo", "utf-8")) + self.assertEqual(ex.decode_bytes(b"\xef\xbb\xbfhi"), ("hi", "utf-8-sig")) + self.assertEqual(ex.decode_bytes("hi".encode("utf-16"))[0], "hi") + self.assertEqual(ex.decode_bytes("hi there".encode("utf-16-le")), ("hi there", "utf-16-le")) + self.assertEqual(ex.decode_bytes(b"caf\xe9"), ("café", "cp1252")) + + def test_html_to_text(self): + out = ex.html_to_text("T" + "

One & two

Three
") + self.assertIn("One & two", out) + self.assertIn("Three", out) + self.assertNotIn("bad()", out) + self.assertNotIn("x{}", out) + + def test_docx(self): + text = ex.extract_docx(zipped({"word/document.xml": DOCX_XML})) + self.assertEqual(text.strip().splitlines(), ["Hello\tWorld", "Second paragraph"]) + + def test_xlsx(self): + text = ex.extract_xlsx(zipped({"xl/sharedStrings.xml": XLSX_SST, + "xl/worksheets/sheet1.xml": XLSX_SHEET})) + self.assertIn("Name\t42", text) + self.assertIn("Ada\tinline", text) + + def test_pptx(self): + text = ex.extract_pptx(zipped({"ppt/slides/slide2.xml": PPTX_SLIDE, + "ppt/slides/slide1.xml": PPTX_SLIDE.replace("title", "first")})) + lines = [l for l in text.splitlines() if l] + self.assertEqual(lines, ["# slide1", "Slide first", "# slide2", "Slide title"]) + + def test_odf(self): + text = ex.extract_odf(zipped({"content.xml": ODT_CONTENT})) + self.assertEqual(text.splitlines(), ["Heading", "Body span", "c1\tc2"]) + + def test_zip_member_cap(self): + big = zipped({"content.xml": b"x" * (ex.ZIP_MEMBER_CAP + 1)}) + with self.assertRaises(ValueError): + ex.extract_odf(big) + + def test_office_refuses_a_dtd(self): + # Office XML never carries a DTD; one is how entity expansion gets in. + bomb = (b'' + b']>' + b'' + b'&b;') + with self.assertRaises(ValueError): + ex.extract_docx(zipped({"word/document.xml": bomb})) + + def test_office_text_cap_stops_with_the_text_so_far(self): + rows = "".join('row %d' % i for i in range(200)) + sheet = ('' + "%s" % rows).encode() + with mock.patch.object(ex, "OFFICE_TEXT_CAP", 300): + text = ex.extract_xlsx(zipped({"xl/worksheets/sheet1.xml": sheet})) + self.assertIn("row 0", text) + self.assertNotIn("row 199", text) + self.assertTrue(text.splitlines()[-1].startswith("[stopped reading here:")) + + def test_office_decompressed_cap_counts_every_member(self): + para = b"x" * 2000 + doc = (b'' + b"" + para + b"") + files = {"word/document.xml": doc} + files.update({"word/header%d.xml" % i: doc for i in range(1, 6)}) + with mock.patch.object(ex, "OFFICE_DOC_CAP", len(doc) * 3): + text = ex.extract_docx(zipped(files)) + self.assertIn("[stopped reading here: the document expands past", text) + + def test_office_parse_is_streamed(self): + # Rows are dropped as they are read: the parsed tree never holds them all. + import tracemalloc + rows = "".join('%s' % ("v" * 50) + for _ in range(60000)) + sheet = ('' + "%s" % rows).encode() + data = zipped({"xl/worksheets/sheet1.xml": sheet}) + tracemalloc.start() + text = ex.extract_xlsx(data) + peak = tracemalloc.get_traced_memory()[1] + tracemalloc.stop() + self.assertEqual(text.count("v" * 50), 60000) + # the XML is ~4.6 MB; a whole-tree parse peaks at many times that + self.assertLess(peak, 3 * len(sheet), peak) + + def test_office_drops_unwanted_siblings_linear(self): + body = b"hi" * 100000 + doc = (b'' + b"" + body + b"") + import time as _t + t0 = _t.monotonic() + text = ex.extract_docx(zipped({"word/document.xml": doc})) + self.assertLess(_t.monotonic() - t0, 10) + self.assertEqual(text.count("hi"), 100000) + self.assertNotIn("[stopped", text) + + def test_office_one_paragraph_with_endless_parts_is_cut_and_reading_goes_on(self): + runs = b"a" * 10 + doc = (b'' + b"" + runs + b"after" + b"") + with mock.patch.object(ex, "OFFICE_PARTS_PER_ITEM", 15): + text = ex.extract_docx(zipped({"word/document.xml": doc})) + lines = text.splitlines() + self.assertTrue(lines[0].startswith("a") and len(lines[0]) < 10, lines[0]) + self.assertIn("after", lines) + self.assertEqual(lines[-1], "[1 paragraph(s) or row(s) held more than 15 parts; each was cut there]") + + def test_office_dropping_past_the_cap_is_linear(self): + # A closing element is not its parent's last child when the pull + # parser has read ahead; per-element remove() was quadratic (400k + # runs: 60 s and the rest of the document lost). + import time as _t + runs = b"a" * 60000 + doc = (b'' + b"" + runs + b"TAIL" + b"") + with mock.patch.object(ex, "OFFICE_PARTS_PER_ITEM", 20000): + t0 = _t.monotonic() + text = ex.extract_docx(zipped({"word/document.xml": doc})) + self.assertLess(_t.monotonic() - t0, 10) + self.assertIn("TAIL", text.splitlines()) + + def test_office_cut_row_has_no_phantom_cells(self): + cells = "".join('v' for _ in range(40)) + sheet = ('' + "%snext" + "" % cells).encode() + with mock.patch.object(ex, "OFFICE_PARTS_PER_ITEM", 30): + text = ex.extract_xlsx(zipped({"xl/worksheets/sheet1.xml": sheet})) + first = text.splitlines()[1] + self.assertTrue(all(c == "v" for c in first.split("\t")), first) + self.assertIn("next", text.splitlines()) + + def test_office_utf16_is_read_and_its_dtd_still_refused(self): + ns = 'xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"' + good = ('' + "wide caf\u00e9" % ns) + for enc in ("utf-16", "utf-16-be"): + data = good.encode(enc) + if enc == "utf-16-be": + data = b"\xfe\xff" + data + self.assertEqual(ex.extract_docx(zipped({"word/document.xml": data})).strip(), "wide caf\u00e9") + bomb = (']>' + "&a;" % ns) + with self.assertRaises(ValueError): + ex.extract_docx(zipped({"word/document.xml": bomb.encode("utf-16")})) + with self.assertRaises(ValueError): # UTF-32: not an Office encoding + ex.extract_docx(zipped({"word/document.xml": good.replace("UTF-16", "UTF-32").encode("utf-32-le")})) + + def test_docx_text_box_follows_the_paragraph_holding_it(self): + ns = b'xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"' + doc = (b"" + b"before" + b"holder" + b"boxed" + b"after") + text = ex.extract_docx(zipped({"word/document.xml": doc})) + self.assertEqual([l for l in text.splitlines() if l], ["before", "holder", "boxed", "after"]) + + def test_eml_plain_and_html(self): + plain = b"From: a@b.c\nTo: d@e.f\nSubject: Hi\n\nbody text\n" + out = ex.extract_eml(plain) + self.assertIn("Subject: Hi", out) + self.assertIn("body text", out) + htm = (b"From: a@b.c\nSubject: H\nContent-Type: text/html\n\n" + b"

rich body

") + out = ex.extract_eml(htm) + self.assertIn("rich body", out) + self.assertNotIn("", out) + + def test_rtf(self): + rtf = (rb"{\rtf1\ansi{\fonttbl{\f0 Arial;}}{\colortbl;\red0;}" + rb"\f0 Hello\par caf\'e9 \u8364? end\tab tabbed}") + out = ex.extract_rtf(rtf) + self.assertEqual(out, "Hello\ncafé € end\ttabbed") + self.assertNotIn("Arial", out) + + def test_pdf_builtin_plain_and_flate(self): + content = b"BT /F1 12 Tf 72 700 Td (Hello) Tj T* [(Wor) -20 (ld)] TJ ET" + self.assertIn("Hello", ex.extract_pdf_builtin(pdf_bytes(content))) + out = ex.extract_pdf_builtin(pdf_bytes(content, compress=True)) + self.assertIn("Hello", out) + self.assertIn("World", out.replace(" ", "").replace("\n", "") and out) + + def test_pdf_builtin_refuses_garbage_and_encrypted(self): + garbage = b"BT (\x01\x02\x03\x04\x05\x06\x07\x08\x0b\x0e\x0f\x10\x11\x12) Tj ET" + self.assertIsNone(ex.extract_pdf_builtin(pdf_bytes(garbage))) + self.assertIsNone(ex.extract_pdf_builtin(pdf_bytes(b"BT (x) Tj ET") + b"/Encrypt 5 0 R")) + self.assertIsNone(ex.extract_pdf_builtin(pdf_bytes(b"no text ops here"))) + + def test_pdf_escapes(self): + from . import pdftext as pt + self.assertEqual(pt._Lexer(rb"(a\)b\\c\101\n)").token(), ("str", b"a)b\\cA\n")) + + +class StageCase(unittest.TestCase): + def setUp(self): + self.root = tempfile.mkdtemp(prefix="fa-ex-") + self.out = tempfile.mkdtemp(prefix="fa-exout-") + put(self.root, "a.txt", "plain text\n") + put(self.root, "copy-of-a.txt", "plain text\n") + put(self.root, "latin.txt", "caf\xe9".encode("cp1252")) + put(self.root, "page.html", "

hello web

") + put(self.root, "data.csv", "a,b\n1,2\n") + put(self.root, "conf.json", '{"k": 1}') + put(self.root, "mail.eml", "From: a@b.c\nSubject: S\n\nhi\n") + put(self.root, "doc.docx", zipped({"word/document.xml": DOCX_XML})) + put(self.root, "sheet.xlsx", zipped({"xl/sharedStrings.xml": XLSX_SST, + "xl/worksheets/sheet1.xml": XLSX_SHEET})) + put(self.root, "deck.pptx", zipped({"ppt/slides/slide1.xml": PPTX_SLIDE})) + put(self.root, "text.odt", zipped({"content.xml": ODT_CONTENT})) + put(self.root, "doc.pdf", pdf_bytes(b"BT (pdf words) Tj ET", compress=True)) + put(self.root, "scan.png", b"\x89PNG\r\n\x1a\n" + b"\x00" * 32) + put(self.root, "blob.bin", b"\x00\x01\x02\x03") + put(self.root, "empty.txt", "") + put(self.root, "noext-empty", "") + put(self.root, "broken.docx", b"PK\x03\x04not really a zip") + put(self.root, "big.txt", "x" * 100_000) + + def tearDown(self): + shutil.rmtree(self.root, ignore_errors=True) + shutil.rmtree(self.out, ignore_errors=True) + + def run_stage(self, use_external=False, **scope): + m = manifest(self.root, **scope) + walk(m, self.out) + s = ex.extract(m, self.out, use_external=use_external) + with open(os.path.join(self.out, "extract.jsonl"), encoding="utf-8") as fh: + idx = {json.loads(l)["path"]: json.loads(l) for l in fh} + return s, idx + + def text_of(self, rec): + with open(ex.store_path(self.out, rec["sha256"]), encoding="utf-8") as fh: + return fh.read() + + +class StageTests(StageCase): + def test_everything_builtin(self): + s, idx = self.run_stage(max_file_bytes=50_000) + self.assertEqual(idx["a.txt"]["status"], "ok") + self.assertEqual(idx["a.txt"]["extractor"], "text:utf-8") + self.assertEqual(self.text_of(idx["a.txt"]), "plain text\n") + self.assertEqual(idx["latin.txt"]["extractor"], "text:cp1252") + self.assertEqual(self.text_of(idx["latin.txt"]), "café\n") + self.assertEqual(self.text_of(idx["page.html"]).strip(), "hello web") + self.assertEqual(self.text_of(idx["data.csv"]), "a\tb\n1\t2\n") + self.assertEqual(idx["conf.json"]["status"], "ok") + self.assertIn("Subject: S", self.text_of(idx["mail.eml"])) + self.assertIn("Hello\tWorld", self.text_of(idx["doc.docx"])) + self.assertIn("Name\t42", self.text_of(idx["sheet.xlsx"])) + self.assertIn("Slide title", self.text_of(idx["deck.pptx"])) + self.assertIn("Heading", self.text_of(idx["text.odt"])) + self.assertEqual(idx["doc.pdf"]["extractor"], "pdf-builtin") + self.assertIn("pdf words", self.text_of(idx["doc.pdf"])) + self.assertEqual(idx["scan.png"]["status"], "unsupported") + self.assertIn("tesseract", idx["scan.png"]["error"]) + self.assertEqual(idx["blob.bin"]["status"], "unsupported") + self.assertEqual(idx["empty.txt"]["status"], "empty") + # same (empty) content as empty.txt -> deduped onto its record + self.assertEqual(idx["noext-empty"]["status"], "empty") + self.assertEqual(idx["noext-empty"]["sha256"], idx["empty.txt"]["sha256"]) + # the inode/x-empty branch on its own + self.assertEqual(ex.Extractor(use_external=False).extract("/nonexistent", "inode/x-empty"), + ("ok", "empty", "", None)) + self.assertEqual(idx["broken.docx"]["status"], "error") + self.assertEqual(idx["big.txt"]["status"], "skipped") + self.assertEqual(idx["big.txt"]["error"], "too_large") + # duplicate content: one store entry, both paths indexed + self.assertEqual(idx["copy-of-a.txt"]["sha256"], idx["a.txt"]["sha256"]) + c = s["counts"] + self.assertEqual(c["files"], len(idx)) + self.assertEqual(c["skipped"], 1) + self.assertEqual(c["error"], 1) + self.assertEqual(c["unsupported"], 2) + self.assertEqual(c["empty"], 2) # empty.txt + its content-twin noext-empty + self.assertEqual(s["tools"], {"pdftotext": None, "tesseract": None}) + + def test_reuse_across_sweeps(self): + s1, _ = self.run_stage() + s2, idx = self.run_stage() + # everything that produced text (or was empty) last sweep, incl. the duplicate copy + self.assertEqual(s2["counts"]["reused"], s1["counts"]["ok"] + s1["counts"]["empty"]) + self.assertEqual(s2["counts"]["ok"], 0) + self.assertTrue(idx["a.txt"]["reused"]) + # unsupported verdicts are reused while the tool set is unchanged + self.assertTrue(idx["scan.png"].get("reused")) + # a changed file is re-extracted + put(self.root, "a.txt", "new text\n") + s3, idx3 = self.run_stage() + self.assertEqual(self.text_of(idx3["a.txt"]), "new text\n") + self.assertFalse(idx3["a.txt"].get("reused")) + + def test_cached_verdicts_invalidate_on_extractor_upgrade(self): + self.run_stage() + with mock.patch.object(ex, "EXTRACT_VERSION", ex.EXTRACT_VERSION + 1): + s, idx = self.run_stage() + self.assertEqual(s["counts"]["reused"], 0) + self.assertFalse(idx["a.txt"].get("reused")) + self.assertFalse(idx["scan.png"].get("reused")) + + def test_unsupported_retried_when_tools_change(self): + self.run_stage() + fake_tess = lambda path, tool: "OCR TEXT\n" + with mock.patch.object(ex, "tool_path", lambda n: "/usr/bin/" + n), \ + mock.patch.object(ex, "extract_image_tool", fake_tess), \ + mock.patch.object(ex, "extract_pdf_tool", lambda p, t: "from poppler\n"): + s, idx = self.run_stage(use_external=True) + self.assertEqual(idx["scan.png"]["status"], "ok") + self.assertEqual(idx["scan.png"]["extractor"], "tesseract") + self.assertEqual(self.text_of(idx["scan.png"]), "OCR TEXT\n") + self.assertEqual(s["tools"]["pdftotext"], "/usr/bin/pdftotext") + # the pdf was already extracted by the builtin path and is reused, not re-run + self.assertTrue(idx["doc.pdf"].get("reused")) + + def test_tool_failure_is_an_error_entry(self): + def boom(path, tool): + raise RuntimeError("tesseract exited 1: no language data") + with mock.patch.object(ex, "tool_path", lambda n: "/usr/bin/" + n if n == "tesseract" else None), \ + mock.patch.object(ex, "extract_image_tool", boom): + s, idx = self.run_stage(use_external=True) + self.assertEqual(idx["scan.png"]["status"], "error") + self.assertIn("no language data", idx["scan.png"]["error"]) + self.assertEqual(s["counts"]["error"], 2) # + broken.docx + + def test_oversized_csv_field_is_an_error_entry_not_a_crash(self): + # csv.Error is not a ValueError; a >128 KiB field used to escape the + # handler and take the whole sweep down + put(self.root, "huge.csv", "a,b\n1," + "x" * 200_000 + "\n") + s, idx = self.run_stage(max_file_bytes=1_000_000) + self.assertEqual(idx["huge.csv"]["status"], "error") + self.assertIn("Error", idx["huge.csv"]["error"]) + self.assertEqual(s["counts"]["error"], 2) # + broken.docx + + def test_requires_walk_first(self): + with self.assertRaises(FileNotFoundError): + ex.extract(manifest(self.root), self.out) + + def test_index_sorted_and_normalised(self): + put(self.root, "crlf.txt", "one\r\ntwo\r\n\x00") + _, idx = self.run_stage() + self.assertEqual(self.text_of(idx["crlf.txt"]), "one\ntwo\n") + paths = list(idx) + self.assertEqual(paths, sorted(paths)) + + +class RunToolTests(unittest.TestCase): + def test_run_tool_ok_and_failure(self): + import sys + out = ex.run_tool([sys.executable, "-c", "print('hi')"]) + self.assertEqual(out.strip(), b"hi") + with self.assertRaises(RuntimeError) as cm: + ex.run_tool([sys.executable, "-c", "import sys; sys.stderr.write('bad\\n'); sys.exit(3)"]) + self.assertIn("exited 3: bad", str(cm.exception)) + with self.assertRaises(RuntimeError): + ex.run_tool(["/nonexistent/tool"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/bastion_agent/test_manifest.py b/bastion_agent/test_manifest.py new file mode 100644 index 0000000..7fb74e0 --- /dev/null +++ b/bastion_agent/test_manifest.py @@ -0,0 +1,273 @@ +"""Unit tests for the bastion agent manifest loader. Run: python3 -m unittest discover .""" + +import copy +import hashlib +import os +import tempfile +import unittest + +import yaml + +from .manifest import ( + ManifestError, + load_manifest, + signing_digest, + signing_payload, + validate_document, +) + +HERE = os.path.dirname(os.path.abspath(__file__)) +EXAMPLE = os.path.join(HERE, "..", "manifests", "home-documents.yaml") + +MINIMAL = { + "freeagent": 1, + "id": "docs", + "owner": {"host": "owner-host"}, + "scope": {"roots": ["~/Documents"]}, + "mask": ["read"], + "stages": ["walk"], + "sweep": "manual", + "sink": {"catalog": "local", "vectors": "local", "plaintext": "never"}, +} + +FULL = { + "freeagent": 1, + "id": "docs", + "owner": {"host": "owner-host"}, + "scope": {"roots": ["/srv/a", "~/b"], "exclude": ["**/.git/**"]}, + "mask": ["read", "search", "index", "summarize"], + "stages": ["walk", "extract", "sanitize", "chunk", "embed", "catalog", "seal"], + "sanitize": {"mode": "sterilize", "verify": True}, + "embed": {"model": "m", "dim": 1024, "chunk": {"size_tokens": 512, "overlap_tokens": 64}}, + "summarize": {"model": "s", "per": ["file"]}, + "sweep": "daily", + "sink": {"catalog": "host", "vectors": "local", "plaintext": "never", "seal_to": ["k1"]}, + "grants": [{"host": "other-host", "ops": ["search", "read_range"], "until": "2026-12-31"}], +} + + +def errors_of(doc, **kw): + try: + validate_document(doc, **kw) + except ManifestError as exc: + return exc.errors + return [] + + +def with_(base, **changes): + doc = copy.deepcopy(base) + for k, v in changes.items(): + if v is None: + doc.pop(k, None) + else: + doc[k] = v + return doc + + +class ValidTests(unittest.TestCase): + def test_minimal(self): + m = validate_document(MINIMAL) + self.assertEqual(m.stages, ("walk",)) + self.assertEqual(m.scope.max_file_bytes, 256 * 1024 * 1024) + self.assertIsNone(m.sanitize) + self.assertIsNone(m.embed) + self.assertEqual(m.grants, ()) + self.assertFalse(m.signed) + + def test_full(self): + m = validate_document(FULL) + self.assertEqual(m.sanitize.scope, "docs") # defaults to the manifest id + self.assertTrue(m.sanitize.verify) + self.assertEqual(m.embed.chunk_overlap_tokens, 64) + self.assertEqual(m.summarize.per, ("file",)) + self.assertEqual(m.sink.seal_to, ("k1",)) + self.assertEqual(m.grants[0].host, "other-host") + self.assertEqual(m.grants[0].until.isoformat(), "2026-12-31") + + def test_example_file_loads(self): + m = load_manifest(EXAMPLE) + self.assertEqual(m.id, "home-documents") + self.assertEqual(m.stages[-1], "seal") + self.assertEqual(len(m.grants), 1) + + def test_yaml_date_grant(self): + # PyYAML parses a bare 2026-12-31 as a date; both forms are accepted + doc = yaml.safe_load("until: 2026-12-31") + g = with_(FULL, grants=[{"host": "h", "ops": ["search"], "until": doc["until"]}]) + self.assertEqual(errors_of(g), []) + + +class DoctrineTests(unittest.TestCase): + """The rules that exist because of what the product promises.""" + + def test_plaintext_is_an_invariant(self): + doc = with_(MINIMAL, sink={"catalog": "local", "vectors": "local", "plaintext": "host"}) + self.assertTrue(any("invariant" in e for e in errors_of(doc))) + + def test_vectors_local_only(self): + doc = with_(MINIMAL, sink={"catalog": "local", "vectors": "host", "plaintext": "never"}) + self.assertTrue(any("vectors" in e for e in errors_of(doc))) + + def test_read_is_mandatory(self): + self.assertTrue(any("mandatory" in e for e in errors_of(with_(MINIMAL, mask=["search"])))) + + def test_write_does_not_exist(self): + self.assertTrue(any("mask" in e for e in errors_of(with_(MINIMAL, mask=["read", "write"])))) + + def test_stage_needs_mask_op(self): + doc = with_(FULL, mask=["read", "search", "summarize"]) + errs = errors_of(doc) + self.assertTrue(any("'embed' requires 'index'" in e for e in errs)) + self.assertTrue(any("'catalog' requires 'index'" in e for e in errs)) + + def test_host_catalog_needs_seal(self): + doc = with_(FULL, stages=["walk", "extract", "sanitize", "chunk", "embed", "catalog"], + grants=None, sink={"catalog": "host", "vectors": "local", "plaintext": "never"}) + self.assertTrue(any("unsealed" in e for e in errors_of(doc))) + + def test_grant_needs_seal(self): + doc = with_(FULL, stages=["walk", "extract", "sanitize", "chunk", "embed", "catalog"], + sink={"catalog": "local", "vectors": "local", "plaintext": "never"}) + self.assertTrue(any("grants: any grant requires the 'seal'" in e for e in errors_of(doc))) + + def test_grant_search_needs_mask_search(self): + doc = with_(FULL, mask=["read", "index", "summarize"]) + self.assertTrue(any("op 'search' requires 'search' in mask" in e for e in errors_of(doc))) + + def test_grant_to_self_refused(self): + doc = with_(FULL, grants=[{"host": "owner-host", "ops": ["search"], "until": "2026-12-31"}]) + self.assertTrue(any("owner" in e for e in errors_of(doc))) + + def test_require_signed(self): + self.assertTrue(any("signature: required" in e + for e in errors_of(MINIMAL, require_signed=True))) + signed = with_(MINIMAL, signature={"alg": "ecdsa-p256-sha256", "cert": "c.crt", + "value": "AAAA"}) + self.assertEqual(errors_of(signed, require_signed=True), []) + + +class StageTests(unittest.TestCase): + def test_walk_first(self): + self.assertTrue(any("first" in e for e in errors_of(with_(MINIMAL, stages=["extract"])))) + + def test_canonical_order(self): + doc = with_(FULL, stages=["walk", "chunk", "extract", "sanitize", "embed", "catalog", "seal"]) + self.assertTrue(any("canonical order" in e for e in errors_of(doc))) + + def test_prerequisite_stage(self): + doc = with_(MINIMAL, mask=["read", "index"], stages=["walk", "catalog"]) + self.assertTrue(any("'catalog' requires 'sanitize'" in e for e in errors_of(doc))) + # chunk and catalog read the sanitized store: extract alone is not enough + doc = with_(MINIMAL, mask=["read", "index"], stages=["walk", "extract", "catalog"]) + self.assertTrue(any("'catalog' requires 'sanitize'" in e for e in errors_of(doc))) + doc = with_(FULL, stages=["walk", "extract", "chunk", "embed", "catalog", "seal"], sanitize=None) + self.assertTrue(any("'chunk' requires 'sanitize'" in e for e in errors_of(doc))) + + def test_unknown_stage(self): + self.assertTrue(errors_of(with_(MINIMAL, stages=["walk", "upload"]))) + + def test_section_iff_stage(self): + errs = errors_of(with_(MINIMAL, sanitize={"mode": "flag"})) + self.assertTrue(any("not in stages" in e for e in errs)) + errs = errors_of(with_(MINIMAL, stages=["walk", "extract", "sanitize"])) + self.assertTrue(any("sanitize: section required" in e for e in errs)) + errs = errors_of(with_(MINIMAL, mask=["read", "index"], sanitize={"mode": "flag"}, + stages=["walk", "extract", "sanitize", "chunk", "embed"])) + self.assertTrue(any("embed: section required" in e for e in errs)) + + def test_summarize_needs_mask_and_catalog(self): + doc = with_(MINIMAL, summarize={"model": "s", "per": ["file"]}) + errs = errors_of(doc) + self.assertTrue(any("'summarize' is not in mask" in e for e in errs)) + self.assertTrue(any("requires the 'catalog' stage" in e for e in errs)) + + +class FieldTests(unittest.TestCase): + def test_unknown_keys_everywhere(self): + self.assertTrue(any("unknown key 'extra'" in e for e in errors_of(with_(MINIMAL, extra=1)))) + doc = with_(MINIMAL, scope={"roots": ["/a"], "excludes": []}) + self.assertTrue(any("unknown key 'excludes'" in e for e in errors_of(doc))) + + def test_relative_root_refused(self): + doc = with_(MINIMAL, scope={"roots": ["Documents"]}) + self.assertTrue(any("absolute" in e for e in errors_of(doc))) + self.assertTrue(errors_of(with_(MINIMAL, scope={"roots": ["C:relative"]}))) + + def test_windows_and_unc_roots_accepted(self): + for root in (r"C:\Users\me\Documents", "D:/data", r"\\nas\share\docs", "~", "~/x", "/x"): + self.assertEqual(errors_of(with_(MINIMAL, scope={"roots": [root]})), [], root) + + def test_schema_version(self): + self.assertTrue(any("schema version" in e for e in errors_of(with_(MINIMAL, freeagent=2)))) + + def test_slugs(self): + self.assertTrue(errors_of(with_(MINIMAL, id="Home Docs"))) + self.assertTrue(errors_of(with_(MINIMAL, owner={"host": "Owner"}))) + + def test_sweep_vocabulary(self): + self.assertTrue(any("sweep" in e for e in errors_of(with_(MINIMAL, sweep="weekly")))) + + def test_embed_overlap_smaller_than_size(self): + doc = with_(FULL, embed={"model": "m", "dim": 8, + "chunk": {"size_tokens": 64, "overlap_tokens": 64}}) + self.assertTrue(any("smaller than size_tokens" in e for e in errors_of(doc))) + + def test_flag_mode_rejects_verify(self): + doc = with_(FULL, sanitize={"mode": "flag", "verify": True}) + self.assertTrue(any("only valid with mode 'sterilize'" in e for e in errors_of(doc))) + + def test_bad_grant_date(self): + doc = with_(FULL, grants=[{"host": "h", "ops": ["search"], "until": "2026-02-30"}]) + self.assertTrue(any("real YYYY-MM-DD" in e for e in errors_of(doc))) + + def test_bad_signature_base64(self): + doc = with_(MINIMAL, signature={"alg": "ecdsa-p256-sha256", "cert": "c", "value": "*!*"}) + self.assertTrue(any("base64" in e for e in errors_of(doc))) + + def test_all_errors_collected(self): + doc = with_(MINIMAL, freeagent=9, id="BAD", sweep="never", mask=["search"]) + self.assertGreaterEqual(len(errors_of(doc)), 4) + + +class SigningTests(unittest.TestCase): + def test_payload_excludes_signature_and_is_canonical(self): + signed = with_(FULL, signature={"alg": "ecdsa-p256-sha256", "cert": "c", "value": "AAAA"}) + self.assertEqual(signing_payload(signed), signing_payload(FULL)) + # key order in the source must not change the bytes + reordered = dict(reversed(list(FULL.items()))) + self.assertEqual(signing_payload(reordered), signing_payload(FULL)) + self.assertNotIn(b" ", signing_payload(FULL)) + + def test_digest_matches_payload(self): + self.assertEqual(signing_digest(FULL), + hashlib.sha256(signing_payload(FULL)).hexdigest()) + + def test_yaml_date_and_quoted_date_sign_identically(self): + bare = yaml.safe_load("until: 2026-12-31")["until"] + a = with_(FULL, grants=[{"host": "h", "ops": ["search"], "until": bare}]) + b = with_(FULL, grants=[{"host": "h", "ops": ["search"], "until": "2026-12-31"}]) + self.assertEqual(signing_payload(a), signing_payload(b)) + + def test_any_field_change_changes_digest(self): + self.assertNotEqual(signing_digest(FULL), signing_digest(with_(FULL, sweep="manual"))) + + +class FileTests(unittest.TestCase): + def test_missing_file(self): + with self.assertRaises(ManifestError): + load_manifest("/nonexistent/manifest.yaml") + + def test_invalid_yaml(self): + fd, path = tempfile.mkstemp(suffix=".yaml") + with os.fdopen(fd, "w") as fh: + fh.write("freeagent: [1\n") + try: + with self.assertRaises(ManifestError) as cm: + load_manifest(path) + self.assertIn("not valid YAML", cm.exception.errors[0]) + finally: + os.unlink(path) + + +if __name__ == "__main__": + unittest.main() diff --git a/bastion_agent/test_pdftext.py b/bastion_agent/test_pdftext.py new file mode 100644 index 0000000..466fc4f --- /dev/null +++ b/bastion_agent/test_pdftext.py @@ -0,0 +1,541 @@ +"""The agent's built-in PDF reader (pdftext) on the shapes real writers make. + +The regex reader it replaced returned NOTHING for a browser-printed PDF: text +there is glyph codes in hex strings through Type0 / Identity-H fonts, mapped +back by /ToUnicode CMaps, one Td per glyph run. These tests build such files +with a small writer (no third-party code, no user documents) and read them. +""" + +import random +import time +import unittest +import zlib +from unittest import mock + +from . import extract as ex +from . import pdftext as pt + +CMAP = b"""/CIDInit /ProcSet findresource begin 12 dict begin begincmap +1 begincodespacerange <0000> endcodespacerange +2 beginbfchar +<0003> <0020> +<0010> <00660069> +endbfchar +1 beginbfrange +<0024> <003D> <0041> +endbfrange +endcmap CMapName currentdict /CMap defineresource pop end end +""" + + +def _enc(text): + """Glyph codes for our CMap: A-Z -> 0x24.., space -> 3, 'fi' -> 0x10.""" + out = b"" + i = 0 + while i < len(text): + if text[i:i + 2] == "fi": + out += b"\x00\x10" + i += 2 + continue + c = text[i] + out += b"\x00\x03" if c == " " else (0x24 + ord(c) - 0x41).to_bytes(2, "big") + i += 1 + return b"<" + out.hex().upper().encode() + b">" + + +class Pdf: + def __init__(self): + self.objs = [] + + def add(self, body, stream=None, compress=True): + n = len(self.objs) + 1 + if stream is not None: + data = zlib.compress(stream) if compress else stream + flt = b"/Filter /FlateDecode " if compress else b"" + body = body.rstrip(b">") + b" " + flt + b"/Length " + str(len(data)).encode() + b" >>" + self.objs.append(b"%d 0 obj\n" % n + body + b"\nstream\n" + data + b"\nendstream\nendobj\n") + else: + self.objs.append(b"%d 0 obj\n" % n + body + b"\nendobj\n") + return n + + def bytes(self): + return b"%PDF-1.7\n" + b"".join(self.objs) + b"trailer << /Root 1 0 R >>\n%%EOF\n" + + +def browser_like(pages, xobject=False): + """A catalog -> pages tree with RESOURCES ON THE PARENT (inherited), a + Type0 Identity-H font with a ToUnicode CMap, and per-glyph-run Td moves.""" + pdf = Pdf() + pdf.add(b"<< /Type /Catalog /Pages 2 0 R >>") # 1 + pdf.add(b"PLACEHOLDER") # 2 pages + cmap = pdf.add(b"<< >>", CMAP) # 3 + font = pdf.add(b"<< /Type /Font /Subtype /Type0 /BaseFont /F /Encoding /Identity-H " + b"/ToUnicode %d 0 R >>" % cmap) # 4 + kids = [] + form = None + if xobject: + form = pdf.add(b"<< /Type /XObject /Subtype /Form /BBox [0 0 10 10] >>", + b"BT /F1 9 Tf 1 0 0 1 10 10 Tm " + _enc("FOOTER") + b" Tj ET") + for lines in pages: + content = b"BT /F1 12 Tf 1 0 0 1 50 700 Tm " + y = 700 + for line in lines: + words = line.split(" ") + # one Td per word run on the SAME line (x moves only) + for k, w in enumerate(words): + run = (w + " ") if k < len(words) - 1 else w + content += b"[" + _enc(run) + b"] TJ 20 0 Td " + y -= 14 + content += b"1 0 0 1 50 %d Tm " % y + content += b"ET" + if form: + content += b" /X1 Do" + c = pdf.add(b"<< >>", content) + kids.append(pdf.add(b"<< /Type /Page /Parent 2 0 R /Contents %d 0 R >>" % c)) + res = b"/Resources << /Font << /F1 %d 0 R >>" % font + if form: + res += b" /XObject << /X1 %d 0 R >>" % form + res += b" >>" + pdf.objs[1] = (b"2 0 obj\n<< /Type /Pages /Count %d /Kids [%s] %s >>\nendobj\n" + % (len(kids), b" ".join(b"%d 0 R" % k for k in kids), res)) + return pdf.bytes() + + +class PdfTextTests(unittest.TestCase): + def test_type0_identity_h_with_tounicode(self): + data = browser_like([["HELLO WORLD", "SECOND LINE"]]) + self.assertEqual(pt.extract_text(data), "HELLO WORLD\nSECOND LINE") + + def test_the_old_reader_found_nothing_here(self): + # The shape of the regression: no literal (…) strings at all. + data = browser_like([["HELLO WORLD"]]) + for chunk in data.split(b"stream\n")[1:]: + raw = chunk.split(b"\nendstream")[0] + try: + self.assertNotIn(b"HELLO", zlib.decompress(raw)) + except zlib.error: + pass + + def test_words_are_not_shredded_by_per_run_moves(self): + out = pt.extract_text(browser_like([["THE QUICK BROWN FOX"]])) + self.assertEqual(out, "THE QUICK BROWN FOX") + + def test_ligature_codes_map_back_to_letters(self): + out = pt.extract_text(browser_like([["CLARIfiES"]])) + self.assertEqual(out, "CLARIFIES".replace("FI", "fi")) + + def test_pages_in_order_and_inherited_resources(self): + out = pt.extract_text(browser_like([["PAGE ONE"], ["PAGE TWO"], ["PAGE THREE"]])) + self.assertEqual(out.split("\f"), ["PAGE ONE", "PAGE TWO", "PAGE THREE"]) + + def test_form_xobject_text_is_read(self): + out = pt.extract_text(browser_like([["BODY"]], xobject=True)) + self.assertIn("BODY", out) + self.assertIn("FOOTER", out) + + def test_object_streams(self): + # The page tree and font live inside an /ObjStm (PDF 1.5+ writers). + cmap_raw = zlib.compress(CMAP) + content = zlib.compress(b"BT /F1 12 Tf " + _enc("PACKED") + b" Tj ET") + inner = [ + b"<< /Type /Catalog /Pages 2 0 R >>", + b"<< /Type /Pages /Count 1 /Kids [3 0 R] >>", + b"<< /Type /Page /Parent 2 0 R /Contents 6 0 R " + b"/Resources << /Font << /F1 4 0 R >> >> >>", + b"<< /Type /Font /Subtype /Type0 /Encoding /Identity-H /ToUnicode 5 0 R >>", + ] + header, body, off = b"", b"", 0 + for i, o in enumerate(inner, start=1): + header += b"%d %d " % (i, off) + body += o + b" " + off += len(o) + 1 + packed = zlib.compress(header + body) + doc = (b"%PDF-1.7\n" + + (b"5 0 obj\n<< /Filter /FlateDecode /Length %d >>\nstream\n" % len(cmap_raw)) + + cmap_raw + b"\nendstream\nendobj\n" + + (b"6 0 obj\n<< /Filter /FlateDecode /Length %d >>\nstream\n" % len(content)) + + content + b"\nendstream\nendobj\n" + + (b"7 0 obj\n<< /Type /ObjStm /N 4 /First %d /Filter /FlateDecode /Length %d >>\nstream\n" + % (len(header), len(packed))) + + packed + b"\nendstream\nendobj\n%%EOF\n") + self.assertEqual(pt.extract_text(doc), "PACKED") + + def test_extract_pdf_builtin_uses_it(self): + self.assertEqual(ex.extract_pdf_builtin(browser_like([["HELLO"]])), "HELLO") + + def test_hostile_input_never_raises(self): + for junk in (b"", b"%PDF-1.4\n", b"%PDF-1.4\n1 0 obj << /Type /Catalog /Pages 1 0 R >> endobj", + b"%PDF-1.4\n1 0 obj << /Length 99999999 >>\nstream\n\x78\x9c\xff\xff\nendstream\nendobj"): + ex.extract_pdf_builtin(junk) # returns None or text; must not raise + + def test_reference_cycles_do_not_hang(self): + doc = (b"%PDF-1.4\n1 0 obj << /Type /Catalog /Pages 2 0 R >> endobj\n" + b"2 0 obj << /Type /Pages /Kids [2 0 R] >> endobj\n%%EOF") + self.assertIsNone(pt.extract_text(doc)) + + +HELVETICA = (b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /FirstChar 32 /LastChar 126 " + b"/Widths [" + b" ".join([b"500"] * 95) + b"] >>") + + +def simple_doc(streams, font=HELVETICA, objects=(), page_extra=b""): + """One page, font /F1, content from `streams` (each its own stream in + /Contents), `objects` appended as-is after the page.""" + pdf = Pdf() + pdf.add(b"<< /Type /Catalog /Pages 2 0 R >>") # 1 + pdf.add(b"<< /Type /Pages /Count 1 /Kids [4 0 R] >>") # 2 + pdf.add(font) # 3 + first = len(pdf.objs) + 2 + refs = b" ".join(b"%d 0 R" % (first + i) for i in range(len(streams))) + pdf.add(b"<< /Type /Page /Parent 2 0 R /Resources << /Font << /F1 3 0 R >> %s >> " + b"/Contents [%s] >>" % (page_extra, refs)) # 4 + for s in streams: + pdf.add(b"<< >>", s) + for o in objects: + pdf.objs.append(o) + return pdf.bytes() + + +class HostileInput(unittest.TestCase): + """Review B1/B2: a user's file is arbitrary bytes. Every shape here once + raised past the extractor or ran without bound.""" + + def test_kids_that_is_not_an_array(self): + doc = (b"%PDF-1.4\n1 0 obj << /Type /Catalog /Pages 2 0 R >> endobj\n" + b"2 0 obj << /Type /Pages /Kids 5 >> endobj\n%%EOF") + self.assertIsNone(pt.extract_text(doc)) + + def test_objstm_offsets_that_are_not_integers(self): + packed = zlib.compress(b"1 1.e999 2 1e5 << /Type /Catalog >> ") + doc = (b"%PDF-1.7\n" + b"7 0 obj\n<< /Type /ObjStm /N 2 /First 14 /Filter /FlateDecode " + b"/Length %d >>\nstream\n" % len(packed) + packed + b"\nendstream\nendobj\n%%EOF") + self.assertIsNone(pt.extract_text(doc)) + + def test_bfrange_destination_past_its_width(self): + cmap = (b"1 begincodespacerange <0000> endcodespacerange\n" + b"1 beginbfrange <0000> endbfrange\n") + pdf = Pdf() + pdf.add(b"<< /Type /Catalog /Pages 2 0 R >>") + pdf.add(b"<< /Type /Pages /Count 1 /Kids [5 0 R] >>") + tu = pdf.add(b"<< >>", cmap) + pdf.add(b"<< /Type /Font /Subtype /Type0 /ToUnicode %d 0 R >>" % tu) + pdf.add(b"<< /Type /Page /Parent 2 0 R /Resources << /Font << /F1 4 0 R >> >> /Contents 6 0 R >>") + pdf.add(b"<< >>", b"BT /F1 12 Tf <00000001> Tj ET") + # In-range codes map (U+FFF0, U+FFF1); the run past 0xFFFF stops, never raises. + self.assertEqual(pt.extract_text(pdf.bytes()), "\ufff0\ufff1") + + def test_deep_nesting_is_skipped_not_a_recursion_error(self): + doc = (b"%PDF-1.4\n9 0 obj " + b"[" * 100000 + b" endobj\n" + + simple_doc([b"BT /F1 12 Tf (STILL READ) Tj ET"])[len(b"%PDF-1.7\n"):]) + self.assertEqual(pt.extract_text(doc), "STILL READ") + + def test_form_fan_out_is_bounded(self): + # X1 draws X2 ten times, X2 draws X3 ten times … six levels: 10^6 + # invocations unbounded. The budget stops it quickly with the text + # read so far. + objs, first = [], 6 + for lvl in range(6): + num = first + lvl + if lvl < 5: + body = b"BT /F1 12 Tf (L%d) Tj ET " % lvl + b"/X Do " * 10 + res = b"/Resources << /Font << /F1 3 0 R >> /XObject << /X %d 0 R >> >>" % (num + 1) + else: + body, res = b"BT /F1 12 Tf (LEAF) Tj ET", b"/Resources << /Font << /F1 3 0 R >> >>" + objs.append(b"%d 0 obj\n<< /Type /XObject /Subtype /Form %s /Length %d >>\nstream\n" + % (num, res, len(body)) + body + b"\nendstream\nendobj\n") + doc = simple_doc([b"/X Do"], objects=objs, page_extra=b"/XObject << /X 6 0 R >>") + t0 = time.monotonic() + out = pt.extract_text(doc) + self.assertLess(time.monotonic() - t0, 10) + self.assertTrue(out.startswith("L0")) + + def test_cmap_entries_are_bounded(self): + cmap = b"1 begincodespacerange <0000> endcodespacerange\n" + \ + b"1 beginbfrange <0000> <0041> endbfrange\n" * 2000 + pdf = Pdf() + pdf.add(b"<< /Type /Catalog /Pages 2 0 R >>") + pdf.add(b"<< /Type /Pages /Count 1 /Kids [5 0 R] >>") + tu = pdf.add(b"<< >>", cmap) + pdf.add(b"<< /Type /Font /Subtype /Type0 /ToUnicode %d 0 R >>" % tu) + pdf.add(b"<< /Type /Page /Parent 2 0 R /Resources << /Font << /F1 4 0 R >> >> /Contents 6 0 R >>") + pdf.add(b"<< >>", b"BT /F1 12 Tf <0000> Tj ET") + b = pt.Budget() + t0 = time.monotonic() + pt.extract_text(pdf.bytes(), b) + self.assertLess(time.monotonic() - t0, 10) + self.assertEqual(b.reason, "cmap entries") + + def test_a_stream_is_decoded_once(self): + doc = simple_doc([b"BT /F1 12 Tf (ONCE) Tj ET /X Do /X Do /X Do /X Do"], + objects=[b"6 0 obj\n<< /Type /XObject /Subtype /Form /Length 30 >>\nstream\n" + b"BT /F1 12 Tf (FORM) Tj ET \nendstream\nendobj\n"], + page_extra=b"/XObject << /X 6 0 R >>") + with mock.patch.object(pt._Doc, "_decode", autospec=True, side_effect=pt._Doc._decode) as dec: + self.assertIn("FORM", pt.extract_text(doc)) + self.assertEqual(sum(1 for c in dec.call_args_list if c.args[2].startswith(b"BT /F1 12 Tf (FORM)")), 1) + + def test_missing_endobj_is_linear(self): + doc = b"%PDF-1.4\n" + b"1 0 obj << /A 1 >>\n" * 60000 + b"endobj\n" + t0 = time.monotonic() + pt.extract_text(doc) + self.assertLess(time.monotonic() - t0, 5) + + def test_wall_time_budget(self): + b = pt.Budget(seconds=0) + out = pt.extract_text(browser_like([["A"]] * 30), b) + self.assertEqual(b.reason, "time") + self.assertIsNone(out) + + def test_mutated_real_shapes_never_raise(self): + rnd = random.Random(1) + seeds = [browser_like([["HELLO WORLD"], ["PAGE TWO"]], xobject=True), + simple_doc([b"BT /F1 12 Tf (plain) Tj ET"])] + for _ in range(400): + d = bytearray(rnd.choice(seeds)) + for _ in range(rnd.randint(1, 12)): + i = rnd.randrange(len(d)) + d[i:i + rnd.randint(1, 4)] = bytes(rnd.randrange(256) for _ in range(rnd.randint(0, 4))) + ex.extract_pdf_builtin(bytes(d)) # None or text; never raises + + +class Layout(unittest.TestCase): + """Review non-blocking items: where words and lines break.""" + + def test_bt_resets_the_text_matrix(self): + # Two blocks on the SAME line: Td in the second block is from the + # origin again, not from where the first block left off. + out = pt.extract_text(simple_doc([b"BT /F1 10 Tf 50 700 Td (LEFT) Tj ET " + b"BT /F1 10 Tf 200 700 Td (RIGHT) Tj ET"])) + self.assertEqual(out, "LEFT RIGHT") + + def test_words_placed_by_position_get_a_space_from_widths(self): + # No space glyph: the gap between HELLO (5 x 500/1000 x 10 = 25 wide) + # and a run moved 40 to the right is the only word boundary. + out = pt.extract_text(simple_doc([b"BT /F1 10 Tf 0 0 Td (HELLO) Tj 40 0 Td (WORLD) Tj ET"])) + self.assertEqual(out, "HELLO WORLD") + + def test_runs_that_continue_a_word_stay_joined(self): + out = pt.extract_text(simple_doc([b"BT /F1 10 Tf 0 0 Td (HEL) Tj 15 0 Td (LO) Tj ET"])) + self.assertEqual(out, "HELLO") + + def test_without_widths_no_space_is_invented(self): + font = b"<< /Type /Font /Subtype /Type1 /BaseFont /SomeFace >>" + out = pt.extract_text(simple_doc([b"BT /F1 10 Tf 0 0 Td (HEL) Tj 15 0 Td (LO) Tj ET"], font=font)) + self.assertEqual(out, "HELLO") + + def test_cid_widths_from_w_array(self): + pdf = Pdf() + pdf.add(b"<< /Type /Catalog /Pages 2 0 R >>") + pdf.add(b"<< /Type /Pages /Count 1 /Kids [6 0 R] >>") + tu = pdf.add(b"<< >>", CMAP) + desc = pdf.add(b"<< /Type /Font /Subtype /CIDFontType2 /DW 1000 /W [36 [600 600 600 600 600 600]] >>") + pdf.add(b"<< /Type /Font /Subtype /Type0 /Encoding /Identity-H /DescendantFonts [%d 0 R] " + b"/ToUnicode %d 0 R >>" % (desc, tu)) + pdf.add(b"<< /Type /Page /Parent 2 0 R /Resources << /Font << /F1 5 0 R >> >> /Contents 7 0 R >>") + # "AB" at 600/1000 x 10 = 12 wide; the next run 20 on is a new word. + pdf.add(b"<< >>", b"BT /F1 10 Tf 0 0 Td " + _enc("AB") + b" Tj 20 0 Td " + _enc("CD") + b" Tj ET") + self.assertEqual(pt.extract_text(pdf.bytes()), "AB CD") + + def test_winansi_encoding(self): + font = b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>" + out = pt.extract_text(simple_doc([b"BT /F1 10 Tf (caf\\351 \\223q\\224 \\200) Tj ET"], font=font)) + self.assertEqual(out, "caf\u00e9 \u201cq\u201d \u20ac") + + def test_macroman_encoding(self): + font = (b"<< /Type /Font /Subtype /Type1 /BaseFont /Times " + b"/Encoding << /BaseEncoding /MacRomanEncoding >> >>") + out = pt.extract_text(simple_doc([b"BT /F1 10 Tf (caf\\216) Tj ET"], font=font)) + self.assertEqual(out, "caf\u00e9") + + def test_font_carries_across_a_pages_content_streams(self): + pdf = Pdf() + pdf.add(b"<< /Type /Catalog /Pages 2 0 R >>") + pdf.add(b"<< /Type /Pages /Count 1 /Kids [5 0 R] >>") + tu = pdf.add(b"<< >>", CMAP) + pdf.add(b"<< /Type /Font /Subtype /Type0 /Encoding /Identity-H /ToUnicode %d 0 R >>" % tu) + pdf.add(b"<< /Type /Page /Parent 2 0 R /Resources << /Font << /F1 4 0 R >> >> " + b"/Contents [6 0 R 7 0 R] >>") + pdf.add(b"<< >>", b"BT /F1 12 Tf") + pdf.add(b"<< >>", _enc("SPLIT") + b" Tj ET") + self.assertEqual(pt.extract_text(pdf.bytes()), "SPLIT") + + def test_incremental_update_later_definition_wins(self): + doc = simple_doc([b"BT /F1 12 Tf (OLD) Tj ET"]) + body = b"BT /F1 12 Tf (NEW) Tj ET" + doc += (b"5 0 obj\n<< /Length %d >>\nstream\n" % len(body)) + body + b"\nendstream\nendobj\n%%EOF\n" + self.assertEqual(pt.extract_text(doc), "NEW") + + def test_stream_holding_endobj_text_is_read_whole(self): + pdf = Pdf() + pdf.add(b"<< /Type /Catalog /Pages 2 0 R >>") + pdf.add(b"<< /Type /Pages /Count 1 /Kids [4 0 R] >>") + pdf.add(HELVETICA) + pdf.add(b"<< /Type /Page /Parent 2 0 R /Resources << /Font << /F1 3 0 R >> >> /Contents 5 0 R >>") + pdf.add(b"<< >>", b"BT /F1 12 Tf (a endobj b) Tj 0 -20 Td (AFTER) Tj ET", compress=False) + self.assertEqual(pt.extract_text(pdf.bytes()), "a endobj b\nAFTER") + + +class ReviewRound2(unittest.TestCase): + """Second review: lexing is what repeats, so lexing is what is charged; + one bad value costs one stream, not the document.""" + + def _pages_sharing(self, body, npages): + pdf = Pdf() + pdf.add(b"<< /Type /Catalog /Pages 2 0 R >>") + kids = b" ".join(b"%d 0 R" % (5 + i) for i in range(npages)) + pdf.add(b"<< /Type /Pages /Count %d /Kids [%s] >>" % (npages, kids)) + pdf.add(HELVETICA) + pdf.add(b"<< >>", body) # 4 + for _ in range(npages): + pdf.add(b"<< /Type /Page /Parent 2 0 R /Resources << /Font << /F1 3 0 R >> >> " + b"/Contents 4 0 R >>") + return pdf.bytes() + + def test_one_stream_lexed_by_many_pages_is_bounded(self): + doc = self._pages_sharing(b"BT /F1 12 Tf (hello) Tj ET " + b"1 " * 1_000_000, 40) + b = pt.Budget(seconds=1) + t0 = time.monotonic() + out = pt.extract_text(doc, b) + self.assertLess(time.monotonic() - t0, 5) + self.assertEqual(b.reason, "time") + self.assertTrue(out.startswith("hello")) + + def test_token_cap(self): + doc = self._pages_sharing(b"BT /F1 12 Tf (hi) Tj ET " + b"1 " * 50_000, 10) + with mock.patch.object(pt, "MAX_TOKENS", 120_000): + b = pt.Budget() + pt.extract_text(doc, b) + self.assertEqual(b.reason, "tokens") + + def test_a_shared_tounicode_is_parsed_once(self): + pdf = Pdf() + pdf.add(b"<< /Type /Catalog /Pages 2 0 R >>") + pdf.add(b"<< /Type /Pages /Count 1 /Kids [6 0 R] >>") + tu = pdf.add(b"<< >>", CMAP) # 3 + pdf.add(b"<< /Type /Font /Subtype /Type0 /ToUnicode %d 0 R >>" % tu) # 4 + pdf.add(b"<< /Type /Font /Subtype /Type0 /ToUnicode %d 0 R >>" % tu) # 5 + pdf.add(b"<< /Type /Page /Parent 2 0 R /Resources << /Font << /F1 4 0 R /F2 5 0 R >> >> " + b"/Contents 7 0 R >>") + pdf.add(b"<< >>", b"BT /F1 12 Tf " + _enc("AB") + b" Tj /F2 12 Tf " + _enc("CD") + b" Tj ET") + with mock.patch.object(pt._Font, "_parse", autospec=True, side_effect=pt._Font._parse) as parse: + self.assertEqual(pt.extract_text(pdf.bytes()), "ABCD") + self.assertEqual(parse.call_count, 1) + + def test_words_far_apart_without_widths_are_separated(self): + font = b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>" + for ops in (b"BT /F1 10 Tf 50 700 Td (Name) Tj 150 0 Td (Value) Tj ET", + b"BT /F1 10 Tf 1 0 0 1 50 700 Tm (Name) Tj 1 0 0 1 200 700 Tm (Value) Tj ET"): + self.assertEqual(pt.extract_text(simple_doc([ops], font=font)), "Name Value") + + def test_standard_font_without_widths_uses_its_real_widths(self): + # Helvetica W is 944/1000: (WWWW) ends at 37.76 at size 10. The rest of + # the word placed exactly there is the same word; a guessed 500 would + # have read a 17.76-unit gap and split it. + font = b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>" + out = pt.extract_text(simple_doc([b"BT /F1 10 Tf 0 0 Td (WWWW) Tj 37.76 0 Td (ORD) Tj ET"], + font=font)) + self.assertEqual(out, "WWWWORD") + self.assertEqual(pt._standard_widths(b"ABCDEF+Arial,Bold")[1][ord("W") - 32], 944) + self.assertEqual(pt._standard_widths(b"Courier-Oblique"), (32, [600] * 95)) + self.assertIsNone(pt._standard_widths(b"SomeFace")) + + def test_unknown_font_without_widths_guesses_wide(self): + font = b"<< /Type /Font /Subtype /Type1 /BaseFont /SomeFace >>" + out = pt.extract_text(simple_doc([b"BT /F1 10 Tf 0 0 Td (WWWW) Tj 37.76 0 Td (ORD) Tj ET"], + font=font)) + self.assertEqual(out, "WWWWORD") + + def test_superscript_rise_is_not_a_new_line(self): + out = pt.extract_text(simple_doc([b"BT /F1 10 Tf 0 0 Td (E=mc) Tj 20 3 Td (2) Tj " + b"5 -3 Td ( is) Tj ET"])) + self.assertEqual(out, "E=mc2 is") + + def test_the_word_stream_inside_a_string_owns_no_bytes(self): + # The stray object sits BEFORE every page object. + doc = simple_doc([b"BT /F1 12 Tf (BODY) Tj ET"]) + head, rest = doc.split(b"1 0 obj", 1) + doc = head + b"9 0 obj\n<< /Title (a live stream\n) >>\nendobj\n1 0 obj" + rest + self.assertEqual(pt.extract_text(doc), "BODY") + + def test_indirect_length(self): + body = b"BT /F1 12 Tf (INDIRECT) Tj ET" + raw = (b"%PDF-1.4\n1 0 obj << /Type /Catalog /Pages 2 0 R >> endobj\n" + b"2 0 obj << /Type /Pages /Count 1 /Kids [3 0 R] >> endobj\n" + b"3 0 obj << /Type /Page /Parent 2 0 R /Contents 4 0 R >> endobj\n" + b"4 0 obj << /Length 5 0 R >>\nstream\n" + body + b"\nendstream\nendobj\n" + b"5 0 obj %d endobj\n%%%%EOF" % len(body)) + self.assertEqual(pt.extract_text(raw), "INDIRECT") + + def test_one_bad_value_costs_one_stream_not_the_document(self): + bad_font = b"<< /Type /Font /Subtype /Type1 /Encoding [/a /b] >>" + self.assertEqual(pt.extract_text(simple_doc([b"BT /F1 12 Tf (still) Tj ET"], font=bad_font)), + "still") + huge = b"BT /F1 12 Tf (hello world) Tj [(a) 1" + b"0" * 400 + b" (b)] TJ ET" + self.assertTrue(pt.extract_text(simple_doc([huge])).startswith("hello world")) + pdf = Pdf() + pdf.add(b"<< /Type /Catalog /Pages 2 0 R >>") + pdf.add(b"<< /Type /Pages /Count 2 /Kids [4 0 R 5 0 R] >>") + pdf.add(HELVETICA) + pdf.add(b"<< /Type /Page /Parent 2 0 R /Resources << /Font << /F1 3 0 R >> >> /Contents 6 0 R >>") + pdf.add(b"<< /Type /Page /Parent 2 0 R /Resources << /Font << /F1 3 0 R >> >> /Contents 7 0 R >>") + pdf.add(b"<< >>", b"BT /F1 12 Tf (PAGE ONE) Tj ET") + pdf.add(b"<< >>", b"BT /F1 12 Tf (PAGE TWO) Tj ET") + real = pt._Reader.content + calls = [] + + def first_page_breaks(self, data, *a, **k): + calls.append(1) + if len(calls) == 1: + raise TypeError("a malformed value") + return real(self, data, *a, **k) + with mock.patch.object(pt._Reader, "content", first_page_breaks): + self.assertEqual(pt.extract_text(pdf.bytes()).split("\f"), ["", "PAGE TWO"]) + + def test_cid_pages_without_a_map_do_not_fall_back_to_junk(self): + pdf = Pdf() + pdf.add(b"<< /Type /Catalog /Pages 2 0 R >>") + pdf.add(b"<< /Type /Pages /Count 1 /Kids [4 0 R] >>") + pdf.add(b"<< /Type /Font /Subtype /Type0 /Encoding /Identity-H >>") + pdf.add(b"<< /Type /Page /Parent 2 0 R /Resources << /Font << /F1 3 0 R >> >> /Contents 5 0 R >>") + pdf.add(b"<< >>", b"BT /F1 12 Tf <00410042> Tj ET") + self.assertIsNone(pt.extract_text(pdf.bytes())) + + def test_index_checks_the_clock(self): + data = b"%PDF-1.4\n" + b"".join(b"%d 0 obj [1 2 3] endobj\n" % i for i in range(200_000)) + b = pt.Budget(seconds=0) + pt.extract_text(data, b) + self.assertEqual(b.reason, "time") + + + +class ReviewRound4(unittest.TestCase): + """Memory, not just time: a lexer step must not allocate per byte.""" + + def _peak(self, fn): + import tracemalloc + tracemalloc.start() + fn() + peak = tracemalloc.get_traced_memory()[1] + tracemalloc.stop() + return peak + + def test_spaced_hex_string_is_stripped_without_per_byte_objects(self): + data = b"<" + b"0 " * 2_000_000 + b"41>" + peak = self._peak(lambda: pt._Lexer(data).token()) + self.assertEqual(pt._Lexer(b"<4 1 4 2>").token(), ("str", b"AB")) + self.assertLess(peak, 3 * len(data), peak) + + def test_comment_lines_are_skipped_without_per_line_state(self): + data = b"%\n" * 1_000_000 + b"(x)" + peak = self._peak(lambda: pt._Lexer(data).token()) + self.assertEqual(pt._Lexer(data).token(), ("str", b"x")) + self.assertLess(peak, len(data), peak) + + def test_a_long_escape_run_checks_the_clock(self): + b = pt.Budget(seconds=0) + with self.assertRaises(pt._Spent): + pt._Lexer(b"(" + b"\\a" * 300_000 + b")", budget=b).token() + + +if __name__ == "__main__": + unittest.main() diff --git a/bastion_agent/test_run.py b/bastion_agent/test_run.py new file mode 100644 index 0000000..1333fa2 --- /dev/null +++ b/bastion_agent/test_run.py @@ -0,0 +1,288 @@ +"""Unit tests for the sweep scheduler and the manifest gate. Run: +python3 -m unittest discover . + +The gate tests generate a real EC key + self-signed cert with openssl (on +PATH here and in CI); the loop is driven with a fake clock and sleep so +a 'daily' cadence runs in milliseconds. +""" + +import json +import os +import shutil +import subprocess +import tempfile +import unittest +from unittest import mock + +import yaml + +from . import run as rn +from .manifest import load_manifest + +HAVE_OPENSSL = bool(shutil.which("openssl")) + + +def write_manifest(path, root, stages, cn="owner-host", sweep="manual", extra=None): + doc = { + "freeagent": 1, "id": "t", "owner": {"host": cn}, + "scope": {"roots": [root]}, "mask": ["read", "index"], + "stages": stages, "sweep": sweep, + "sink": {"catalog": "local", "vectors": "local", "plaintext": "never"}, + } + if "sanitize" in stages: + doc["sanitize"] = {"mode": "flag"} + if extra: + doc.update(extra) + with open(path, "w", encoding="utf-8") as fh: + yaml.safe_dump(doc, fh, sort_keys=False) + return path + + +def make_identity(dirpath, cn): + key = os.path.join(dirpath, "host.key") + cert = os.path.join(dirpath, "host.crt") + subprocess.run(["openssl", "ecparam", "-name", "prime256v1", "-genkey", "-noout", "-out", key], + check=True, capture_output=True) + subprocess.run(["openssl", "req", "-new", "-x509", "-key", key, "-out", cert, "-days", "2", + "-subj", "/CN=%s" % cn], check=True, capture_output=True) + return key, cert + + +class Case(unittest.TestCase): + def setUp(self): + self.root = tempfile.mkdtemp(prefix="fa-run-") + self.work = tempfile.mkdtemp(prefix="fa-runwork-") + self.out = os.path.join(self.work, "out") + with open(os.path.join(self.root, "a.txt"), "w", encoding="utf-8") as fh: + fh.write("hello m@example.com\n") + self.manifest = os.path.join(self.work, "m.yaml") + + def tearDown(self): + shutil.rmtree(self.root, ignore_errors=True) + shutil.rmtree(self.work, ignore_errors=True) + + def runs(self): + with open(os.path.join(self.out, "run.jsonl"), encoding="utf-8") as fh: + return [json.loads(l) for l in fh] + + +@unittest.skipUnless(HAVE_OPENSSL, "openssl not on PATH") +class GateTests(Case): + def test_sign_then_verify(self): + write_manifest(self.manifest, self.root, ["walk"]) + key, cert = make_identity(self.work, "owner-host") + rn.sign_manifest_file(self.manifest, key, "host.crt") + m = load_manifest(self.manifest, require_signed=True) + gate = rn.verify_signature(m, {"FREEAGENT_CA_ROOT": cert}, manifest_dir=self.work) + self.assertEqual(gate["cn"], "owner-host") + self.assertTrue(gate["chained"]) + # no root, no gate: "signed by the key next to the manifest" proves nothing + with self.assertRaises(rn.GateError) as cm: + rn.verify_signature(m, {}, manifest_dir=self.work) + self.assertIn("FREEAGENT_CA_ROOT", str(cm.exception)) + # re-signing replaces the block rather than stacking a second one + rn.sign_manifest_file(self.manifest, key, "host.crt") + with open(self.manifest, encoding="utf-8") as fh: + self.assertEqual(fh.read().count("signature:"), 1) + + def test_tampered_payload_fails(self): + write_manifest(self.manifest, self.root, ["walk"]) + key, cert = make_identity(self.work, "owner-host") + rn.sign_manifest_file(self.manifest, key, "host.crt") + with open(self.manifest, encoding="utf-8") as fh: + text = fh.read() + with open(self.manifest, "w", encoding="utf-8") as fh: + fh.write(text.replace("sweep: manual", "sweep: daily")) + m = load_manifest(self.manifest, require_signed=True) + with self.assertRaises(rn.GateError) as cm: + rn.verify_signature(m, {"FREEAGENT_CA_ROOT": cert}, manifest_dir=self.work) + self.assertIn("does not verify", str(cm.exception)) + + def test_wrong_cn_fails(self): + write_manifest(self.manifest, self.root, ["walk"], cn="owner-host") + key, cert = make_identity(self.work, "someone-else") + rn.sign_manifest_file(self.manifest, key, "host.crt") + m = load_manifest(self.manifest, require_signed=True) + with self.assertRaises(rn.GateError) as cm: + rn.verify_signature(m, {"FREEAGENT_CA_ROOT": cert}, manifest_dir=self.work) + self.assertIn("not the manifest owner", str(cm.exception)) + + def test_multi_cn_cert_refused(self): + # a cert carrying the owner's CN next to its real one must not pass + write_manifest(self.manifest, self.root, ["walk"], cn="owner-host") + key = os.path.join(self.work, "host.key") + cert = os.path.join(self.work, "host.crt") + subprocess.run(["openssl", "ecparam", "-name", "prime256v1", "-genkey", "-noout", "-out", key], + check=True, capture_output=True) + subprocess.run(["openssl", "req", "-new", "-x509", "-key", key, "-out", cert, "-days", "2", + "-subj", "/CN=attacker/CN=owner-host"], check=True, capture_output=True) + rn.sign_manifest_file(self.manifest, key, "host.crt") + m = load_manifest(self.manifest, require_signed=True) + with self.assertRaises(rn.GateError) as cm: + rn.verify_signature(m, {"FREEAGENT_CA_ROOT": cert}, manifest_dir=self.work) + self.assertIn("exactly one CN", str(cm.exception)) + + def test_ca_chain(self): + write_manifest(self.manifest, self.root, ["walk"]) + key, cert = make_identity(self.work, "owner-host") + rn.sign_manifest_file(self.manifest, key, "host.crt") + m = load_manifest(self.manifest, require_signed=True) + # self-signed: chains to itself, not to a stranger's root + gate = rn.verify_signature(m, {"FREEAGENT_CA_ROOT": cert}, manifest_dir=self.work) + self.assertTrue(gate["chained"]) + _, other_root = make_identity(tempfile.mkdtemp(prefix="fa-ca-"), "other-ca") + with self.assertRaises(rn.GateError) as cm: + rn.verify_signature(m, {"FREEAGENT_CA_ROOT": other_root}, manifest_dir=self.work) + self.assertIn("chain", str(cm.exception)) + + def test_unsigned_refused_unless_allowed(self): + write_manifest(self.manifest, self.root, ["walk"]) + rec, _ = rn.run_once(self.manifest, self.out, env={}) + self.assertFalse(rec["ok"]) + self.assertIn("unsigned", rec["error"]) + self.assertEqual(rec["stages"], []) + # the refusal is on the append-only record + self.assertEqual(len(self.runs()), 1) + self.assertIn("gate:", self.runs()[0]["error"]) + rec, _ = rn.run_once(self.manifest, self.out, env={}, allow_unsigned=True) + self.assertTrue(rec["ok"]) + self.assertTrue(rec["allow_unsigned"]) + self.assertIn("bypassed", rec["gate"]) + + +class RunTests(Case): + def test_stages_in_order_and_logged(self): + write_manifest(self.manifest, self.root, ["walk", "extract", "sanitize", "catalog"]) + rec, m = rn.run_once(self.manifest, self.out, env={}, allow_unsigned=True, now="2026-08-28T00:00:00Z") + self.assertTrue(rec["ok"], rec) + self.assertEqual([s["stage"] for s in rec["stages"]], ["walk", "extract", "sanitize", "catalog"]) + self.assertTrue(all(s["status"] == "ok" for s in rec["stages"])) + self.assertEqual(rec["stages"][0]["files"], 1) + self.assertEqual(rec["stages"][2]["ok"], 1) + self.assertTrue(os.path.exists(os.path.join(self.out, "catalog.jsonl"))) + self.assertEqual(len(self.runs()), 1) + self.assertEqual(self.runs()[0]["started"], "2026-08-28T00:00:00Z") + + def test_default_out_is_the_folder_serve_uses(self): + # without --out, sweep must write where serve reads: /.agent + write_manifest(self.manifest, self.root, ["walk"]) + rec, m = rn.run_once(self.manifest, None, env={}, allow_unsigned=True) + self.assertTrue(rec["ok"], rec) + want = os.path.join(os.path.abspath(self.root), ".agent") + self.assertEqual(rec["out"], want) + from .__main__ import default_out # what serve and every stage command use + self.assertEqual(default_out(m), want) + self.assertTrue(os.path.exists(os.path.join(want, "run.jsonl"))) + + def test_failure_stops_the_run(self): + # sterilize with an unstartable recall backend stops at sanitize + write_manifest(self.manifest, self.root, ["walk", "extract", "sanitize", "catalog"], + extra={"sanitize": {"mode": "sterilize"}}) + rec, _ = rn.run_once(self.manifest, self.out, env={"PII_RECALL": "nope"}, allow_unsigned=True) + self.assertFalse(rec["ok"]) + self.assertEqual([s["stage"] for s in rec["stages"]], ["walk", "extract", "sanitize"]) + self.assertEqual(rec["stages"][-1]["status"], "failed") + self.assertIn("PII_RECALL", rec["error"]) + self.assertFalse(os.path.exists(os.path.join(self.out, "catalog.jsonl"))) + + def test_invalid_manifest_is_a_recorded_error(self): + with open(self.manifest, "w") as fh: + fh.write("freeagent: 2\n") + rec, m = rn.run_once(self.manifest, self.out, env={}, allow_unsigned=True) + self.assertIsNone(m) + self.assertIn("manifest:", rec["error"]) + self.assertIn("manifest:", self.runs()[0]["error"]) # refused manifests are logged too + + def test_lock_blocks_and_reclaims(self): + write_manifest(self.manifest, self.root, ["walk"]) + os.makedirs(self.out, exist_ok=True) + with rn.Lock(self.out): + with self.assertRaises(rn.RunError): + with rn.Lock(self.out): + pass + rec, _ = rn.run_once(self.manifest, self.out, env={}, allow_unsigned=True) + self.assertIn("another sweep holds", rec["error"]) + # a stale lock (dead pid) is reclaimed + with open(os.path.join(self.out, "sweep.lock"), "w") as fh: + json.dump({"pid": 2 ** 22 + 12345, "since": 0}, fh) + rec, _ = rn.run_once(self.manifest, self.out, env={}, allow_unsigned=True) + self.assertTrue(rec["ok"]) + self.assertIn("lock_reclaimed", rec) + self.assertFalse(os.path.exists(os.path.join(self.out, "sweep.lock"))) + # a LIVE holder keeps its lock however old the file is (never stolen) + with open(os.path.join(self.out, "sweep.lock"), "w") as fh: + json.dump({"pid": os.getpid(), "since": 0}, fh) + rec, _ = rn.run_once(self.manifest, self.out, env={}, allow_unsigned=True) + self.assertIn("another sweep holds", rec["error"]) + os.unlink(os.path.join(self.out, "sweep.lock")) + # a truncated lock is judged by file age, not instantly stale + with open(os.path.join(self.out, "sweep.lock"), "w") as fh: + fh.write("{") + rec, _ = rn.run_once(self.manifest, self.out, env={}, allow_unsigned=True) + self.assertIn("another sweep holds", rec["error"]) + os.unlink(os.path.join(self.out, "sweep.lock")) + + def test_next_due(self): + self.assertEqual(rn.next_due("hourly", 1000, True, 0), 4600) + self.assertEqual(rn.next_due("15m", 1000, True, 0), 1900) + self.assertIsNone(rn.next_due("manual", 1000, True, 0)) + self.assertEqual(rn.next_due("daily", 1000, False, 1), 1060) # 1 min + self.assertEqual(rn.next_due("daily", 1000, False, 3), 1240) # 4 min + self.assertEqual(rn.next_due("15m", 1000, False, 20), 1900) # capped at cadence + + +class SweepLoopTests(Case): + def test_manual_runs_once(self): + write_manifest(self.manifest, self.root, ["walk"], sweep="manual") + sleeps = [] + last = rn.sweep(self.manifest, self.out, env={}, allow_unsigned=True, + sleep=sleeps.append, clock=lambda: 1000.0, log=lambda s: None) + self.assertTrue(last["ok"]) + self.assertEqual(sleeps, []) + state = rn.load_state(self.out) + self.assertIsNone(state["next_due"]) + self.assertEqual(state["consecutive_failures"], 0) + + def test_cadence_loops_with_fake_clock(self): + write_manifest(self.manifest, self.root, ["walk"], sweep="15m") + clock = {"t": 1000.0} + sleeps = [] + + def sleep(s): + sleeps.append(s) + clock["t"] += s + + logs = [] + last = rn.sweep(self.manifest, self.out, env={}, allow_unsigned=True, sleep=sleep, + clock=lambda: clock["t"], max_runs=3, log=logs.append) + self.assertTrue(last["ok"]) + self.assertEqual(len(self.runs()), 3) + self.assertEqual(sum(sleeps), 2 * 15 * 60) # slept out two full cadences + self.assertTrue(all(s <= 30 for s in sleeps)) # in short naps, so a stop lands quickly + state = rn.load_state(self.out) + self.assertEqual(state["runs_this_process"], 3) + self.assertEqual(state["next_due"], "1970-01-01T01:01:40Z") # 1000 + 2*900 + 900 + self.assertEqual(len(logs), 3) + self.assertIn("ok", logs[0]) + + def test_failure_backoff_and_counter(self): + write_manifest(self.manifest, self.root, ["walk", "extract", "sanitize"], sweep="hourly", + extra={"sanitize": {"mode": "sterilize"}}) + clock = {"t": 1000.0} + sleeps = [] + + def sleep(s): + sleeps.append(s) + clock["t"] += s + + rn.sweep(self.manifest, self.out, env={"PII_RECALL": "nope"}, allow_unsigned=True, + sleep=sleep, clock=lambda: clock["t"], max_runs=3, log=lambda s: None) + state = rn.load_state(self.out) + self.assertEqual(state["consecutive_failures"], 3) + self.assertFalse(state["last_ok"]) + # retries at 60 s then 120 s, not the hourly cadence + self.assertEqual(sum(sleeps), 60 + 120) + + +if __name__ == "__main__": + unittest.main() diff --git a/bastion_agent/test_sanitize.py b/bastion_agent/test_sanitize.py new file mode 100644 index 0000000..90bcb26 --- /dev/null +++ b/bastion_agent/test_sanitize.py @@ -0,0 +1,383 @@ +"""Unit tests for the sanitize stage. Run: python3 -m unittest discover . + +The real pii pipeline and vault run in-process; the recall backend +is a fake (no model), the service path is a stub HTTP server speaking the +real /sterilize wire format. Nothing here needs a model or the network. +""" + +import json +import os +import shutil +import stat +import tempfile +import threading +import unittest +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from unittest import mock + +from . import sanitize as sz +from .extract import extract +from .manifest import validate_document +from .walk import walk + +IS_WINDOWS = os.name == "nt" + + +def manifest(root, mode="flag", verify=False, scope=None): + san = {"mode": mode, "verify": verify} + if scope: + san["scope"] = scope + return validate_document({ + "freeagent": 1, "id": "t", "owner": {"host": "h"}, + "scope": {"roots": [root]}, + "mask": ["read"], "stages": ["walk", "extract", "sanitize"], "sweep": "manual", + "sanitize": san, + "sink": {"catalog": "local", "vectors": "local", "plaintext": "never"}, + }) + + +def put(root, rel, text): + path = os.path.join(root, *rel.split("/")) + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8") as fh: + fh.write(text) + + +class FakeBackend: + name = "fake" + calls = 0 + + def __init__(self, findings=(), fail=False): + self.findings = list(findings) + self.fail = fail + + def describe(self): + return "fake" + + def extract(self, text): + FakeBackend.calls += 1 + if self.fail: + _, recall, _ = sz.pii_modules() + raise recall.RecallError("model unreachable") + return list(self.findings) # proposes everything; grounding must drop the ghosts + + def verify(self, text): + return [] + + +class Case(unittest.TestCase): + def setUp(self): + self.root = tempfile.mkdtemp(prefix="fa-sz-") + self.out = tempfile.mkdtemp(prefix="fa-szout-") + put(self.root, "note.txt", "Marisol's email is m@example.com, card 4111 1111 1111 1111.\n") + put(self.root, "twin.txt", "Marisol's email is m@example.com, card 4111 1111 1111 1111.\n") + put(self.root, "clean.txt", "nothing personal here\n") + put(self.root, "blob.bin", "\x00\x01") + FakeBackend.calls = 0 + + def tearDown(self): + shutil.rmtree(self.root, ignore_errors=True) + shutil.rmtree(self.out, ignore_errors=True) + + def prep(self, m): + walk(m, self.out) + extract(m, self.out, use_external=False) + + def run_stage(self, m, env=None, backend=None): + self.prep(m) + if backend is not None: + with mock.patch.object(sz, "BUILD_BACKEND", lambda e: backend): + s = sz.sanitize(m, self.out, env=env or {}) + else: + s = sz.sanitize(m, self.out, env=env or {}) + with open(os.path.join(self.out, "sanitize.jsonl"), encoding="utf-8") as fh: + idx = {json.loads(l)["path"]: json.loads(l) for l in fh} + return s, idx + + def text_of(self, rec): + with open(sz.store_path(self.out, rec["sha256"]), encoding="utf-8") as fh: + return fh.read() + + +class FlagModeTests(Case): + def test_rules_only_redaction(self): + s, idx = self.run_stage(manifest(self.root, "flag")) + out = self.text_of(idx["note.txt"]) + self.assertNotIn("m@example.com", out) + self.assertNotIn("4111", out) + self.assertIn("Marisol", out) # contextual PII is NOT caught by flag mode + self.assertIn("[PII_EMAIL_1]", out) + self.assertIn("[PII_CREDIT_CARD_1]", out) + self.assertEqual(idx["note.txt"]["types"], {"EMAIL": 1, "CREDIT_CARD": 1}) + self.assertEqual(s["sterilizer"], "flag:rules-only") + self.assertEqual(s["types"], {"EMAIL": 1, "CREDIT_CARD": 1}) + # twin shares content -> deduped, counted once as ok + self.assertEqual(idx["twin.txt"]["sha256"], idx["note.txt"]["sha256"]) + self.assertEqual(s["counts"]["ok"], 3) + self.assertEqual(idx["blob.bin"]["status"], "skipped") + self.assertEqual(s["counts"]["skipped"], 1) + + def test_index_and_summary_carry_no_values(self): + s, idx = self.run_stage(manifest(self.root, "flag")) + blob = json.dumps(idx) + json.dumps(s) + self.assertNotIn("m@example.com", blob) + self.assertNotIn("4111", blob) + + @unittest.skipIf(IS_WINDOWS, "POSIX permission bits") + def test_vault_is_private(self): + self.run_stage(manifest(self.root, "flag")) + mode = stat.S_IMODE(os.stat(os.path.join(self.out, "sanitize", "vault.db")).st_mode) + self.assertEqual(mode, 0o600) + + def test_reuse_and_invalidate(self): + s1, _ = self.run_stage(manifest(self.root, "flag")) + s2, idx = self.run_stage(manifest(self.root, "flag")) + self.assertEqual(s2["counts"]["reused"], s1["counts"]["ok"]) + self.assertTrue(idx["note.txt"]["reused"]) + # a different scope is a different cache key -> redone + s3, _ = self.run_stage(manifest(self.root, "flag", scope="other")) + self.assertEqual(s3["counts"]["reused"], 0) + # a changed file is redone + put(self.root, "clean.txt", "now with a@b.co\n") + s4, idx4 = self.run_stage(manifest(self.root, "flag")) + self.assertIn("[PII_EMAIL_", self.text_of(idx4["clean.txt"])) + + def test_result_from_an_older_pipeline_version_is_not_reused(self): + # the version is the only reuse-key part that changes with the + # pipeline: output written by older code must be redone on upgrade + with mock.patch.object(sz, "SANITIZE_VERSION", 1): + s1, _ = self.run_stage(manifest(self.root, "flag")) + self.assertGreaterEqual(sz.SANITIZE_VERSION, 2) + s2, idx = self.run_stage(manifest(self.root, "flag")) + self.assertEqual(s2["counts"]["reused"], 0) + self.assertEqual(s2["counts"]["ok"], s1["counts"]["ok"]) + self.assertFalse(idx["note.txt"].get("reused", False)) + + def test_tokens_stable_across_files_in_scope(self): + put(self.root, "again.txt", "ping m@example.com again\n") + _, idx = self.run_stage(manifest(self.root, "flag")) + self.assertIn("[PII_EMAIL_1]", self.text_of(idx["again.txt"])) + self.assertIn("[PII_EMAIL_1]", self.text_of(idx["note.txt"])) + + def test_vault_dir_env_keeps_vault_out_of_out(self): + vdir = tempfile.mkdtemp(prefix="fa-vault-") + try: + s, idx = self.run_stage(manifest(self.root, "flag"), env={"FREEAGENT_VAULT_DIR": vdir}) + self.assertTrue(os.path.exists(os.path.join(vdir, "vault.db"))) + self.assertFalse(os.path.exists(os.path.join(self.out, "sanitize", "vault.db"))) + self.assertIn("[PII_EMAIL_1]", self.text_of(idx["note.txt"])) + finally: + shutil.rmtree(vdir, ignore_errors=True) + + def test_requires_extract_first(self): + with self.assertRaises(FileNotFoundError): + sz.sanitize(manifest(self.root, "flag"), self.out, env={}) + + +class SterilizeModeTests(Case): + def test_recall_backend_findings_are_tokenized(self): + fake = FakeBackend([("Marisol", "PERSON"), ("Ghost", "PERSON")]) + s, idx = self.run_stage(manifest(self.root, "sterilize"), backend=fake) + out = self.text_of(idx["note.txt"]) + self.assertNotIn("Marisol", out) + self.assertIn("[PII_PERSON_1]", out) + self.assertEqual(idx["note.txt"]["types"]["PERSON"], 1) + self.assertEqual(idx["note.txt"]["stats"]["llm_ungrounded"], 1) + self.assertEqual(s["sterilizer"], "sterilize:fake") + self.assertEqual(FakeBackend.calls, 2) # note (twin deduped) + clean + + def test_fail_closed_stops_the_stage(self): + s, idx = self.run_stage(manifest(self.root, "sterilize"), backend=FakeBackend(fail=True)) + self.assertEqual(s["counts"]["ok"], 0) + self.assertEqual(s["counts"]["not_run"], 3) + self.assertIn("model unreachable", s["stopped"]) + self.assertEqual(idx["note.txt"]["status"], "not_run") + self.assertFalse(os.path.exists(sz.store_path(self.out, idx["note.txt"]["sha256"]))) + self.assertEqual(FakeBackend.calls, 1) # stopped after the first failure + + def test_backend_that_cannot_start_raises(self): + self.prep(manifest(self.root, "sterilize")) + with self.assertRaises(sz.SanitizeError): + sz.sanitize(manifest(self.root, "sterilize"), self.out, env={"PII_RECALL": "nope"}) + + def test_chunking_round_trip(self): + big = ("para %d with m@example.com\n\n" % i for i in range(2000)) + text = "".join(big) + put(self.root, "big.txt", text) + chunks = sz.split_chunks(text, limit=5000) + self.assertGreater(len(chunks), 5) + self.assertEqual("".join(chunks), text) + self.assertTrue(all(len(c) <= 5000 for c in chunks)) + sizes = [] + real = sz.InProcess.sterilize + + def spy(front, chunk, scope): + if chunk.startswith("para "): # big.txt's chunks, not the other files + sizes.append(len(chunk)) + return real(front, chunk, scope) + + with mock.patch.object(sz, "SANITIZE_CHUNK_CHARS", 5000), \ + mock.patch.object(sz.InProcess, "sterilize", spy): + _, idx = self.run_stage(manifest(self.root, "flag")) + # the stage itself must have split at the patched limit, not the default + self.assertLessEqual(max(sizes), 5000) + self.assertEqual(len(sizes), len(chunks)) + out = self.text_of(idx["big.txt"]) + self.assertNotIn("m@example.com", out) + self.assertEqual(out.count("[PII_EMAIL_1]"), 2000) # one token, every chunk + + def test_hard_cut_never_splits_a_placeholder(self): + _, _, Vault = sz.pii_modules() + from pii import pipeline + self.assertEqual(sz._PLACEHOLDER_RE.pattern, pipeline.TOKEN_RE.pattern) + vault = Vault(os.path.join(self.out, "v.db")) + vault.token_for("s", "EMAIL", "ana@x.org") # [PII_EMAIL_1] is live + # the corpus stage escapes every placeholder in a document, but only + # one that reaches the sterilizer whole: split in half, neither half + # is recognised and the literal is stored raw, to rehydrate later as + # the vault's value + text = "x" * 15 + "[PII_EMAIL_1]" + "y" * 10 # one line, no newline + chunks = sz.split_chunks(text, limit=20) + self.assertEqual("".join(chunks), text) + self.assertTrue(any("[PII_EMAIL_1]" in c for c in chunks), chunks) + out = "".join(pipeline.sterilize(c, "s", vault, lambda t: [], escape_held_literals=True)[0] + for c in chunks) + self.assertNotIn("[PII_EMAIL_1]", out) + self.assertEqual(pipeline.rehydrate(out, "s", vault)[0], text) + # a placeholder at the very start of an over-long piece stays whole + self.assertEqual(sz.split_chunks("[PII_PERSON_1]" + "z" * 30, limit=10)[0], "[PII_PERSON_1]") + + def test_hard_cut_clears_a_placeholder_of_any_length(self): + long_ph = "[PII_" + "OTHER_" * 20 + "123456789]" # well over 64 characters + for before in (30, 90, 150): + with self.subTest(before=before): + text = "x" * before + long_ph + "y" * 40 + chunks = sz.split_chunks(text, limit=100) + self.assertEqual("".join(chunks), text) + self.assertTrue(any(long_ph in c for c in chunks), [len(c) for c in chunks]) + + def test_corpus_document_quoting_a_live_placeholder_round_trips(self): + # one manifest-wide scope: a.txt mints [PII_EMAIL_1]; b.txt quotes that + # placeholder literally and must come back as written, not as a's email + for name in ("note.txt", "twin.txt", "clean.txt", "blob.bin"): + os.remove(os.path.join(self.root, name)) + put(self.root, "a.txt", "mail ana@x.org\n") + quote = "the export shows [PII_EMAIL_1] where the address was\n" + put(self.root, "b.txt", quote) + _, idx = self.run_stage(manifest(self.root, "flag")) + _, _, Vault = sz.pii_modules() + from pii import pipeline + vault = Vault(os.path.join(self.out, "sanitize", "vault.db")) + self.assertEqual(vault.value_for("t", "[PII_EMAIL_1]"), "ana@x.org") + stored = self.text_of(idx["b.txt"]) + self.assertNotIn("[PII_EMAIL_1]", stored) + self.assertEqual(pipeline.rehydrate(stored, "t", vault)[0], quote) + + def test_split_chunks_hard_cuts_a_giant_line(self): + text = "x" * 12000 + "\n" + "y" * 100 + chunks = sz.split_chunks(text, limit=5000) + self.assertEqual("".join(chunks), text) + self.assertTrue(all(len(c) <= 5000 for c in chunks)) + self.assertEqual(sz.split_chunks("", 10), []) + self.assertEqual(sz.split_chunks("short", 10), ["short"]) + + +class _StubPii(BaseHTTPRequestHandler): + seen = [] + status = 200 + echo = "honour" # "honour", "strip" (a service that predates the field) or "false" + + def do_POST(self): + n = int(self.headers.get("Content-Length", "0")) + req = json.loads(self.rfile.read(n)) + _StubPii.seen.append((self.path, self.headers.get("Authorization"), req)) + if _StubPii.status != 200: + self.send_response(_StubPii.status) + self.send_header("Content-Length", "0") + self.end_headers() + return + body = json.dumps({"text": req["text"].replace("Marisol", "[PII_PERSON_1]"), + "findings": [{"token": "[PII_PERSON_1]", "type": "PERSON"}], + "stats": dict({"rules": 0, "llm": 1, "recall": "llm"}, **( + {} if _StubPii.echo == "strip" else + {"escape_held_literals": _StubPii.echo == "honour" + and req.get("escape_held_literals") is True}))}).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *a): + pass + + +class ServiceModeTests(Case): + @classmethod + def setUpClass(cls): + cls.srv = ThreadingHTTPServer(("127.0.0.1", 0), _StubPii) + threading.Thread(target=cls.srv.serve_forever, daemon=True).start() + cls.addr = "127.0.0.1:%d" % cls.srv.server_address[1] + + @classmethod + def tearDownClass(cls): + cls.srv.shutdown() + cls.srv.server_close() + + def setUp(self): + super().setUp() + _StubPii.seen, _StubPii.status, _StubPii.echo = [], 200, "honour" + + def test_service_path(self): + env = {"PII_ADDR": self.addr, "PII_API_KEY": "k1"} + s, idx = self.run_stage(manifest(self.root, "sterilize", scope="conv9"), env=env) + self.assertEqual(s["sterilizer"], "service:http://" + self.addr) + self.assertNotIn("Marisol", self.text_of(idx["note.txt"])) + path, auth, req = _StubPii.seen[0] + self.assertEqual((path, auth, req["scope"], req["verify"]), ("/sterilize", "Bearer k1", "conv9", False)) + self.assertIs(req["escape_held_literals"], True) # documents: escape every literal + # the echo is a handshake, not a count: it never reaches the index + self.assertNotIn("escape_held_literals", idx["note.txt"]["stats"]) + with open(os.path.join(self.out, "sanitize.jsonl"), encoding="utf-8") as fh: + self.assertNotIn("escape_held_literals", fh.read()) + self.assertNotIn("escape_held_literals", json.dumps(s)) + self.assertEqual(idx["note.txt"]["stats"]["recall"], "llm") + # no local vault in service mode — the service owns it + self.assertFalse(os.path.exists(os.path.join(self.out, "sanitize", "vault.db"))) + + def test_service_502_fails_closed(self): + _StubPii.status = 502 + env = {"PII_ADDR": self.addr, "PII_API_KEY": "k1"} + s, idx = self.run_stage(manifest(self.root, "sterilize"), env=env) + self.assertEqual(s["counts"]["not_run"], 3) + self.assertIn("502", s["stopped"]) + + def test_service_that_does_not_echo_the_literal_rule_stops_the_stage(self): + # an older service ignores escape_held_literals and answers 200 under + # the narrower rule; the stage must not store what it cannot trust + env = {"PII_ADDR": self.addr, "PII_API_KEY": "k1"} + for echo in ("strip", "false"): + with self.subTest(echo): + _StubPii.echo = echo + shutil.rmtree(self.out, ignore_errors=True) + os.makedirs(self.out) + s, idx = self.run_stage(manifest(self.root, "sterilize"), env=env) + self.assertIn("did not honour escape_held_literals", s["stopped"]) + self.assertEqual(s["counts"]["ok"], 0) + self.assertEqual(s["counts"]["not_run"], 3) + self.assertFalse(os.path.exists(sz.store_path(self.out, idx["note.txt"]["sha256"]))) + + def test_service_without_key_refused(self): + self.prep(manifest(self.root, "sterilize")) + with self.assertRaises(sz.SanitizeError): + sz.sanitize(manifest(self.root, "sterilize"), self.out, env={"PII_ADDR": self.addr}) + + def test_flag_mode_never_uses_the_service(self): + env = {"PII_ADDR": self.addr, "PII_API_KEY": "k1"} + s, _ = self.run_stage(manifest(self.root, "flag"), env=env) + self.assertEqual(_StubPii.seen, []) + self.assertEqual(s["sterilizer"], "flag:rules-only") + + +if __name__ == "__main__": + unittest.main() diff --git a/bastion_agent/test_seal.py b/bastion_agent/test_seal.py new file mode 100644 index 0000000..b9ea00e --- /dev/null +++ b/bastion_agent/test_seal.py @@ -0,0 +1,197 @@ +"""Unit tests for the seal stage. Run: python3 -m unittest discover . + +Round trips use a real age identity generated with age-keygen when the +binaries are on PATH (they are on the server and in CI); the envelope +tests are skipped where age is absent, the fail-closed tests are not. +""" + +import io +import json +import os +import shutil +import subprocess +import tarfile +import tempfile +import unittest +from unittest import mock + +from . import seal as sl +from .catalog import catalog +from .extract import extract +from .manifest import validate_document +from .sanitize import sanitize +from .walk import walk + +HAVE_AGE = bool(shutil.which("age") and shutil.which("age-keygen")) + + +def manifest(root, seal_to=(), stages=None): + doc = { + "freeagent": 1, "id": "t", "owner": {"host": "h"}, + "scope": {"roots": [root]}, + "mask": ["read", "search", "index"], + "stages": stages or ["walk", "extract", "sanitize", "catalog", "seal"], "sweep": "manual", + "sanitize": {"mode": "flag"}, + "sink": {"catalog": "host", "vectors": "local", "plaintext": "never", "seal_to": list(seal_to)}, + } + return validate_document(doc) + + +def keygen(dirpath): + ident = os.path.join(dirpath, "key.txt") + out = subprocess.run(["age-keygen", "-o", ident], capture_output=True, text=True, check=True) + pub = [l for l in (out.stderr + out.stdout).splitlines() if "age1" in l][-1].split()[-1] + return ident, pub + + +class Case(unittest.TestCase): + def setUp(self): + self.root = tempfile.mkdtemp(prefix="fa-seal-") + self.out = tempfile.mkdtemp(prefix="fa-sealout-") + with open(os.path.join(self.root, "note.txt"), "w", encoding="utf-8") as fh: + fh.write("Marisol's email is m@example.com and card 4111 1111 1111 1111.\n") + m = manifest(self.root) + walk(m, self.out) + extract(m, self.out, use_external=False) + sanitize(m, self.out, env={}) + catalog(m, self.out, env={}) + + def tearDown(self): + shutil.rmtree(self.root, ignore_errors=True) + shutil.rmtree(self.out, ignore_errors=True) + + +class FailClosedTests(Case): + def test_no_owner_recipient(self): + # age is faked present: seal checks for it first, and this test is + # about the recipient refusal, whether or not age is installed + with mock.patch.object(sl, "tool_path", lambda n: "/usr/bin/" + n): + with self.assertRaises(sl.SealError) as cm: + sl.seal(manifest(self.root), self.out, env={}) + self.assertIn("FREEAGENT_OWNER_RECIPIENT", str(cm.exception)) + self.assertFalse(os.path.exists(os.path.join(self.out, "sealed"))) + + def test_bad_recipient_strings(self): + env = {"FREEAGENT_OWNER_RECIPIENT": "not-a-key"} + with mock.patch.object(sl, "tool_path", lambda n: "/usr/bin/" + n): + with self.assertRaises(sl.SealError) as cm: + sl.seal(manifest(self.root), self.out, env=env) + self.assertIn("not an age recipient", str(cm.exception)) + env = {"FREEAGENT_OWNER_RECIPIENT": "age1" + "q" * 58} + with self.assertRaises(sl.SealError) as cm: + sl.recipients_for(manifest(self.root, seal_to=["partner-key"]), env) + self.assertIn("partner-key", str(cm.exception)) + + def test_no_age_on_path(self): + env = {"FREEAGENT_OWNER_RECIPIENT": "age1" + "q" * 58} + with mock.patch.object(sl, "tool_path", lambda n: None): + with self.assertRaises(sl.SealError) as cm: + sl.seal(manifest(self.root), self.out, env=env) + self.assertIn("age", str(cm.exception)) + + def test_seal_not_in_stages(self): + # (a `catalog: host` manifest cannot even omit seal — the schema refuses it) + m = validate_document({ + "freeagent": 1, "id": "t", "owner": {"host": "h"}, "scope": {"roots": [self.root]}, + "mask": ["read", "index"], "stages": ["walk", "extract", "sanitize", "catalog"], + "sweep": "manual", "sanitize": {"mode": "flag"}, + "sink": {"catalog": "local", "vectors": "local", "plaintext": "never"}}) + with self.assertRaises(sl.SealError): + sl.seal(m, self.out, env={"FREEAGENT_OWNER_RECIPIENT": "age1" + "q" * 58}) + + def test_bundle_allowlist(self): + plain, inv = sl.build_bundle(manifest(self.root), self.out, "20260828T000000Z") + with tarfile.open(fileobj=io.BytesIO(plain)) as tar: + names = tar.getnames() + self.assertIn("catalog.jsonl", names) + self.assertIn("catalog.dirs.jsonl", names) + self.assertIn("manifest.json", names) + self.assertIn("seal.manifest.json", names) + self.assertIn("walk.summary.json", names) + self.assertIn("sanitize.summary.json", names) + for forbidden in ("walk.jsonl", "extract.jsonl", "sanitize.jsonl", "vectors.db", "vault.db"): + self.assertFalse(any(n.endswith(forbidden) for n in names), forbidden) + self.assertFalse(any(n.startswith(("extract/", "sanitize/")) for n in names)) + # nothing in the bundle is a value + self.assertNotIn(b"m@example.com", plain) + self.assertNotIn(b"4111", plain) + self.assertIn(b"[PII_EMAIL_1]", plain) # sanitized head is there, as designed + self.assertEqual(set(inv["members"]), set(names)) + + def test_bundle_reproducible(self): + a, _ = sl.build_bundle(manifest(self.root), self.out, "20260828T000000Z") + b, _ = sl.build_bundle(manifest(self.root), self.out, "20260828T000000Z") + self.assertEqual(a, b) + + +@unittest.skipUnless(HAVE_AGE, "age / age-keygen not on PATH") +class RoundTripTests(Case): + def test_seal_and_unseal(self): + ident, pub = keygen(self.out) + other_ident, other_pub = keygen(tempfile.mkdtemp(prefix="fa-k2-")) + env = {"FREEAGENT_OWNER_RECIPIENT": pub} + s = sl.seal(manifest(self.root, seal_to=[other_pub]), self.out, env=env, now="20260828T000000Z") + sealed = os.path.join(self.out, s["file"]) + self.assertTrue(os.path.exists(sealed)) + self.assertEqual(s["recipients"], [pub, other_pub]) + self.assertGreater(s["bytes_sealed"], 0) + self.assertIsNone(s["signature"]) + with open(sealed, "rb") as fh: + blob = fh.read() + self.assertTrue(blob.startswith(b"age-encryption.org/v1")) + self.assertNotIn(b"PII_EMAIL", blob) # sealed: the catalog text is not visible + with open(os.path.join(self.out, "sealed", "latest.json")) as fh: + self.assertEqual(json.load(fh)["file"], s["file"]) + # both recipients can open it; a stranger cannot + for who in (ident, other_ident): + dest = tempfile.mkdtemp(prefix="fa-un-") + inv = sl.unseal(sealed, who, dest) + self.assertEqual(inv["manifest_id"], "t") + with open(os.path.join(dest, "catalog.jsonl"), encoding="utf-8") as fh: + self.assertIn("[PII_EMAIL_1]", fh.read()) + shutil.rmtree(dest, ignore_errors=True) + stranger, _ = keygen(tempfile.mkdtemp(prefix="fa-k3-")) + with self.assertRaises(sl.SealError): + sl.unseal(sealed, stranger, tempfile.mkdtemp(prefix="fa-un-")) + + def test_owner_recipient_deduped_and_signature(self): + ident, pub = keygen(self.out) + key = os.path.join(self.out, "sign.pem") + subprocess.run(["openssl", "ecparam", "-name", "prime256v1", "-genkey", "-noout", "-out", key], + check=True, capture_output=True) + env = {"FREEAGENT_OWNER_RECIPIENT": pub, "SEAL_SIGN_KEY": key} + s = sl.seal(manifest(self.root, seal_to=[pub]), self.out, env=env) + self.assertEqual(s["recipients"], [pub]) + sig = os.path.join(self.out, "sealed", s["signature"]) + self.assertTrue(os.path.exists(sig)) + # the signature verifies against the public half + pubkey = os.path.join(self.out, "sign.pub") + subprocess.run(["openssl", "ec", "-in", key, "-pubout", "-out", pubkey], + check=True, capture_output=True) + v = subprocess.run(["openssl", "dgst", "-sha256", "-verify", pubkey, "-signature", sig, + os.path.join(self.out, s["file"])], capture_output=True, text=True) + self.assertEqual(v.returncode, 0, v.stderr) + + def test_tampered_member_refused(self): + ident, pub = keygen(self.out) + env = {"FREEAGENT_OWNER_RECIPIENT": pub} + s = sl.seal(manifest(self.root), self.out, env=env) + # rebuild a bundle with a modified catalog but the old inventory, seal it, and unseal + plain, _ = sl.build_bundle(manifest(self.root), self.out, "x") + buf = io.BytesIO() + with tarfile.open(fileobj=io.BytesIO(plain)) as src, tarfile.open(fileobj=buf, mode="w") as dst: + for m in src.getmembers(): + data = src.extractfile(m).read() + if m.name == "catalog.jsonl": + data = data + b"\n{\"forged\": true}\n" + m.size = len(data) + dst.addfile(m, io.BytesIO(data)) + bad = os.path.join(self.out, "bad.tar.age") + subprocess.run(["age", "-r", pub, "-o", bad], input=buf.getvalue(), check=True) + with self.assertRaises(sl.SealError) as cm: + sl.unseal(bad, ident, tempfile.mkdtemp(prefix="fa-un-")) + self.assertIn("does not match the inventory", str(cm.exception)) + + +if __name__ == "__main__": + unittest.main() diff --git a/bastion_agent/test_serve.py b/bastion_agent/test_serve.py new file mode 100644 index 0000000..c18d7be --- /dev/null +++ b/bastion_agent/test_serve.py @@ -0,0 +1,351 @@ +"""Tests for the query endpoint. Run: python3 -m unittest discover . + +A miniature PKI is minted with openssl per test class: a root CA, a +server cert, an owner client cert, a grantee client cert, an expired- +grant client cert, and a client cert from a DIFFERENT CA. The server +binds loopback here ONLY because client and server are the same test +process; production binds the mesh interface (docs/serve.md). +""" + +import http.client +import json +import os +import shutil +import ssl +import subprocess +import tempfile +import threading +import unittest + +from . import embed as em +from . import run as rn +from . import serve as sv +from .catalog import catalog +from .extract import extract +from .manifest import validate_document +from .sanitize import sanitize +from .test_embed import _Stub as EmbedStub # deterministic bag-of-words embedder +from .walk import walk + +HAVE_OPENSSL = bool(shutil.which("openssl")) +import datetime as _dt +TODAY = _dt.date(2026, 8, 28) + + +def sh(*argv): + subprocess.run(list(argv), check=True, capture_output=True) + + +def mint_pki(d): + """root CA + server + clients; returns dict of paths.""" + p = lambda n: os.path.join(d, n) + def root(name, cn): + sh("openssl", "ecparam", "-name", "prime256v1", "-genkey", "-noout", "-out", p(name + ".key")) + sh("openssl", "req", "-new", "-x509", "-key", p(name + ".key"), "-out", p(name + ".crt"), "-days", "2", + "-subj", "/CN=" + cn, "-addext", "basicConstraints=critical,CA:TRUE", + "-addext", "keyUsage=critical,keyCertSign,cRLSign") + + root("ca", "test-root") + root("other-ca", "other-root") + + def leaf(name, cn, ca="ca", san=None): + sh("openssl", "ecparam", "-name", "prime256v1", "-genkey", "-noout", "-out", p(name + ".key")) + sh("openssl", "req", "-new", "-key", p(name + ".key"), "-out", p(name + ".csr"), "-subj", "/CN=" + cn) + ext = p(name + ".ext") + with open(ext, "w") as fh: + fh.write("subjectAltName=%s\nbasicConstraints=CA:FALSE\n" + "extendedKeyUsage=serverAuth,clientAuth\n" % (san or "DNS:" + cn)) + sh("openssl", "x509", "-req", "-in", p(name + ".csr"), "-CA", p(ca + ".crt"), "-CAkey", p(ca + ".key"), + "-CAcreateserial", "-out", p(name + ".crt"), "-days", "2", "-extfile", ext) + return p(name + ".crt"), p(name + ".key") + + out = {"ca": p("ca.crt"), "other_ca": p("other-ca.crt")} + out["server"] = leaf("server", "localhost", san="DNS:localhost,IP:127.0.0.1") + out["owner"] = leaf("owner", "owner-host") + out["partner"] = leaf("partner", "partner-host") + out["expired"] = leaf("expired", "old-host") + out["stranger"] = leaf("stranger", "owner-host", ca="other-ca") # right CN, wrong CA + out["nobody"] = leaf("nobody", "nobody-host") + return out + + +@unittest.skipUnless(HAVE_OPENSSL, "openssl not on PATH") +class ServeTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.work = tempfile.mkdtemp(prefix="fa-serve-") + cls.pki = mint_pki(cls.work) + cls.root = os.path.join(cls.work, "corpus") + os.makedirs(cls.root) + for name, text in (("apples.txt", "apples pears plums orchard harvest m@example.com\n"), + ("routers.txt", "router switch firewall vlan network segment\n")): + with open(os.path.join(cls.root, name), "w", encoding="utf-8") as fh: + fh.write(text) + cls.out = os.path.join(cls.work, "out") + cls.manifest = os.path.join(cls.work, "m.yaml") + cls.write_manifest(grants=[{"host": "partner-host", "ops": ["search"], "until": "2026-12-31"}, + {"host": "old-host", "ops": ["search", "read_range"], "until": "2026-01-01"}]) + m = cls.load() + # embedding stub + from http.server import ThreadingHTTPServer + EmbedStub.dim = 64 + cls.estub = ThreadingHTTPServer(("127.0.0.1", 0), EmbedStub) + threading.Thread(target=cls.estub.serve_forever, daemon=True).start() + cls.env = {"EMBED_URL": "http://127.0.0.1:%d" % cls.estub.server_address[1], + "FREEAGENT_TLS_CERT": cls.pki["server"][0], "FREEAGENT_TLS_KEY": cls.pki["server"][1], + "FREEAGENT_CA_ROOT": cls.pki["ca"]} + walk(m, cls.out) + extract(m, cls.out, use_external=False) + sanitize(m, cls.out, env={}) + em.chunk(m, cls.out) + em.embed(m, cls.out, env=cls.env) + catalog(m, cls.out, env={}) + cls.srv = sv.build_server(cls.manifest, cls.out, env=cls.env, bind="127.0.0.1:0", today=TODAY) + cls.port = cls.srv.server_address[1] + threading.Thread(target=cls.srv.serve_forever, daemon=True).start() + + @classmethod + def tearDownClass(cls): + cls.srv.shutdown() + cls.srv.server_close() + cls.estub.shutdown() + shutil.rmtree(cls.work, ignore_errors=True) + + @classmethod + def write_manifest(cls, grants): + import yaml + doc = {"freeagent": 1, "id": "t", "owner": {"host": "owner-host"}, + "scope": {"roots": [cls.root]}, "mask": ["read", "search", "index"], + "stages": ["walk", "extract", "sanitize", "chunk", "embed", "catalog", "seal"], + "sanitize": {"mode": "flag"}, + "embed": {"model": "stub-model", "dim": 64, "chunk": {"size_tokens": 8, "overlap_tokens": 2}}, + "sweep": "manual", + "sink": {"catalog": "host", "vectors": "local", "plaintext": "never"}, + "grants": grants} + with open(cls.manifest, "w", encoding="utf-8") as fh: + yaml.safe_dump(doc, fh, sort_keys=False) + rn.sign_manifest_file(cls.manifest, cls.pki["owner"][1], cls.pki["owner"][0]) + + @classmethod + def load(cls): + from .manifest import load_manifest + return load_manifest(cls.manifest) + + # ── client ─────────────────────────────────────────────────────────── + def client(self, who, ca=None): + ctx = ssl.create_default_context(cafile=ca or self.pki["ca"]) + ctx.check_hostname = False # test server cert is for localhost/127.0.0.1; keep simple + if who: + ctx.load_cert_chain(*self.pki[who]) + return http.client.HTTPSConnection("127.0.0.1", self.port, context=ctx, timeout=10) + + def call(self, who, method, path, body=None): + c = self.client(who) + data = json.dumps(body).encode() if body is not None else None + c.request(method, path, body=data, headers={"Content-Type": "application/json"}) + r = c.getresponse() + out = json.loads(r.read().decode("utf-8")) + c.close() + return r.status, out + + def audit(self): + with open(os.path.join(self.out, "serve.audit.jsonl"), encoding="utf-8") as fh: + return [json.loads(l) for l in fh] + + # ── tests ──────────────────────────────────────────────────────────── + def test_owner_everything(self): + st, body = self.call("owner", "GET", "/health") + self.assertEqual((st, body["ok"], body["serving"]), (200, True, True)) + st, body = self.call("owner", "POST", "/search", {"query": "firewall vlan", "k": 2}) + self.assertEqual(st, 200) + self.assertEqual(len(body["hits"]), 2) + self.assertIn("router", body["hits"][0]["text"]) + doc = body["hits"][0]["doc"] + st, body = self.call("owner", "POST", "/read_range", {"doc": doc, "start": 0, "end": 20}) + self.assertEqual(st, 200) + self.assertEqual(body["text"], "router switch firewa") + self.assertEqual(body["total"], 44) + st, body = self.call("owner", "GET", "/catalog") + self.assertEqual(st, 200) + self.assertEqual({c["path"] for c in body["cards"]}, {"apples.txt", "routers.txt"}) + # sanitized store only: the apples card head carries a token, not the address + apples = [c for c in body["cards"] if c["path"] == "apples.txt"][0] + self.assertIn("[PII_EMAIL_1]", apples["head"]) + st, body = self.call("owner", "GET", "/status") + self.assertEqual(st, 200) + # the map: directory cards, root "" aggregates everything + st, body = self.call("owner", "GET", "/catalog/dirs") + self.assertEqual((st, body["kind"]), (200, "dirs")) + root = [c for c in body["cards"] if c["path"] == ""][0] + self.assertEqual(root["files"], 2) + self.assertEqual(root["entities"], {"EMAIL": 1}) + self.assertEqual(self.call("partner", "GET", "/catalog/dirs")[0], 403) + # paging: total/offset/limit/next_offset; prefix filters by path + st, body = self.call("owner", "GET", "/catalog?limit=1") + self.assertEqual((st, body["total"], len(body["cards"]), body["next_offset"]), (200, 2, 1, 1)) + st, body = self.call("owner", "GET", "/catalog?limit=1&offset=1") + self.assertEqual((len(body["cards"]), body["next_offset"]), (1, None)) + st, body = self.call("owner", "GET", "/catalog?prefix=rou") + self.assertEqual([c["path"] for c in body["cards"]], ["routers.txt"]) + self.assertEqual(self.call("owner", "GET", "/catalog?limit=x")[0], 400) + + def test_grantee_gets_only_its_ops(self): + st, body = self.call("partner", "POST", "/search", {"query": "apples"}) + self.assertEqual(st, 200) + self.assertIn("[PII_EMAIL_1]", body["hits"][0]["text"]) # sanitized, always + doc = body["hits"][0]["doc"] + st, body = self.call("partner", "POST", "/read_range", {"doc": doc, "start": 0, "end": 5}) + self.assertEqual(st, 403) + self.assertIn("partner-host", body["error"]) + self.assertEqual(self.call("partner", "GET", "/catalog")[0], 403) + self.assertEqual(self.call("partner", "GET", "/status")[0], 403) + self.assertEqual(self.call("partner", "GET", "/health")[0], 200) + + def test_expired_grant_and_unknown_cn(self): + self.assertEqual(self.call("expired", "POST", "/search", {"query": "x"})[0], 403) + self.assertEqual(self.call("nobody", "POST", "/search", {"query": "x"})[0], 403) + self.assertEqual(self.call("nobody", "GET", "/health")[0], 200) # chained, so alive + + def test_no_cert_or_foreign_ca_never_reaches_a_handler(self): + with self.assertRaises((ssl.SSLError, ConnectionError, OSError)): + self.call(None, "GET", "/health") + with self.assertRaises((ssl.SSLError, ConnectionError, OSError)): + self.call("stranger", "GET", "/health") # CN 'owner-host' but signed by another CA + # and no audit line was written for either + self.assertFalse(any(a["cn"] is None for a in self.audit())) + + def test_read_range_is_bounded_and_sanitized_only(self): + st, body = self.call("owner", "POST", "/search", {"query": "apples", "k": 1}) + doc = body["hits"][0]["doc"] + st, body = self.call("owner", "POST", "/read_range", {"doc": doc, "start": 0, "end": 10 ** 9}) + self.assertEqual(st, 200) + self.assertLessEqual(body["end"] - body["start"], sv.READ_RANGE_MAX) + self.assertNotIn("m@example.com", body["text"]) + # one id per document: the extract and sanitized stores share the + # content hash, and the route reads ONLY the sanitized directory — + # so the extract-side plaintext for the same id is never what comes back + with open(os.path.join(self.out, "extract.jsonl"), encoding="utf-8") as fh: + ext_sha = [json.loads(l) for l in fh if "apples" in l][0]["sha256"] + st, body = self.call("owner", "POST", "/read_range", {"doc": ext_sha, "start": 0, "end": 200}) + self.assertEqual(st, 200) + self.assertIn("[PII_EMAIL_1]", body["text"]) + self.assertNotIn("m@example.com", body["text"]) + with open(os.path.join(self.out, "extract", ext_sha[:2], ext_sha + ".txt"), encoding="utf-8") as fh: + self.assertIn("m@example.com", fh.read()) # the plaintext exists on disk; it has no route + st, _ = self.call("owner", "POST", "/read_range", {"doc": "0" * 64, "start": 0, "end": 10}) + self.assertEqual(st, 404) + self.assertEqual(self.call("owner", "POST", "/read_range", {"doc": "zz", "start": 0, "end": 1})[0], 400) + self.assertEqual(self.call("owner", "POST", "/search", {"query": ""})[0], 400) + self.assertEqual(self.call("owner", "POST", "/nope", {})[0], 404) + + def test_silent_tcp_connection_does_not_stall_other_clients(self): + # with the handshake on the accept loop one idle raw TCP + # connection blocked every client; now the handshake runs in the + # connection's own thread with its own timeout + import socket + staller = socket.create_connection(("127.0.0.1", self.port)) + try: + st, body = self.call("owner", "GET", "/health") + self.assertEqual((st, body["ok"]), (200, True)) + finally: + staller.close() + + def test_audit_hashes_query_never_logs_it(self): + self.call("owner", "POST", "/search", {"query": "secret question about marisol"}) + lines = self.audit() + last = [a for a in lines if a["op"] == "search"][-1] + self.assertEqual(last["cn"], "owner-host") + self.assertEqual(len(last["q"]), 16) + blob = json.dumps(lines) + self.assertNotIn("marisol", blob) + + def test_revoke_grant_by_resigning(self): + self.assertEqual(self.call("partner", "POST", "/search", {"query": "apples"})[0], 200) + self.write_manifest(grants=[]) # drop the grant, re-sign + self.assertEqual(self.call("partner", "POST", "/search", {"query": "apples"})[0], 403) + self.assertEqual(self.call("owner", "POST", "/search", {"query": "apples"})[0], 200) + # tamper without re-signing -> the endpoint refuses everything but health + with open(self.manifest, encoding="utf-8") as fh: + text = fh.read() + with open(self.manifest, "w", encoding="utf-8") as fh: + fh.write(text.replace("sweep: manual", "sweep: daily")) + st, body = self.call("owner", "POST", "/search", {"query": "apples"}) + self.assertEqual(st, 503) + self.assertIn("gate", body["error"]) + self.assertEqual(self.call("owner", "GET", "/health")[1]["serving"], False) + # restore + self.write_manifest(grants=[{"host": "partner-host", "ops": ["search"], "until": "2026-12-31"}, + {"host": "old-host", "ops": ["search", "read_range"], "until": "2026-01-01"}]) + self.assertEqual(self.call("owner", "POST", "/search", {"query": "apples"})[0], 200) + + +class UnitTests(unittest.TestCase): + def test_policy_fails_closed_when_the_manifest_cannot_be_read(self): + # Review: an unreadable manifest (PermissionError, bad encoding) is + # not a ManifestError, and it escaped after _mtime advanced — leaving + # the PREVIOUS manifest in force. + from unittest import mock + work = tempfile.mkdtemp(prefix="fa-pol-") + self.addCleanup(shutil.rmtree, work, ignore_errors=True) + man = os.path.join(work, "m.yaml") + with open(man, "w") as fh: + fh.write("x: 1\n") + good = object() + with mock.patch.object(sv, "load_manifest", return_value=good), \ + mock.patch.object(sv, "verify_signature", return_value={}): + pol = sv.Policy(man, {}) + self.assertIs(pol.manifest, good) + os.utime(man, ns=(1, 1)) + with mock.patch.object(sv, "load_manifest", side_effect=PermissionError("denied")): + pol.reload() + self.assertIsNone(pol.manifest) + self.assertIn("manifest unreadable", pol.gate_error) + + # skipped rather than faked: the port file is written by build_server + # only after it has loaded a real certificate chain and verified a signed + # manifest, and minting those needs openssl + @unittest.skipUnless(HAVE_OPENSSL, "openssl not on PATH: needed to mint the test PKI and sign the manifest") + def test_port_file_written_after_bind(self): + # a supervisor binds :0 and reads the port back — one atomic file + work = tempfile.mkdtemp(prefix="fa-pf-") + try: + pki = mint_pki(work) + man = os.path.join(work, "m.yaml") + import yaml + with open(man, "w") as fh: + yaml.safe_dump({"freeagent": 1, "id": "p", "owner": {"host": "owner-host"}, + "scope": {"roots": [work]}, "mask": ["read"], "stages": ["walk"], + "sweep": "manual", + "sink": {"catalog": "local", "vectors": "local", "plaintext": "never"}}, fh) + rn.sign_manifest_file(man, pki["owner"][1], pki["owner"][0]) + env = {"FREEAGENT_TLS_CERT": pki["server"][0], "FREEAGENT_TLS_KEY": pki["server"][1], + "FREEAGENT_CA_ROOT": pki["ca"]} + pf = os.path.join(work, "port") + srv = sv.build_server(man, os.path.join(work, "out"), env=env, bind="127.0.0.1:0", port_file=pf) + try: + with open(pf) as fh: + self.assertEqual(int(fh.read().strip()), srv.server_address[1]) + finally: + srv.server_close() + finally: + shutil.rmtree(work, ignore_errors=True) + + def test_parse_bind(self): + self.assertEqual(sv.parse_bind("[::]:8470"), ("::", 8470)) + self.assertEqual(sv.parse_bind("10.0.0.5:9"), ("10.0.0.5", 9)) + self.assertEqual(sv.parse_bind("[fd00:1::31]:8470"), ("fd00:1::31", 8470)) + with self.assertRaises(sv.ServeError): + sv.parse_bind("nope") + + def test_peer_cn(self): + self.assertEqual(sv.peer_cn({"subject": ((("commonName", "x"),),)}), "x") + self.assertIsNone(sv.peer_cn({})) + self.assertIsNone(sv.peer_cn(None)) + + def test_ssl_context_requires_all_three(self): + with self.assertRaises(sv.ServeError): + sv.build_ssl_context("", "", "") + + +if __name__ == "__main__": + unittest.main() diff --git a/bastion_agent/test_verify_release.py b/bastion_agent/test_verify_release.py new file mode 100644 index 0000000..d7c987c --- /dev/null +++ b/bastion_agent/test_verify_release.py @@ -0,0 +1,213 @@ +"""Tests for the release verifier. Run: python3 -m unittest discover .""" + +import base64 +import contextlib +import hashlib +import io +import json +import os +import shutil +import subprocess +import tempfile +import unittest +from unittest import mock + +from . import build as bd +from . import verify_release as vr + +HERE = os.path.dirname(os.path.abspath(__file__)) +REPO = os.path.dirname(HERE) + + +def _sign(seed, msg): + """RFC 8032 signing, for the tests only: the verifier never signs.""" + h = hashlib.sha512(seed).digest() + a = int.from_bytes(h[:32], "little") + a &= (1 << 254) - 8 + a |= 1 << 254 + pub = vr._compress(vr._mul(a, vr._G)) + r = int.from_bytes(hashlib.sha512(h[32:] + msg).digest(), "little") % vr._Q + big_r = vr._compress(vr._mul(r, vr._G)) + k = int.from_bytes(hashlib.sha512(big_r + pub + msg).digest(), "little") % vr._Q + return big_r + ((r + k * a) % vr._Q).to_bytes(32, "little") + + +class Ed25519Tests(unittest.TestCase): + # RFC 8032, section 7.1, tests 1 and 2. + RFC = ( + ("9d61b19deffd5a60ba844af492ec2cc44449c5697b326919703bac031cae7f60", + "d75a980182b10ab7d54bfed3c964073a0ee172f3daa62325af021a68f707511a", + "", + "e5564300c360ac729086e2cc806e828a84877f1eb8e5d974d873e065224901555fb8821590a33bacc61e39701cf9b46bd25bf5f0595bbe24655141438e7a100b"), + ("4ccd089b28ff96da9db6c346ec114e0f5b8a319f35aba624da8cf6ed4fb8a6fb", + "3d4017c3e843895a92b70aa74d1b7ebc9c982ccf2ec4968cc0cd55f12af4660c", + "72", + "92a009a9f0d4cab8720e820b5f642540a2b27b5416503f8fb3762223ebdb69da085ac1e43e15996e458f3613d0f11d8c387b2eaeb4302aeeb00d291612bb0c00"), + ) + + def test_rfc8032_vectors(self): + for seed, pub, msg, sig in self.RFC: + seed, pub, msg, sig = (bytes.fromhex(x) for x in (seed, pub, msg, sig)) + self.assertEqual(vr.ed25519_public_key(seed), pub) + self.assertTrue(vr.ed25519_verify(pub, msg, sig)) + self.assertEqual(_sign(seed, msg), sig) + + def test_any_changed_bit_is_refused(self): + seed, pub, msg, sig = (bytes.fromhex(x) for x in self.RFC[1]) + self.assertFalse(vr.ed25519_verify(pub, msg + b"\x00", sig)) + for i in (0, 31, 32, 63): + bad = bytearray(sig) + bad[i] ^= 1 + self.assertFalse(vr.ed25519_verify(pub, msg, bytes(bad))) + other = vr.ed25519_public_key(b"\x02" * 32) + self.assertFalse(vr.ed25519_verify(other, msg, sig)) + + def test_malformed_input_is_false_not_an_error(self): + _, pub, msg, sig = (bytes.fromhex(x) for x in self.RFC[0]) + self.assertFalse(vr.ed25519_verify(pub[:31], msg, sig)) + self.assertFalse(vr.ed25519_verify(pub, msg, sig[:63])) + # S at or above the group order is refused (signature malleability). + s = int.from_bytes(sig[32:], "little") + vr._Q + if s < 1 << 256: + self.assertFalse(vr.ed25519_verify(pub, msg, sig[:32] + s.to_bytes(32, "little"))) + # A y coordinate that is on no point of the curve. + self.assertFalse(vr.ed25519_verify(b"\x02" + b"\x00" * 31, msg, sig)) + + +class StatementTests(unittest.TestCase): + def test_statement_bytes(self): + self.assertEqual(vr.statement("bastion-agent", "0.1.13", "ab" * 32), + b"privatae-release-signature-v1\nbastion-agent\n0.1.13\n" + b"ab" * 32 + b"\n") + + def test_the_signing_tool_and_this_check_agree(self): + # The vector the bastion's release-signing tool and the bastion share: + # seed of 32 bytes of 1, this manifest, kind "bastion", this signature. + # If the statement format drifted here, this would fail. + pub = base64.b64encode(vr.ed25519_public_key(b"\x01" * 32)).decode() + body = b'{"version":"1.2.3","binaries":{}}' + sig = "tr5h3oXjBDmwvwQiZSriW4T6Y6oBFQDymfofLEGXMdTJSXfoQAKN+ajhCs89FO9vdCEZH3Y8mYzYXPZ7RzZVDw==" + m, i = vr.check_signature(body, sig, keys=(pub,), kind="bastion") + self.assertEqual((m["version"], i), ("1.2.3", 0)) + # Bound to its kind: a bastion manifest's signature is not an agent's. + with self.assertRaises(vr.VerifyError): + vr.check_signature(body, sig, keys=(pub,), kind="bastion-agent") + + def test_signature_file_name(self): + body = b'{"version":"0.1.13"}' + self.assertEqual(vr.sig_name(body), + "version.json.%s.sig" % hashlib.sha256(body).hexdigest()[:16]) + + def test_pinned_keys_are_two_valid_points(self): + self.assertEqual(len(vr.RELEASE_KEYS), 2) + for k in vr.RELEASE_KEYS: + raw = base64.b64decode(k, validate=True) + self.assertEqual(len(raw), 32) + self.assertIsNotNone(vr._decompress(raw)) + + def test_refusals(self): + seed = b"\x03" * 32 + pub = base64.b64encode(vr.ed25519_public_key(seed)).decode() + + def signed(body): + msg = vr.statement("bastion-agent", json.loads(body)["version"], hashlib.sha256(body).hexdigest()) + return base64.b64encode(_sign(seed, msg)).decode() + + body = b'{"version":"0.2.0"}' + self.assertEqual(vr.check_signature(body, signed(body), keys=(pub,))[1], 0) + # The key that verified is reported by position: the spare is second. + self.assertEqual(vr.check_signature(body, signed(body), keys=(vr.RELEASE_KEYS[0], pub))[1], 1) + cases = ( + (b"not json", "AAAA"), + (b'{"version":""}', signed(body)), + (b'{"version":"0.2.0\\n"}', signed(body)), + (body, "not base64!"), + (body, base64.b64encode(b"short").decode()), + (body + b" ", signed(body)), # the manifest changed after signing + (body, signed(body)), # the production keys did not sign it + ) + for i, (b, s) in enumerate(cases): + keys = None if i == len(cases) - 1 else (pub,) + with self.subTest(i=i), self.assertRaises(vr.VerifyError): + vr.check_signature(b, s, keys=keys) + + +@unittest.skipUnless(shutil.which("git"), "needs git") +class EndToEndTests(unittest.TestCase): + """The whole check against a throwaway repository holding this tree.""" + + SEED = b"\x04" * 32 + + @classmethod + def setUpClass(cls): + cls.tmp = tempfile.mkdtemp(prefix="agent-verify-test-") + cls.repo = os.path.join(cls.tmp, "repo") + ignore = shutil.ignore_patterns("__pycache__", "*.pyc") + shutil.copytree(HERE, os.path.join(cls.repo, "bastion_agent"), ignore=ignore) + shutil.copytree(os.path.join(REPO, "pii"), os.path.join(cls.repo, "pii"), ignore=ignore) + shutil.copy(os.path.join(REPO, "LICENSE"), cls.repo) + git = ("git", "-C", cls.repo, "-c", "user.name=test", "-c", "user.email=test@example.invalid", + "-c", "commit.gpgsign=false", "-c", "tag.gpgsign=false") + run = lambda *a: subprocess.run(git + a, check=True, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, universal_newlines=True).stdout.strip() + subprocess.run(("git", "init", "-q", cls.repo), check=True) + run("add", "-A") + run("commit", "-q", "-m", "release") + cls.commit = run("rev-parse", "HEAD") + cls.run_git = staticmethod(run) + cls.version = bd.version() + # The same files as this tree, so the same bytes as building this tree. + cls.digest = hashlib.sha256(bd.build_bytes()).hexdigest() + cls.pub = base64.b64encode(vr.ed25519_public_key(cls.SEED)).decode() + + @classmethod + def tearDownClass(cls): + shutil.rmtree(cls.tmp, ignore_errors=True) + + def manifest(self, built_from, digest=None): + name = "bastion-agent-%s.pyz" % self.version + return json.dumps({"version": self.version, "built_from": built_from, + "files": {name: {"sha256": digest or self.digest}, + name + ".sha256": {"sha256": "0" * 64}}}).encode() + + def verify(self, body, seed=None): + msg = vr.statement("bastion-agent", self.version, hashlib.sha256(body).hexdigest()) + sig = base64.b64encode(_sign(seed or self.SEED, msg)) + served = {"/version.json": body, "/" + vr.sig_name(body): sig} + out, err = io.StringIO(), io.StringIO() + with mock.patch.object(vr, "fetch", lambda url: served[url[len("https://test"):]]), \ + contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + code = vr.main(["--base", "https://test"], repo=self.repo, keys=(self.pub,)) + return code, out.getvalue() + err.getvalue() + + def test_tag_first_then_built_from(self): + code, out = self.verify(self.manifest("0000000")) + self.assertEqual(code, 1, out) + self.assertIn("neither the tag v%s" % self.version, out) + + code, out = self.verify(self.manifest(self.commit[:7])) + self.assertEqual(code, 0, out) + self.assertIn("source: built_from", out) + self.assertIn("matches %s" % self.digest, out) + + self.run_git("tag", "v" + self.version) + try: + code, out = self.verify(self.manifest("0000000")) + self.assertEqual(code, 0, out) + self.assertIn("source: tag v%s" % self.version, out) + finally: + self.run_git("tag", "-d", "v" + self.version) + + def test_a_different_build_is_reported(self): + code, out = self.verify(self.manifest(self.commit, digest="1" * 64)) + self.assertEqual(code, 1, out) + self.assertIn("DIFFERS", out) + + def test_a_foreign_signature_stops_before_any_checkout(self): + code, out = self.verify(self.manifest(self.commit), seed=b"\x05" * 32) + self.assertEqual(code, 1, out) + self.assertIn("does not verify", out) + self.assertNotIn("source:", out) + + +if __name__ == "__main__": + unittest.main() diff --git a/bastion_agent/test_walk.py b/bastion_agent/test_walk.py new file mode 100644 index 0000000..c9dc40a --- /dev/null +++ b/bastion_agent/test_walk.py @@ -0,0 +1,271 @@ +"""Unit tests for the walk stage. Run: python3 -m unittest discover . + +Builds a real temp tree so the walker is exercised on whatever OS runs the +suite; Windows-only behaviours (junctions, casefold) are covered by the +pieces that can be tested everywhere (the exclude matcher, _is_link fallback) +and by the platform guard in each test. +""" + +import hashlib +import json +import os +import shutil +import sys +import tempfile +import unittest +from unittest import mock + +from .manifest import validate_document +from .walk import Excluder, Walker, _glob_to_regex, load_hash_cache, walk + +IS_WINDOWS = os.name == "nt" + + +def manifest(root, **scope): + doc = { + "freeagent": 1, "id": "t", "owner": {"host": "h"}, + "scope": dict({"roots": [root]}, **scope), + "mask": ["read"], "stages": ["walk"], "sweep": "manual", + "sink": {"catalog": "local", "vectors": "local", "plaintext": "never"}, + } + return validate_document(doc) + + +def touch(path, data=b""): + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "wb") as fh: + fh.write(data) + + +def read_jsonl(path): + with open(path, encoding="utf-8") as fh: + return [json.loads(line) for line in fh] + + +class TreeCase(unittest.TestCase): + def setUp(self): + self.root = tempfile.mkdtemp(prefix="fa-walk-") + self.out = tempfile.mkdtemp(prefix="fa-out-") + touch(os.path.join(self.root, "a.txt"), b"hello\n") + touch(os.path.join(self.root, "docs", "b.pdf"), b"%PDF-1.7 fake") + touch(os.path.join(self.root, "docs", "c.bin"), b"\x00\x01\x02") + touch(os.path.join(self.root, "docs", "empty"), b"") + touch(os.path.join(self.root, "node_modules", "x", "y.js"), b"1") + touch(os.path.join(self.root, "secret.key"), b"k") + touch(os.path.join(self.root, "report.docx"), b"PK\x03\x04zip") + + def tearDown(self): + shutil.rmtree(self.root, ignore_errors=True) + shutil.rmtree(self.out, ignore_errors=True) + + def by_path(self, entries): + return {e["path"]: e for e in entries} + + +class WalkTests(TreeCase): + def test_map_shape_and_sorting(self): + s = walk(manifest(self.root), self.out, now="2026-08-28T00:00:00Z") + entries = read_jsonl(os.path.join(self.out, "walk.jsonl")) + paths = [e["path"] for e in entries] + self.assertEqual(paths, sorted(paths)) + m = self.by_path(entries) + self.assertEqual(m["a.txt"]["kind"], "file") + self.assertEqual(m["a.txt"]["size"], 6) + self.assertEqual(m["a.txt"]["mime"], "text/plain") + self.assertEqual(m["a.txt"]["sha256"], hashlib.sha256(b"hello\n").hexdigest()) + self.assertTrue(m["a.txt"]["mtime"].endswith("Z")) + self.assertEqual(m["docs"]["kind"], "dir") + self.assertEqual(m["docs/b.pdf"]["mime"], "application/pdf") + self.assertEqual(m["docs/c.bin"]["mime"], "application/octet-stream") + self.assertEqual(m["docs/empty"]["mime"], "inode/x-empty") + self.assertEqual(m["report.docx"]["mime"], + "application/vnd.openxmlformats-officedocument.wordprocessingml.document") + self.assertEqual(s["entries"], len(entries)) + self.assertEqual(s["counts"]["files"], 7) + self.assertEqual(s["counts"]["errors"], 0) + self.assertEqual(s["walked_at"], "2026-08-28T00:00:00Z") + self.assertTrue(s["manifest_digest"].startswith("sha256:")) + self.assertEqual(s["roots"][0]["resolved"], os.path.abspath(self.root)) + # paths are '/'-separated on every OS + self.assertTrue(all("\\" not in e["path"] for e in entries)) + + def test_agent_dir_is_always_excluded(self): + touch(os.path.join(self.root, ".agent", "catalog.jsonl"), b"{}") + touch(os.path.join(self.root, "sub", ".agent", "x"), b"x") + touch(os.path.join(self.root, "other", ".Agent", "y"), b"y") # case-insensitive volumes + walk(manifest(self.root), self.out) + paths = {e["path"] for e in read_jsonl(os.path.join(self.out, "walk.jsonl"))} + self.assertFalse(any(".agent" in p.lower() for p in paths), paths) + + def test_excludes_prune(self): + m = manifest(self.root, exclude=["node_modules", "*.key", "docs/c.*"]) + s = walk(m, self.out) + paths = {e["path"] for e in read_jsonl(os.path.join(self.out, "walk.jsonl"))} + self.assertNotIn("node_modules", paths) + self.assertNotIn("node_modules/x/y.js", paths) + self.assertNotIn("secret.key", paths) + self.assertNotIn("docs/c.bin", paths) + self.assertIn("docs/b.pdf", paths) + self.assertEqual(s["counts"]["excluded"], 3) + + def test_too_large_is_metadata_only(self): + m = manifest(self.root, max_file_bytes=4) + walk(m, self.out) + e = self.by_path(read_jsonl(os.path.join(self.out, "walk.jsonl"))) + self.assertEqual(e["a.txt"]["skip"], "too_large") + self.assertIsNone(e["a.txt"]["sha256"]) + self.assertEqual(e["a.txt"]["mime"], "text/plain") # still sniffed + self.assertEqual(e["docs/c.bin"]["sha256"], hashlib.sha256(b"\x00\x01\x02").hexdigest()) + + def test_no_hash(self): + s = walk(manifest(self.root), self.out, do_hash=False) + self.assertEqual(s["counts"]["hashed"], 0) + self.assertFalse(s["hashing"]) + e = self.by_path(read_jsonl(os.path.join(self.out, "walk.jsonl"))) + self.assertIsNone(e["a.txt"]["sha256"]) + + def test_hash_cache_reuse_and_invalidation(self): + walk(manifest(self.root), self.out) + s2 = walk(manifest(self.root), self.out) + self.assertEqual(s2["counts"]["hashed"], 0) + self.assertEqual(s2["counts"]["hash_reused"], 7) + # a content change with a new mtime is re-hashed + p = os.path.join(self.root, "a.txt") + touch(p, b"changed\n") + os.utime(p, ns=(1, 1_700_000_000_000_000_000)) + s3 = walk(manifest(self.root), self.out) + self.assertEqual(s3["counts"]["hashed"], 1) + e = self.by_path(read_jsonl(os.path.join(self.out, "walk.jsonl"))) + self.assertEqual(e["a.txt"]["sha256"], hashlib.sha256(b"changed\n").hexdigest()) + + def test_deterministic_output(self): + walk(manifest(self.root), self.out) + first = open(os.path.join(self.out, "walk.jsonl"), "rb").read() + walk(manifest(self.root), self.out) + self.assertEqual(first, open(os.path.join(self.out, "walk.jsonl"), "rb").read()) + + def test_missing_root_is_an_error_not_a_crash(self): + missing = os.path.join(self.root, "nope") + m = manifest(missing) + s = walk(m, self.out) + self.assertEqual(s["counts"]["errors"], 1) + self.assertIn("error", s["roots"][0]) + self.assertEqual(s["entries"], 0) + + def test_unreadable_dir_recorded(self): + real = os.scandir + + def flaky(path): + if os.path.basename(path) == "docs": + raise PermissionError(13, "Permission denied") + return real(path) + + with mock.patch("bastion_agent.walk.os.scandir", flaky): + s = walk(manifest(self.root), self.out) + e = self.by_path(read_jsonl(os.path.join(self.out, "walk.jsonl"))) + self.assertEqual(s["counts"]["errors"], 1) + self.assertIn("Permission denied", e["docs"]["error"]) + self.assertNotIn("docs/b.pdf", e) + + def test_unicode_paths(self): + touch(os.path.join(self.root, "résumé — 履歴書.txt"), b"x") + walk(manifest(self.root), self.out) + e = self.by_path(read_jsonl(os.path.join(self.out, "walk.jsonl"))) + self.assertIn("résumé — 履歴書.txt", e) + + def test_home_root_expands(self): + with mock.patch.dict(os.environ, {"HOME": self.root, "USERPROFILE": self.root}): + s = walk(manifest("~"), self.out) + self.assertEqual(s["roots"][0]["resolved"], os.path.abspath(self.root)) + self.assertGreater(s["entries"], 0) + + @unittest.skipIf(IS_WINDOWS, "symlink creation needs a privilege on Windows") + def test_symlink_recorded_not_followed(self): + os.symlink(os.path.join(self.root, "docs"), os.path.join(self.root, "link-to-docs")) + os.symlink("a.txt", os.path.join(self.root, "link-to-a")) + s = walk(manifest(self.root), self.out) + e = self.by_path(read_jsonl(os.path.join(self.out, "walk.jsonl"))) + self.assertEqual(e["link-to-docs"]["kind"], "symlink") + self.assertEqual(e["link-to-a"]["target"], "a.txt") + self.assertNotIn("link-to-docs/b.pdf", e) + self.assertEqual(s["counts"]["symlinks"], 2) + + @unittest.skipIf(IS_WINDOWS, "symlink creation needs a privilege on Windows") + def test_follow_symlinks_when_granted(self): + os.symlink(os.path.join(self.root, "docs"), os.path.join(self.root, "link-to-docs")) + walk(manifest(self.root, follow_symlinks=True), self.out) + e = self.by_path(read_jsonl(os.path.join(self.out, "walk.jsonl"))) + self.assertEqual(e["link-to-docs"]["kind"], "dir") + self.assertIn("link-to-docs/b.pdf", e) + + +class ExcludeTests(unittest.TestCase): + def test_glob_translation(self): + import re + rx = lambda p: re.compile("^" + _glob_to_regex(p) + "$") + self.assertTrue(rx("**/node_modules/**").match("a/b/node_modules/c/d.js")) + self.assertTrue(rx("**/node_modules/**").match("node_modules/c")) + self.assertTrue(rx("*.key").match("id.key")) + self.assertFalse(rx("*.key").match("a/id.key")) + self.assertTrue(rx("docs/*.pdf").match("docs/x.pdf")) + self.assertFalse(rx("docs/*.pdf").match("docs/sub/x.pdf")) + self.assertTrue(rx("docs/**").match("docs/sub/x.pdf")) + self.assertTrue(rx("[!a]?.txt").match("bx.txt")) + self.assertFalse(rx("[!a]?.txt").match("ax.txt")) + + def test_double_star_dir_pattern_prunes_the_dir(self): + ex = Excluder(["**/.git/**"]) + self.assertTrue(ex.matches(".git")) + self.assertTrue(ex.matches("a/.git")) + self.assertTrue(ex.matches("a/.git/HEAD")) + self.assertFalse(ex.matches("a/.gitignore")) + + def test_segment_patterns_match_at_any_depth(self): + ex = Excluder(["node_modules", "*.key", ".git"]) + self.assertTrue(ex.matches("node_modules")) + self.assertTrue(ex.matches("a/b/node_modules")) + self.assertTrue(ex.matches("deep/id.key")) + self.assertTrue(ex.matches("proj/.git")) + self.assertFalse(ex.matches("proj/gitignore")) + + def test_path_patterns_are_anchored(self): + ex = Excluder(["docs/private/**", "**/*.tmp"]) + self.assertTrue(ex.matches("docs/private/x")) + self.assertTrue(ex.matches("docs/private")) # the dir itself is pruned + self.assertFalse(ex.matches("docs")) + self.assertFalse(ex.matches("other/docs/private/x")) + self.assertTrue(ex.matches("a.tmp")) + self.assertTrue(ex.matches("a/b/c.tmp")) + + def test_backslash_patterns_normalised(self): + ex = Excluder(["docs\\private\\**"]) + self.assertTrue(ex.matches("docs/private/x")) + + def test_case_sensitivity_matches_os(self): + ex = Excluder(["*.PDF"]) + self.assertEqual(ex.matches("x.pdf"), IS_WINDOWS) + + +class CacheTests(unittest.TestCase): + def test_cache_ignores_garbage_and_non_files(self): + fd, path = tempfile.mkstemp(suffix=".jsonl") + with os.fdopen(fd, "w") as fh: + fh.write('{"root":0,"path":"a","kind":"file","size":1,"mtime_ns":2,"sha256":"h"}\n') + fh.write('{"root":0,"path":"d","kind":"dir"}\n') + fh.write("not json\n") + try: + c = load_hash_cache(path) + finally: + os.unlink(path) + self.assertEqual(c, {(0, "a"): (1, 2, "h")}) + + def test_missing_cache_is_empty(self): + self.assertEqual(load_hash_cache("/nonexistent/walk.jsonl"), {}) + + +# The Python 3.8 syntax floor is enforced for this package by +# test_syntax_floor.py (shared with pii). + + +if __name__ == "__main__": + unittest.main() diff --git a/bastion_agent/test_x509gate.py b/bastion_agent/test_x509gate.py new file mode 100644 index 0000000..faace7d --- /dev/null +++ b/bastion_agent/test_x509gate.py @@ -0,0 +1,139 @@ +"""The manifest gate needs no openssl (x509gate). + +A Windows machine ships no openssl, and the gate used to shell out to it: the +agent refused every manifest there and never ran. These tests mint real +certificates with openssl (a test-machine tool only) and then run the gate with openssl REMOVED from PATH, and cover +the managed install's signing lane — the bastion signs `payload` with its +own key and records it with `sign --value`. +""" + +import base64 +import datetime as _dt +import os +import shutil +import subprocess +import tempfile +import unittest +from unittest import mock + +from . import run as _run +from . import x509gate as x +from .manifest import load_manifest +from .test_run import write_manifest + +HAVE_OPENSSL = bool(shutil.which("openssl")) + + + +def sh(*argv, data=None): + return subprocess.run(argv, input=data, check=True, stdout=subprocess.PIPE, + stderr=subprocess.PIPE).stdout + + +def _der_sign(key, data): + return sh("openssl", "dgst", "-sha256", "-sign", key, data=data) + + +@unittest.skipUnless(HAVE_OPENSSL, "openssl is needed to MINT the test certificates") +class GateWithoutOpenssl(unittest.TestCase): + def setUp(self): + self.d = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.d, ignore_errors=True) + p = lambda n: os.path.join(self.d, n) # noqa: E731 + self.p = p + sh("openssl", "ecparam", "-name", "prime256v1", "-genkey", "-noout", "-out", p("host.key")) + sh("openssl", "req", "-new", "-x509", "-key", p("host.key"), "-out", p("host.crt"), + "-days", "2", "-subj", "/CN=desk-example") + os.makedirs(p("root")) + write_manifest(p("m.yaml"), p("root"), ["walk"], cn="desk-example") + self.env = {"FREEAGENT_CA_ROOT": p("host.crt")} + + def _no_openssl(self): + return mock.patch.object(shutil, "which", lambda name, *a, **k: None) + + def _bastion_sign(self): + """What the managed install does: sign `payload` with its own key and + record it with `sign --value` — no openssl on the machine.""" + der = _der_sign(self.p("host.key"), _run.manifest_payload(self.p("m.yaml"))) + with self._no_openssl(): + _run.write_signature(self.p("m.yaml"), der, self.p("host.crt")) + + def test_gate_verifies_with_openssl_removed_from_path(self): + self._bastion_sign() + with self._no_openssl(): + m = load_manifest(self.p("m.yaml"), require_signed=True) + got = _run.verify_signature(m, self.env, manifest_dir=self.d) + self.assertEqual(got["cn"], "desk-example") + self.assertTrue(got["chained"]) + + def test_tampered_manifest_refused_without_openssl(self): + self._bastion_sign() + with open(self.p("m.yaml")) as fh: + text = fh.read() + with open(self.p("m.yaml"), "w") as fh: + fh.write(text.replace("sweep: manual", "sweep: hourly", 1)) + with self._no_openssl(): + m = load_manifest(self.p("m.yaml"), require_signed=True) + with self.assertRaises(_run.GateError) as cm: + _run.verify_signature(m, self.env, manifest_dir=self.d) + self.assertIn("does not verify", str(cm.exception)) + + def test_cli_sign_value_records_a_bastion_signature(self): + der = _der_sign(self.p("host.key"), _run.manifest_payload(self.p("m.yaml"))) + env = dict(os.environ, PATH="/nonexistent") # no openssl for the CLI either + r = subprocess.run( + [shutil.which("python3") or "python3", "-m", "bastion_agent", "sign", self.p("m.yaml"), + "--value", base64.b64encode(der).decode(), "--cert", self.p("host.crt")], + cwd=os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + self.assertEqual(r.returncode, 0, r.stderr) + with self._no_openssl(): + m = load_manifest(self.p("m.yaml"), require_signed=True) + _run.verify_signature(m, self.env, manifest_dir=self.d) + + def test_expired_cert_refused(self): + self._bastion_sign() + leaf = x.load_cert_file(self.p("host.crt")) + later = _dt.datetime.now(_dt.timezone.utc) + _dt.timedelta(days=30) + with self.assertRaises(x.CertError): + x.verify_chain(leaf, [leaf], now=later) + + def test_cert_signed_by_another_ca_refused(self): + p = self.p + for n in ("ca", "other"): + sh("openssl", "ecparam", "-name", "prime256v1", "-genkey", "-noout", "-out", p(n + ".key")) + sh("openssl", "req", "-new", "-x509", "-key", p(n + ".key"), "-out", p(n + ".crt"), + "-days", "2", "-subj", "/CN=Same Name CA") + sh("openssl", "ecparam", "-name", "prime256v1", "-genkey", "-noout", "-out", p("leaf.key")) + sh("openssl", "req", "-new", "-key", p("leaf.key"), "-out", p("leaf.csr"), "-subj", "/CN=desk-example") + sh("openssl", "x509", "-req", "-in", p("leaf.csr"), "-CA", p("other.crt"), "-CAkey", p("other.key"), + "-CAcreateserial", "-out", p("leaf.crt"), "-days", "2") + leaf = x.load_cert_file(p("leaf.crt")) + ca = x.load_cert_file(p("ca.crt")) + # Same subject, different key: the name matches, the signature must not. + with self.assertRaises(x.CertError): + x.verify_chain(leaf, [ca]) + x.verify_chain(leaf, [x.load_cert_file(p("other.crt"))]) + + def test_signature_bit_flips_never_verify(self): + leaf = x.load_cert_file(self.p("host.crt")) + for i in range(8): + msg = os.urandom(40 + i) + der = _der_sign(self.p("host.key"), msg) + self.assertTrue(x.ecdsa_p256_sha256_verify(leaf.public_key, msg, der)) + bad = bytearray(der) + bad[-1 - i] ^= 0x01 + self.assertFalse(x.ecdsa_p256_sha256_verify(leaf.public_key, msg, bytes(bad))) + self.assertFalse(x.ecdsa_p256_sha256_verify(leaf.public_key, msg + b"x", der)) + + +class DerSafety(unittest.TestCase): + def test_garbage_is_a_cert_error_not_a_crash(self): + for junk in (b"", b"\x30", b"\x30\x84\xff\xff\xff\xff", os.urandom(64)): + with self.assertRaises(x.CertError): + x.Cert(junk) + self.assertFalse(x.ecdsa_p256_sha256_verify(x._G, b"m", b"\x30\x00")) + + +if __name__ == "__main__": + unittest.main() diff --git a/bastion_agent/verify_release.py b/bastion_agent/verify_release.py new file mode 100644 index 0000000..31c02b6 --- /dev/null +++ b/bastion_agent/verify_release.py @@ -0,0 +1,318 @@ +"""Rebuild the published bastion agent from source and check it is what was published. + + python3 -m bastion_agent.verify_release [--base URL] + +Run it from a clone of this repository. It needs Python 3.8 or later, git, +network access to the download address, and PyYAML at the version build.py +pins (the build vendors it). It needs no key or account of ours. + +It fetches the release manifest (version.json) and the signature beside it, +checks the signature against the release keys the bastion pins, checks out the +source of that version in a separate worktree, rebuilds the .pyz with that +version's own build.py, and compares the sha256 with the manifest's. A match +means the file every bastion downloads is exactly what this source produces, +and the signature says the release key holder published it. + +The source is the tag v. The public repository is a snapshot of the +development one, so the commit the manifest names (built_from) exists only +there; the tag marks the same tree in the public history. When the tag is +absent and built_from is present (a clone of the development repository), +built_from is used instead. + +The signature check is ed25519 in pure Python, because the standard library +has no ed25519 and this has to run where nothing else is installed. It only +verifies public data, so speed and side channels do not matter here; the code +follows the reference implementation in RFC 8032, section 6. + +This module is a tool for checking releases. build.py leaves it out of the +.pyz: a verifier shipped inside the file it verifies would prove nothing. +""" + +import argparse +import base64 +import hashlib +import json +import os +import shutil +import subprocess +import sys +import tempfile +import urllib.request + +HERE = os.path.dirname(os.path.abspath(__file__)) +REPO = os.path.dirname(HERE) +DEFAULT_BASE = "https://privatae.ai/bastion-agent" +KIND = "bastion-agent" + +# The public halves of the release signing keys: base64 of the raw 32-byte +# ed25519 key, the primary first and then the spare kept offline. They are the +# keys the bastion compiles in (productionReleaseKeys in client-go/release_sign.go +# in the bastion's repository), because a bastion refuses any agent manifest +# that does not verify against those. When the bastion rotates a key, this list +# changes in the same way, or this check fails where a bastion would pass. +RELEASE_KEYS = ( + "tWOyZm38DwTwbawSnvExQG/CQsVJBKYWc46arCBZj9E=", + "CKL+J0VSdqrfY4AmxChvylJ1ha4e8GvcrYF2o17Ah+g=", +) + +FETCH_TIMEOUT = 30 +FETCH_MAX = 1 << 20 # a manifest or a signature is a few kilobytes + + +class VerifyError(Exception): + pass + + +# ---- ed25519 verification (RFC 8032, section 6) ----------------------------- + +_P = 2 ** 255 - 19 +_Q = 2 ** 252 + 27742317777372353535851937790883648493 +_D = -121665 * pow(121666, _P - 2, _P) % _P +_SQRT_M1 = pow(2, (_P - 1) // 4, _P) + + +def _inv(x): + return pow(x, _P - 2, _P) + + +def _add(a, b): + # Extended homogeneous coordinates (X, Y, Z, T). + ea = (a[1] - a[0]) * (b[1] - b[0]) % _P + eb = (a[1] + a[0]) * (b[1] + b[0]) % _P + ec = 2 * a[3] * b[3] * _D % _P + ed = 2 * a[2] * b[2] % _P + e, f, g, h = eb - ea, ed - ec, ed + ec, eb + ea + return (e * f % _P, g * h % _P, f * g % _P, e * h % _P) + + +def _mul(s, pt): + acc = (0, 1, 1, 0) + while s > 0: + if s & 1: + acc = _add(acc, pt) + pt = _add(pt, pt) + s >>= 1 + return acc + + +def _equal(a, b): + return (a[0] * b[2] - b[0] * a[2]) % _P == 0 and (a[1] * b[2] - b[1] * a[2]) % _P == 0 + + +def _recover_x(y, sign): + if y >= _P: + return None + x2 = (y * y - 1) * _inv(_D * y * y + 1) + if x2 == 0: + return None if sign else 0 + x = pow(x2, (_P + 3) // 8, _P) + if (x * x - x2) % _P != 0: + x = x * _SQRT_M1 % _P + if (x * x - x2) % _P != 0: + return None + if (x & 1) != sign: + x = _P - x + return x + + +def _decompress(b): + if len(b) != 32: + return None + y = int.from_bytes(b, "little") + sign = y >> 255 + y &= (1 << 255) - 1 + x = _recover_x(y, sign) + if x is None: + return None + return (x, y, 1, x * y % _P) + + +def _compress(pt): + zi = _inv(pt[2]) + x, y = pt[0] * zi % _P, pt[1] * zi % _P + return (y | ((x & 1) << 255)).to_bytes(32, "little") + + +_GY = 4 * _inv(5) % _P +_G = (_recover_x(_GY, 0), _GY, 1, _recover_x(_GY, 0) * _GY % _P) + + +def ed25519_verify(public, msg, sig): + """True when sig is a valid ed25519 signature of msg by the 32-byte key public.""" + if len(public) != 32 or len(sig) != 64: + return False + a = _decompress(public) + r = _decompress(sig[:32]) + if a is None or r is None: + return False + s = int.from_bytes(sig[32:], "little") + if s >= _Q: + return False + h = int.from_bytes(hashlib.sha512(sig[:32] + public + msg).digest(), "little") % _Q + return _equal(_mul(s, _G), _add(r, _mul(h, a))) + + +def ed25519_public_key(seed): + """The public key for a 32-byte seed. Used by the tests to check the + arithmetic against published vectors; verification never needs it.""" + h = hashlib.sha512(seed).digest() + a = int.from_bytes(h[:32], "little") + a &= (1 << 254) - 8 + a |= 1 << 254 + return _compress(_mul(a, _G)) + + +# ---- the release signature -------------------------------------------------- + +def statement(kind, version, manifest_sha256_hex): + """The exact text a release signature covers. The bastion builds the same + text (releaseStatement in client-go/release_sign.go), and so does the + signing tool the key holder runs.""" + return ("privatae-release-signature-v1\n%s\n%s\n%s\n" + % (kind, version, manifest_sha256_hex)).encode("utf-8") + + +def sig_name(body): + """The signature file's name beside version.json: named by the first 16 hex + of the manifest's sha256, so a signature can only pair with its manifest.""" + return "version.json.%s.sig" % hashlib.sha256(body).hexdigest()[:16] + + +def check_signature(body, sig_text, keys=None, kind=KIND): + """Return the manifest (parsed) when sig_text verifies over body against + one of keys, and the index of the key that verified. Raise VerifyError + otherwise; there is no unsigned fallback.""" + try: + m = json.loads(body.decode("utf-8")) + except (UnicodeDecodeError, ValueError): + raise VerifyError("the manifest is not JSON") + ver = m.get("version") if isinstance(m, dict) else None + if not isinstance(ver, str) or not ver or "\n" in ver or "\r" in ver: + raise VerifyError("the manifest names no usable version") + try: + raw = base64.b64decode(sig_text.strip(), validate=True) + except (ValueError, TypeError): + raise VerifyError("the signature is not base64") + if len(raw) != 64: + raise VerifyError("the signature is malformed") + msg = statement(kind, ver, hashlib.sha256(body).hexdigest()) + for i, k in enumerate(RELEASE_KEYS if keys is None else keys): + pub = base64.b64decode(k) + if ed25519_verify(pub, msg, raw): + return m, i + raise VerifyError("the signature does not verify against any pinned release key") + + +# ---- fetching, checking out, rebuilding ------------------------------------- + +def fetch(url): + # A named client: the download address sits behind Cloudflare, whose + # Browser Integrity Check refuses Python's default user agent (error + # 1010, a 403), the signature generic scrapers send. + req = urllib.request.Request(url, headers={"User-Agent": "bastion-agent-verify-release"}) + with urllib.request.urlopen(req, timeout=FETCH_TIMEOUT) as resp: + data = resp.read(FETCH_MAX + 1) + if len(data) > FETCH_MAX: + raise VerifyError("%s is larger than a release manifest can be" % url) + return data + + +def _git(repo, *args, check=True): + return subprocess.run(("git", "-C", repo) + args, check=check, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True) + + +def _commit(repo, ref): + r = _git(repo, "rev-parse", "-q", "--verify", ref + "^{commit}", check=False) + return r.stdout.strip() if r.returncode == 0 else "" + + +def pick_source(repo, version, built_from): + """(label, commit) of the source to rebuild: the tag v, else the + manifest's built_from, else VerifyError.""" + tag = "v" + version + # Fetch the tag if the clone lacks it; a failure (offline, no such tag) + # just leaves the local refs to decide. + _git(repo, "fetch", "-q", "origin", "refs/tags/%s:refs/tags/%s" % (tag, tag), check=False) + c = _commit(repo, "refs/tags/" + tag) + if c: + return "tag " + tag, c + if built_from: + c = _commit(repo, built_from) + if c: + return "built_from " + built_from, c + raise VerifyError("this clone has neither the tag %s nor the commit %s. Check that " + "the public repository has the tag (git fetch --tags origin); if it " + "does not, the release predates tagging and can only be checked from a " + "clone that has its built_from commit" % (tag, built_from or "(none named)")) + + +def rebuild(repo, commit, version, work): + """sha256 of the .pyz that commit's own build.py produces.""" + src = os.path.join(work, "src") + _git(repo, "worktree", "add", "-q", "--detach", src, commit) + try: + with open(os.path.join(src, "bastion_agent", "VERSION"), encoding="utf-8") as fh: + got = fh.read().strip() + if got != version: + raise VerifyError("the source says version %s, the manifest %s" % (got, version)) + dist = os.path.join(work, "dist") + r = subprocess.run((sys.executable, "-m", "bastion_agent.build", "--out", dist), + cwd=src, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True) + if r.returncode != 0: + raise VerifyError("the build failed: %s" % (r.stderr.strip() or r.stdout.strip())) + with open(os.path.join(dist, "bastion-agent-%s.pyz" % version), "rb") as fh: + return hashlib.sha256(fh.read()).hexdigest() + finally: + _git(repo, "worktree", "remove", "--force", src, check=False) + + +def main(argv=None, repo=REPO, keys=None): + ap = argparse.ArgumentParser(prog="bastion_agent.verify_release", + description="Rebuild the published agent and compare it with the release manifest.") + ap.add_argument("--base", default=os.environ.get("MANIFEST_BASE", DEFAULT_BASE), + help="where version.json is served (default %s)" % DEFAULT_BASE) + args = ap.parse_args(argv) + base = args.base.rstrip("/") + work = tempfile.mkdtemp(prefix="agent-verify-") + try: + body = fetch(base + "/version.json") + sig = fetch(base + "/" + sig_name(body)).decode("ascii", "replace") + m, key = check_signature(body, sig, keys) + ver, built_from = m["version"], m.get("built_from") or "" + print("published: v%s built from %s" % (ver, built_from or "(not named)")) + print("signature: ok (release key %d of %d, %s)" + % (key + 1, len(RELEASE_KEYS if keys is None else keys), "primary" if key == 0 else "spare")) + files = m.get("files") if isinstance(m.get("files"), dict) else {} + # Every .pyz the manifest lists: the artifact under its own name, and + # the same bytes under the legacy name while that is still published. + pyz = {n: (e or {}).get("sha256") for n, e in sorted(files.items()) if n.endswith(".pyz")} + if "bastion-agent-%s.pyz" % ver not in pyz: + raise VerifyError("the manifest lists no bastion-agent-%s.pyz" % ver) + label, commit = pick_source(repo, ver, built_from) + print("source: %s (%s)" % (label, commit[:12])) + got = rebuild(repo, commit, ver, work) + bad = 0 + for name, want in pyz.items(): + same = got == want + bad += not same + print("rebuild: %-28s %s" % (name, "matches %s" % got if same + else "DIFFERS (rebuilt %s, published %s)" % (got, want))) + return 1 if bad else 0 + except VerifyError as exc: + print("verify: %s" % exc, file=sys.stderr) + return 1 + except subprocess.CalledProcessError as exc: + print("verify: git %s failed: %s" % (" ".join(exc.cmd[3:]), (exc.stderr or "").strip()), file=sys.stderr) + return 1 + except OSError as exc: # network errors from urllib are OSErrors + print("verify: %s" % exc, file=sys.stderr) + return 1 + finally: + shutil.rmtree(work, ignore_errors=True) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bastion_agent/walk.py b/bastion_agent/walk.py new file mode 100644 index 0000000..859f93f --- /dev/null +++ b/bastion_agent/walk.py @@ -0,0 +1,476 @@ +"""bastion agent `walk` stage — the filesystem + metadata map. + +The cheapest thing a host can plan from: every entry under the manifest's +roots with path, kind, size, mtime, a sniffed MIME type and (for files within +`max_file_bytes`) a SHA-256. Nothing is read beyond an 8 KiB sniff and the +hash stream; nothing is written outside the output directory. + +Portability is a requirement, not a nicety: this runs on the owner's Mac, +Windows box or Linux server. So: stdlib only, no syntax newer than Python +3.8, paths recorded `/`-separated relative to their root regardless of OS, +`~` expanded per-OS, symlinks AND Windows junctions treated alike (recorded, +never followed unless the manifest says so), and every entry sorted so two +runs over the same tree produce byte-identical maps. + +Output (docs/walk.md): `/walk.jsonl` (one JSON object per +entry, sorted by root index then path) and `/walk.summary.json`. Both +are written atomically. A previous walk.jsonl in the same directory is used +as a hash cache: a file whose size and mtime_ns are unchanged keeps its +hash without being re-read. +""" + +import datetime as _dt +import hashlib +import json +import os +import platform +import re +import sys +import tempfile + +from .manifest import AGENT_DIR, signing_digest + +HASH_BUF = 1024 * 1024 +SNIFF_BYTES = 16 +TEXT_PROBE_BYTES = 8192 + +# (prefix, mime) — a small, deterministic magic table for the formats a +# home corpus is made of. Anything else falls back to the extension, then +# to a text/binary probe. Order matters only where prefixes overlap. +MAGIC = ( + (b"%PDF-", "application/pdf"), + (b"\x89PNG\r\n\x1a\n", "image/png"), + (b"\xff\xd8\xff", "image/jpeg"), + (b"GIF87a", "image/gif"), + (b"GIF89a", "image/gif"), + (b"RIFF", "application/x-riff"), # refined below: WEBP / WAV / AVI + (b"PK\x03\x04", "application/zip"), # refined below: docx/xlsx/pptx by ext + (b"\x1f\x8b", "application/gzip"), + (b"BZh", "application/x-bzip2"), + (b"\xfd7zXZ\x00", "application/x-xz"), + (b"7z\xbc\xaf\x27\x1c", "application/x-7z-compressed"), + (b"Rar!\x1a\x07", "application/vnd.rar"), + (b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1", "application/x-ole-storage"), + (b"SQLite format 3\x00", "application/vnd.sqlite3"), + (b"\x7fELF", "application/x-executable"), + (b"MZ", "application/x-dosexec"), + (b"\xca\xfe\xba\xbe", "application/x-mach-binary"), + (b"\xcf\xfa\xed\xfe", "application/x-mach-binary"), + (b"ID3", "audio/mpeg"), + (b"fLaC", "audio/flac"), + (b"OggS", "audio/ogg"), + (b"\x00\x00\x00\x1cftyp", "video/mp4"), + (b"\x00\x00\x00\x18ftyp", "video/mp4"), + (b"\x00\x00\x00\x20ftyp", "video/mp4"), + (b"\x1aE\xdf\xa3", "video/x-matroska"), + (b"{\\rtf", "application/rtf"), +) +ZIP_BY_EXT = { + ".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + ".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ".pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation", + ".odt": "application/vnd.oasis.opendocument.text", + ".ods": "application/vnd.oasis.opendocument.spreadsheet", + ".odp": "application/vnd.oasis.opendocument.presentation", + ".epub": "application/epub+zip", + ".jar": "application/java-archive", + ".apk": "application/vnd.android.package-archive", +} +OLE_BY_EXT = { + ".doc": "application/msword", + ".xls": "application/vnd.ms-excel", + ".ppt": "application/vnd.ms-powerpoint", + ".msg": "application/vnd.ms-outlook", +} +# extension table kept here rather than the `mimetypes` module: that module +# reads the OS registry / /etc/mime.types, so the same file would get a +# different type on a different machine. Determinism wins. +EXT_MIME = { + ".txt": "text/plain", ".md": "text/markdown", ".markdown": "text/markdown", + ".rst": "text/x-rst", ".csv": "text/csv", ".tsv": "text/tab-separated-values", + ".json": "application/json", ".jsonl": "application/x-ndjson", + ".yaml": "application/yaml", ".yml": "application/yaml", ".toml": "application/toml", + ".xml": "application/xml", ".html": "text/html", ".htm": "text/html", + ".css": "text/css", ".js": "text/javascript", ".mjs": "text/javascript", + ".ts": "text/x-typescript", ".tsx": "text/x-typescript", ".jsx": "text/javascript", + ".py": "text/x-python", ".go": "text/x-go", ".rs": "text/x-rust", + ".java": "text/x-java", ".c": "text/x-c", ".h": "text/x-c", ".cpp": "text/x-c++", + ".hpp": "text/x-c++", ".cs": "text/x-csharp", ".rb": "text/x-ruby", + ".php": "text/x-php", ".sh": "text/x-shellscript", ".bash": "text/x-shellscript", + ".ps1": "text/x-powershell", ".bat": "text/x-msdos-batch", ".sql": "application/sql", + ".ini": "text/plain", ".cfg": "text/plain", ".conf": "text/plain", ".log": "text/plain", + ".eml": "message/rfc822", ".ics": "text/calendar", ".vcf": "text/vcard", + ".svg": "image/svg+xml", ".bmp": "image/bmp", ".tif": "image/tiff", ".tiff": "image/tiff", + ".heic": "image/heic", ".webp": "image/webp", + ".mp3": "audio/mpeg", ".wav": "audio/wav", ".m4a": "audio/mp4", ".aac": "audio/aac", + ".mp4": "video/mp4", ".mov": "video/quicktime", ".avi": "video/x-msvideo", + ".mkv": "video/x-matroska", ".webm": "video/webm", + ".pdf": "application/pdf", ".zip": "application/zip", ".gz": "application/gzip", + ".tar": "application/x-tar", ".key": "application/x-pem-file", ".pem": "application/x-pem-file", +} +EXT_MIME.update(ZIP_BY_EXT) +EXT_MIME.update(OLE_BY_EXT) + +KIND_FILE, KIND_DIR, KIND_SYMLINK, KIND_OTHER = "file", "dir", "symlink", "other" +SKIP_TOO_LARGE = "too_large" + +_CASEFOLD = os.name == "nt" + + +class WalkError(Exception): + pass + + +# ── exclude globs ─────────────────────────────────────────────────────────── + +def _glob_to_regex(pat): + """`**` spans segments, `*`/`?` stay inside one, `[...]` passes through.""" + i, n, out = 0, len(pat), [] + while i < n: + c = pat[i] + if c == "*": + if pat[i:i + 3] == "**/": + out.append("(?:.*/)?") + i += 3 + continue + if pat[i:i + 2] == "**": + out.append(".*") + i += 2 + continue + out.append("[^/]*") + elif c == "?": + out.append("[^/]") + elif c == "[": + j = pat.find("]", i + 1) + if j == -1: + out.append(re.escape(c)) + else: + body = pat[i + 1:j] + if body.startswith("!"): + body = "^" + body[1:] + out.append("[" + body.replace("\\", "\\\\") + "]") + i = j + else: + out.append(re.escape(c)) + i += 1 + return "".join(out) + + +class Excluder: + """Compiled `scope.exclude`. A pattern with a `/` matches the whole + root-relative path; one without also matches any single segment (so + `node_modules` or `*.key` prune at any depth, gitignore-style).""" + + def __init__(self, patterns): + flags = re.IGNORECASE if _CASEFOLD else 0 + self._path = [] + self._segment = [] + for p in patterns: + p = p.replace("\\", "/").rstrip("/") + rx = re.compile("^" + _glob_to_regex(p) + "$", flags) + self._path.append(rx) + if "/" not in p: + self._segment.append(rx) + elif p.endswith("/**"): + # `docs/**` and `**/.git/**` prune the directory itself, not + # just what is under it — otherwise the dir entry leaks into + # the map while its contents are excluded + self._path.append(re.compile("^" + _glob_to_regex(p[:-3]) + "$", flags)) + + def matches(self, relpath): + if not relpath: + return False + for rx in self._path: + if rx.match(relpath): + return True + if self._segment: + for seg in relpath.split("/"): + for rx in self._segment: + if rx.match(seg): + return True + return False + + +# ── helpers ───────────────────────────────────────────────────────────────── + +def _utc_iso(ns): + secs, rem = divmod(ns, 1_000_000_000) + dt = _dt.datetime.fromtimestamp(secs, _dt.timezone.utc) + return dt.strftime("%Y-%m-%dT%H:%M:%S") + ".%06dZ" % (rem // 1000) + + +def _relposix(root, path): + rel = os.path.relpath(path, root) + return "" if rel == os.curdir else rel.replace(os.sep, "/") + + +def _is_link(entry): + """Symlink on every OS; on Windows also a junction (3.12+ exposes + is_junction; older Pythons see junctions as directories, in which case + the reparse tag on the lstat result is the tell).""" + try: + if entry.is_symlink(): + return True + except OSError: + return False + is_junction = getattr(entry, "is_junction", None) + if is_junction is not None: + try: + return bool(is_junction()) + except OSError: + return False + if os.name == "nt": + try: + st = entry.stat(follow_symlinks=False) + tag = getattr(st, "st_reparse_tag", 0) + return tag == 0xA0000003 # IO_REPARSE_TAG_MOUNT_POINT + except OSError: + return False + return False + + +def _sniff(path, ext): + """MIME by magic first, extension second, text/binary probe last.""" + try: + with open(path, "rb") as fh: + head = fh.read(TEXT_PROBE_BYTES) + except OSError as exc: + return None, "%s: %s" % (type(exc).__name__, exc.strerror or exc) + for prefix, mime in MAGIC: + if head.startswith(prefix): + if mime == "application/zip": + return ZIP_BY_EXT.get(ext, mime), None + if mime == "application/x-ole-storage": + return OLE_BY_EXT.get(ext, mime), None + if mime == "application/x-riff": + sub = head[8:12] + return {b"WEBP": "image/webp", b"WAVE": "audio/wav", + b"AVI ": "video/x-msvideo"}.get(sub, mime), None + return mime, None + if ext in EXT_MIME: + return EXT_MIME[ext], None + if not head: + return "inode/x-empty", None + if b"\x00" in head: + return "application/octet-stream", None + return "text/plain", None + + +def _sha256(path): + h = hashlib.sha256() + with open(path, "rb") as fh: + while True: + buf = fh.read(HASH_BUF) + if not buf: + break + h.update(buf) + return h.hexdigest() + + +def load_hash_cache(jsonl_path): + """(root, path) -> (size, mtime_ns, sha256) from a previous walk.""" + cache = {} + try: + with open(jsonl_path, encoding="utf-8") as fh: + for line in fh: + try: + rec = json.loads(line) + except ValueError: + continue + if rec.get("kind") == KIND_FILE and rec.get("sha256"): + cache[(rec["root"], rec["path"])] = ( + rec.get("size"), rec.get("mtime_ns"), rec["sha256"]) + except OSError: + pass + return cache + + +def _atomic_write(path, write_fn): + d = os.path.dirname(path) or "." + fd, tmp = tempfile.mkstemp(prefix=".tmp-", dir=d) + try: + with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as fh: + write_fn(fh) + os.replace(tmp, path) + except BaseException: + try: + os.unlink(tmp) + except OSError: + pass + raise + + +# ── the walk ──────────────────────────────────────────────────────────────── + +class Walker: + def __init__(self, manifest, hash_cache=None, do_hash=True): + self.m = manifest + # .agent/ is always excluded: it is the agent's own output, and indexing + # it would feed the catalog back into itself + self.excluder = Excluder(list(manifest.scope.exclude) + [AGENT_DIR]) + # …and on every OS regardless of case: a pre-existing ".Agent" on a + # case-insensitive macOS volume IS the agent dir + self.excluder._segment.append(re.compile("^" + re.escape(AGENT_DIR) + "$", re.IGNORECASE)) + self.cache = hash_cache or {} + self.do_hash = do_hash + self.entries = [] + self.counts = {k: 0 for k in ( + "files", "dirs", "symlinks", "other", "excluded", "errors", + "too_large", "hashed", "hash_reused")} + self.bytes = 0 + + # records --------------------------------------------------------------- + def _rec(self, root_idx, rel, kind, **extra): + rec = {"root": root_idx, "path": rel, "kind": kind} + rec.update(extra) + self.entries.append(rec) + return rec + + def _error(self, root_idx, rel, exc, kind=KIND_OTHER): + self.counts["errors"] += 1 + msg = "%s: %s" % (type(exc).__name__, getattr(exc, "strerror", None) or exc) + return self._rec(root_idx, rel, kind, error=msg) + + # per-entry -------------------------------------------------------------- + def _file(self, root_idx, rel, entry_path, st): + size = st.st_size + mtime_ns = st.st_mtime_ns + ext = os.path.splitext(rel)[1].lower() + mime, err = _sniff(entry_path, ext) + rec = {"size": size, "mtime": _utc_iso(mtime_ns), "mtime_ns": mtime_ns, + "mime": mime, "sha256": None} + if err is not None: + self.counts["errors"] += 1 + rec["error"] = err + self._rec(root_idx, rel, KIND_FILE, **rec) + return + self.counts["files"] += 1 + self.bytes += size + if size > self.m.scope.max_file_bytes: + self.counts["too_large"] += 1 + rec["skip"] = SKIP_TOO_LARGE + elif self.do_hash: + cached = self.cache.get((root_idx, rel)) + if cached and cached[0] == size and cached[1] == mtime_ns: + rec["sha256"] = cached[2] + self.counts["hash_reused"] += 1 + else: + try: + rec["sha256"] = _sha256(entry_path) + self.counts["hashed"] += 1 + except OSError as exc: + self.counts["errors"] += 1 + rec["error"] = "%s: %s" % (type(exc).__name__, exc.strerror or exc) + self._rec(root_idx, rel, KIND_FILE, **rec) + + def _link(self, root_idx, rel, entry_path): + self.counts["symlinks"] += 1 + try: + target = os.readlink(entry_path) + except OSError as exc: + target = None + err = "%s: %s" % (type(exc).__name__, exc.strerror or exc) + self._rec(root_idx, rel, KIND_SYMLINK, target=None, error=err) + self.counts["errors"] += 1 + return + if isinstance(target, bytes): + target = os.fsdecode(target) + self._rec(root_idx, rel, KIND_SYMLINK, target=target.replace(os.sep, "/")) + + # recursion -------------------------------------------------------------- + def _walk_dir(self, root_idx, root, dirpath): + try: + it = os.scandir(dirpath) + except OSError as exc: + self._error(root_idx, _relposix(root, dirpath), exc, KIND_DIR) + return + with it: + try: + entries = list(it) + except OSError as exc: + self._error(root_idx, _relposix(root, dirpath), exc, KIND_DIR) + return + for entry in entries: + rel = _relposix(root, entry.path) + if self.excluder.matches(rel): + self.counts["excluded"] += 1 + continue + follow = self.m.scope.follow_symlinks + if _is_link(entry) and not follow: + self._link(root_idx, rel, entry.path) + continue + try: + is_dir = entry.is_dir(follow_symlinks=follow) + except OSError as exc: + self._error(root_idx, rel, exc) + continue + if is_dir: + self.counts["dirs"] += 1 + try: + st = entry.stat(follow_symlinks=follow) + self._rec(root_idx, rel, KIND_DIR, + mtime=_utc_iso(st.st_mtime_ns), mtime_ns=st.st_mtime_ns) + except OSError as exc: + self._error(root_idx, rel, exc, KIND_DIR) + continue + self._walk_dir(root_idx, root, entry.path) + continue + try: + is_file = entry.is_file(follow_symlinks=follow) + st = entry.stat(follow_symlinks=follow) + except OSError as exc: + self._error(root_idx, rel, exc) + continue + if is_file: + self._file(root_idx, rel, entry.path, st) + else: + self.counts["other"] += 1 # socket, fifo, device + self._rec(root_idx, rel, KIND_OTHER) + + def run(self): + roots = [] + for idx, raw in enumerate(self.m.scope.roots): + resolved = os.path.abspath(os.path.expanduser(raw)) + roots.append({"index": idx, "path": raw, "resolved": resolved}) + if not os.path.isdir(resolved): + self.counts["errors"] += 1 + roots[-1]["error"] = "not a directory or not present on this machine" + continue + self._walk_dir(idx, resolved, resolved) + self.entries.sort(key=lambda r: (r["root"], r["path"])) + return roots + + +def walk(manifest, out_dir, do_hash=True, now=None): + """Run the walk stage for a validated manifest; write walk.jsonl and + walk.summary.json into out_dir. Returns the summary dict.""" + os.makedirs(out_dir, exist_ok=True) + jsonl = os.path.join(out_dir, "walk.jsonl") + summary_path = os.path.join(out_dir, "walk.summary.json") + cache = load_hash_cache(jsonl) if do_hash else {} + w = Walker(manifest, hash_cache=cache, do_hash=do_hash) + roots = w.run() + + def _write_jsonl(fh): + for rec in w.entries: + fh.write(json.dumps(rec, ensure_ascii=False, sort_keys=True)) + fh.write("\n") + + _atomic_write(jsonl, _write_jsonl) + walked_at = now or _dt.datetime.now(_dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + summary = { + "freeagent": 1, + "stage": "walk", + "manifest_id": manifest.id, + "manifest_digest": "sha256:" + signing_digest(manifest.raw), + "walked_at": walked_at, + "platform": {"system": platform.system(), "python": platform.python_version()}, + "roots": roots, + "entries": len(w.entries), + "bytes": w.bytes, + "counts": w.counts, + "hashing": bool(do_hash), + } + _atomic_write(summary_path, lambda fh: fh.write( + json.dumps(summary, ensure_ascii=False, indent=2, sort_keys=True) + "\n")) + return summary diff --git a/bastion_agent/x509gate.py b/bastion_agent/x509gate.py new file mode 100644 index 0000000..a38eb90 --- /dev/null +++ b/bastion_agent/x509gate.py @@ -0,0 +1,423 @@ +"""The manifest gate's certificate and signature checks, in the standard library. + +The gate used to shell out to `openssl` for three things: read the signing +cert's CN, check that the cert chains to FREEAGENT_CA_ROOT, and verify the +ECDSA P-256 / SHA-256 signature over the manifest payload. Windows ships no +`openssl`, so on a Windows machine the agent refused every manifest and never +ran — its bastion reported `missing-openssl` and the user's context store was +never indexed. + +This module does exactly those three checks with nothing but the standard +library, so the gate runs wherever Python does: + + * a minimal DER reader for the X.509 fields the gate needs (subject and + issuer names, validity, the EC public key, basicConstraints, and the + signed TBS bytes with their signature); + * ECDSA verification over NIST P-256 (verification only — no secret is ever + held here, so constant-time arithmetic is not a concern). + +Only what the managed agent's certificates use is accepted: EC P-256 keys and +ecdsa-with-SHA256 signatures. Anything else is refused by name, never guessed. +""" + +import base64 +import datetime as _dt +import hashlib +import re + +# ── P-256 (FIPS 186-4, D.1.2.3) ───────────────────────────────────────────── +_P = 0xffffffff00000001000000000000000000000000ffffffffffffffffffffffff +_A = _P - 3 +_B = 0x5ac635d8aa3a93e7b3ebbd55769886bc651d06b0cc53b0f63bce3c3e27d2604b +_N = 0xffffffff00000000ffffffffffffffffbce6faada7179e84f3b9cac2fc632551 +_G = (0x6b17d1f2e12c4247f8bce6e563a440f277037d812deb33a0f4a13945d898c296, + 0x4fe342e2fe1a7f9b8ee7eb4a7c0f9e162bce33576b315ececbb6406837bf51f5) + +# OIDs, as their DER content bytes. +_OID_CN = bytes.fromhex("550403") # 2.5.4.3 +_OID_EC_PUBKEY = bytes.fromhex("2a8648ce3d0201") # 1.2.840.10045.2.1 +_OID_P256 = bytes.fromhex("2a8648ce3d030107") # 1.2.840.10045.3.1.7 +_OID_ECDSA_SHA256 = bytes.fromhex("2a8648ce3d040302") # 1.2.840.10045.4.3.2 +_OID_BASIC_CONSTRAINTS = bytes.fromhex("551d13") # 2.5.29.19 +_OID_KEY_USAGE = bytes.fromhex("551d0f") # 2.5.29.15 +# Critical extensions the gate may meet and still decide correctly: those it +# reads (above) and the ones Go's x509 writes on the managed install's cert. +_UNDERSTOOD_EXTENSIONS = frozenset({ + _OID_BASIC_CONSTRAINTS, _OID_KEY_USAGE, + bytes.fromhex("551d25"), # extKeyUsage + bytes.fromhex("551d11"), # subjectAltName + bytes.fromhex("551d0e"), # subjectKeyIdentifier + bytes.fromhex("551d23"), # authorityKeyIdentifier +}) +_KU_KEY_CERT_SIGN = 0x0400 # keyUsage bit 5, as a big-endian 16-bit field + + +# The directory-string types a name value may carry, and how each decodes. +_NAME_STRING_CODECS = {0x0c: "utf-8", 0x13: "ascii", 0x16: "ascii", + 0x14: "latin-1", 0x1e: "utf-16-be"} + + +class CertError(Exception): + """A certificate or signature the gate cannot accept; the message says why.""" + + +# ── DER ───────────────────────────────────────────────────────────────────── + +def _tlv(buf, off): + """(tag, content_start, content_end, next_off) of the element at `off`.""" + if off + 2 > len(buf): + raise CertError("truncated DER") + tag = buf[off] + ln = buf[off + 1] + off += 2 + if ln & 0x80: + n = ln & 0x7f + if n == 0 or n > 4 or off + n > len(buf): + raise CertError("unsupported DER length") + ln = int.from_bytes(buf[off:off + n], "big") + # DER: the long form only for lengths >= 128, with no leading zero + # byte — the one encoding a length has, as openssl requires. + if ln < 0x80 or buf[off] == 0: + raise CertError("non-minimal DER length") + off += n + end = off + ln + if end > len(buf): + raise CertError("truncated DER") + return tag, off, end, end + + +def _children(buf, start, end): + out = [] + off = start + while off < end: + tag, cs, ce, off = _tlv(buf, off) + out.append((tag, cs, ce)) + return out + + +def _expect(el, tag, what): + if el[0] != tag: + raise CertError("%s: unexpected DER tag 0x%02x" % (what, el[0])) + return el + + +def _name_parts(buf, start, end): + """[(oid_bytes, value_str)] of an X.501 Name (SEQUENCE OF SET OF ATV).""" + parts = [] + for rdn in _children(buf, start, end): + _expect(rdn, 0x31, "name RDN") + for atv in _children(buf, rdn[1], rdn[2]): + _expect(atv, 0x30, "name attribute") + kids = _children(buf, atv[1], atv[2]) + if len(kids) != 2 or kids[0][0] != 0x06: + raise CertError("malformed name attribute") + raw = bytes(buf[kids[1][1]:kids[1][2]]) + codec = _NAME_STRING_CODECS.get(kids[1][0]) + if codec is None: + raise CertError("name attribute has string type 0x%02x" % kids[1][0]) + try: + val = raw.decode(codec) + except UnicodeDecodeError: + raise CertError("name attribute is not text") + parts.append((bytes(buf[kids[0][1]:kids[0][2]]), val)) + return parts + + +def _time(buf, el): + raw = buf[el[1]:el[2]].decode("ascii", "replace") + if el[0] == 0x17: # UTCTime YYMMDDHHMMSSZ + m = re.fullmatch(r"(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})Z", raw) + if not m: + raise CertError("unsupported UTCTime %r" % raw) + y = int(m.group(1)) + y += 2000 if y < 50 else 1900 + rest = [int(g) for g in m.groups()[1:]] + elif el[0] == 0x18: # GeneralizedTime YYYYMMDDHHMMSSZ + m = re.fullmatch(r"(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})Z", raw) + if not m: + raise CertError("unsupported GeneralizedTime %r" % raw) + y = int(m.group(1)) + rest = [int(g) for g in m.groups()[1:]] + else: + raise CertError("validity time has tag 0x%02x" % el[0]) + return _dt.datetime(y, *rest, tzinfo=_dt.timezone.utc) + + +class Cert: + """The fields of one X.509 certificate the gate reads.""" + + def __init__(self, der): + self.der = bytes(der) + buf = self.der + top = _tlv(buf, 0) + if top[0] != 0x30 or top[3] != len(buf): + raise CertError("not a single DER certificate") + parts = _children(buf, top[1], top[2]) + if len(parts) != 3: + raise CertError("a certificate is exactly tbsCertificate, algorithm, signature") + tbs, sig_alg, sig_val = parts + _expect(tbs, 0x30, "tbsCertificate") + # The signed bytes are the WHOLE tbsCertificate element, header included. + tbs_hdr_start = top[1] + self.tbs_bytes = buf[tbs_hdr_start:tbs[2]] + self.sig_alg = self._alg_oid(buf, sig_alg) + _expect(sig_val, 0x03, "signatureValue") + if sig_val[2] - sig_val[1] < 2 or buf[sig_val[1]] != 0: + raise CertError("signatureValue is empty or has unused bits") + self.signature = buf[sig_val[1] + 1:sig_val[2]] + + f = _children(buf, tbs[1], tbs[2]) + i = 0 + if f and f[0][0] == 0xa0: # [0] EXPLICIT version + i = 1 + # serial, signature alg, issuer, validity, subject, spki, then optional + # [1]/[2] unique ids and [3] extensions. + if len(f) < i + 6: + raise CertError("tbsCertificate is missing fields") + self.tbs_sig_alg = self._alg_oid(buf, f[i + 1]) + issuer, validity, subject, spki = f[i + 2], f[i + 3], f[i + 4], f[i + 5] + _expect(issuer, 0x30, "issuer") + _expect(subject, 0x30, "subject") + self.issuer_der = buf[issuer[1]:issuer[2]] + self.subject_der = buf[subject[1]:subject[2]] + self.subject = _name_parts(buf, subject[1], subject[2]) + v = _children(buf, validity[1], validity[2]) + if len(v) != 2: + raise CertError("malformed validity") + self.not_before, self.not_after = _time(buf, v[0]), _time(buf, v[1]) + self.public_key = self._ec_point(buf, spki) + self.is_ca = False + self.key_usage = None # None = no keyUsage extension + seen_ext = set() + for ext_wrap in f[i + 6:]: + if ext_wrap[0] != 0xa3: + continue + exts = _children(buf, ext_wrap[1], ext_wrap[2]) + if len(exts) != 1: + raise CertError("malformed extensions") + for ext in _children(buf, exts[0][1], exts[0][2]): + k = _children(buf, ext[1], ext[2]) + if len(k) not in (2, 3) or k[0][0] != 0x06 or k[-1][0] != 0x04: + raise CertError("malformed extension") + oid = bytes(buf[k[0][1]:k[0][2]]) + if oid in seen_ext: + raise CertError("duplicate extension") + seen_ext.add(oid) + critical = len(k) == 3 and k[1][0] == 0x01 and buf[k[1][1]:k[1][2]] != b"\x00" + octets = k[-1] + if oid == _OID_BASIC_CONSTRAINTS: + bc = _tlv(buf, octets[1]) + inner = _children(buf, bc[1], bc[2]) + self.is_ca = bool(inner and inner[0][0] == 0x01 + and buf[inner[0][1]:inner[0][2]] != b"\x00") + elif oid == _OID_KEY_USAGE: + ku = _tlv(buf, octets[1]) + bits_ = bytes(buf[ku[1]:ku[2]]) + self.key_usage = int.from_bytes(bits_[1:3].ljust(2, b"\x00"), "big") if bits_ else 0 + elif critical and oid not in _UNDERSTOOD_EXTENSIONS: + # openssl refuses a certificate carrying a critical + # extension it does not understand; so does the gate. + raise CertError("certificate has an unknown critical extension") + + @staticmethod + def _alg_oid(buf, el): + _expect(el, 0x30, "algorithm") + k = _children(buf, el[1], el[2]) + if not k or k[0][0] != 0x06: + raise CertError("malformed algorithm identifier") + return bytes(buf[k[0][1]:k[0][2]]) + + @staticmethod + def _ec_point(buf, spki): + _expect(spki, 0x30, "subjectPublicKeyInfo") + alg, bits = (_children(buf, spki[1], spki[2]) + [None, None])[:2] + if bits is None: + raise CertError("malformed subjectPublicKeyInfo") + a = _children(buf, alg[1], alg[2]) + if (len(a) != 2 or buf[a[0][1]:a[0][2]] != _OID_EC_PUBKEY + or buf[a[1][1]:a[1][2]] != _OID_P256): + raise CertError("signing key is not an EC P-256 key — the gate verifies " + "ecdsa-p256-sha256 only") + _expect(bits, 0x03, "public key") + if bits[2] - bits[1] < 1 or buf[bits[1]] != 0: + raise CertError("public key BITSTRING is empty or has unused bits") + pt = buf[bits[1] + 1:bits[2]] + if len(pt) != 65 or pt[0] != 0x04: + raise CertError("EC public key is not an uncompressed P-256 point") + q = (int.from_bytes(pt[1:33], "big"), int.from_bytes(pt[33:], "big")) + if not _on_curve(q): + raise CertError("EC public key is not on P-256") + return q + + def cn(self): + cns = [v for oid, v in self.subject if oid == _OID_CN] + if len(cns) != 1: + # exactly one CN: a second one is how a cert could carry the + # owner's name next to its real one + raise CertError("signing cert must carry exactly one CN, has %d" % len(cns)) + return cns[0] + + def valid_at(self, when): + return self.not_before <= when <= self.not_after + + +def load_pem_certs(text): + """Every CERTIFICATE block in a PEM text, as Cert objects (in order).""" + if isinstance(text, bytes): + text = text.decode("ascii", "replace") + blocks = re.findall(r"-----BEGIN CERTIFICATE-----(.*?)-----END CERTIFICATE-----", text, re.S) + if not blocks: + raise CertError("no PEM certificate found") + out = [] + for b in blocks: + try: + out.append(Cert(base64.b64decode("".join(b.split()), validate=True))) + except CertError: + raise + except (ValueError, TypeError, IndexError, OverflowError) as exc: + raise CertError("unreadable PEM certificate: %s" % exc) + return out + + +def load_root_file(text): + """The trust anchors in a CA file: every certificate the gate can use. + One the gate cannot use (another key type, say) is skipped, not fatal — + openssl would skip it too; an empty result is refused.""" + if isinstance(text, bytes): + text = text.decode("ascii", "replace") + blocks = re.findall(r"-----BEGIN CERTIFICATE-----(.*?)-----END CERTIFICATE-----", text, re.S) + roots = [] + for b in blocks: + try: + roots.append(Cert(base64.b64decode("".join(b.split()), validate=True))) + except (CertError, ValueError, TypeError, IndexError, OverflowError): + continue + if not roots: + raise CertError("FREEAGENT_CA_ROOT holds no usable P-256 certificate") + return roots + + +def load_cert_file(path): + with open(path, "rb") as fh: + certs = load_pem_certs(fh.read()) + return certs[0] + + +# ── P-256 arithmetic (affine; verification only) ──────────────────────────── + +def _on_curve(pt): + x, y = pt + if not (0 <= x < _P and 0 <= y < _P): + return False + return (y * y - (x * x * x + _A * x + _B)) % _P == 0 + + +def _add(p1, p2): + if p1 is None: + return p2 + if p2 is None: + return p1 + x1, y1 = p1 + x2, y2 = p2 + if x1 == x2: + if (y1 + y2) % _P == 0: + return None + lam = (3 * x1 * x1 + _A) * pow(2 * y1, -1, _P) % _P + else: + lam = (y2 - y1) * pow(x2 - x1, -1, _P) % _P + x3 = (lam * lam - x1 - x2) % _P + return (x3, (lam * (x1 - x3) - y1) % _P) + + +def _mul(k, pt): + acc = None + while k: + if k & 1: + acc = _add(acc, pt) + pt = _add(pt, pt) + k >>= 1 + return acc + + +def _der_sig(sig): + top = _tlv(sig, 0) + if top[0] != 0x30 or top[3] != len(sig): + raise CertError("signature is not a DER ECDSA signature") + k = _children(sig, top[1], top[2]) + if len(k) != 2 or k[0][0] != 0x02 or k[1][0] != 0x02: + raise CertError("signature is not a DER ECDSA signature") + vals = [] + for _, cs, ce in k: + raw = sig[cs:ce] + # DER INTEGER: non-empty, positive here, and minimal — no leading + # zero byte unless the next byte's top bit needs it. openssl refuses + # a padded signature; so does the gate (one encoding per signature). + if not raw or raw[0] & 0x80 or (len(raw) > 1 and raw[0] == 0 and not raw[1] & 0x80): + raise CertError("signature integers are not minimal DER") + vals.append(int.from_bytes(raw, "big")) + return vals[0], vals[1] + + +def ecdsa_p256_sha256_verify(pub, message, der_signature): + """True iff `der_signature` is a valid ECDSA P-256 / SHA-256 signature by + `pub` over `message`. Malformed input is simply False.""" + try: + r, s = _der_sig(bytes(der_signature)) + except CertError: + return False + if not (1 <= r < _N and 1 <= s < _N) or not _on_curve(pub): + return False + e = int.from_bytes(hashlib.sha256(message).digest(), "big") + w = pow(s, -1, _N) + pt = _add(_mul(e * w % _N, _G), _mul(r * w % _N, pub)) + return pt is not None and pt[0] % _N == r + + +# ── the chain ─────────────────────────────────────────────────────────────── + +def _signed_by(child, issuer): + if child.sig_alg != _OID_ECDSA_SHA256 or child.tbs_sig_alg != _OID_ECDSA_SHA256: + raise CertError("certificate is not signed with ecdsa-with-SHA256") + return ecdsa_p256_sha256_verify(issuer.public_key, child.tbs_bytes, child.signature) + + +def verify_chain(leaf, roots, now=None): + """Raise CertError unless `leaf` is one of `roots` or is signed directly by + one of them. For the managed install — whose cert IS its root — this is + the answer `openssl verify -CAfile root leaf` gave. For a separate issuer + it enforces validity, CA basicConstraints and keyUsage keyCertSign, but it + treats every cert in the file as a trust anchor and does not build longer + chains or check pathLen / AKID: a one-level PKI only.""" + now = now or _dt.datetime.now(_dt.timezone.utc) + if not leaf.valid_at(now): + raise CertError("signing cert is outside its validity window (%s — %s)" + % (leaf.not_before.isoformat(), leaf.not_after.isoformat())) + for root in roots: + if root.der == leaf.der: + # The managed install's cert is its own root. A trust anchor is + # trusted as given; its self-signature is still required to hold. + if not _signed_by(leaf, leaf): + raise CertError("self-signed root does not verify") + return root + why = "signing cert does not chain to FREEAGENT_CA_ROOT" + for root in roots: + if root.subject_der != leaf.issuer_der: + continue + # Every same-named root is tried: a name is not an identity. + if not root.valid_at(now): + why = "the CA root is outside its validity window" + continue + if not root.is_ca: + why = "the issuing certificate is not a CA" + continue + if root.key_usage is not None and not root.key_usage & _KU_KEY_CERT_SIGN: + why = "the issuing certificate may not sign certificates (keyUsage)" + continue + try: + if _signed_by(leaf, root): + return root + except CertError as exc: + why = str(exc) + continue + why = "signing cert's signature does not verify against the CA root" + raise CertError(why) diff --git a/docs/catalog.md b/docs/catalog.md new file mode 100644 index 0000000..b634a1d --- /dev/null +++ b/docs/catalog.md @@ -0,0 +1,102 @@ +# bastion agent `catalog` stage: the curated dataset + +The catalog is the small, structured thing a host reasons over and the thing +`seal` ships: one card per file and one per directory, built only from the +sanitized text. It is `bastion_agent/catalog.py`, standard library only. + +`python3 -m bastion_agent catalog [--out DIR] [--require-signed]` +needs the walk, extract and sanitize outputs, and reads `chunks.jsonl` if it +is there. It writes: + +- **`catalog.jsonl`**: one card per file the walk found, sorted by + `(root, path)`. +- **`catalog.dirs.jsonl`**: one card per directory that holds files (every + ancestor of every file; the root is `""`), sorted the same way. +- **`catalog.summary.json`**: counts, the summarizer that ran (or `null`), + `stopped`, and `bytes_catalog` (the length of all the cards, in + characters). + +## The file card + +| Field | From | Notes | +|---|---|---| +| `root`, `path`, `name`, `kind`, `size`, `mtime`, `mime`, `sha256` | walk | `path` and `name` after the name mask (below). | +| `text` | extract | `ok`, `empty`, `unsupported`, `error`, `skipped` or `not_extracted`. | +| `extractor` | extract | Which reader, e.g. `pdftotext`, `pdf-builtin`, `docx`, `text:utf-8`. | +| `sanitized` | sanitize | `ok`, `skipped`, `error`, `not_run`, or `null` when the sanitize stage has no record for it. | +| `title` | **sanitized text** | The first line of at least three characters, list and heading markers stripped, at most 120 characters. | +| `head` | **sanitized text** | The first 300 characters, whitespace collapsed. | +| `chars` | sanitized text | Length. | +| `keywords` | **sanitized text** | The top 8 by TF-IDF across the catalog: words of three or more letters, stopwords and placeholders removed, ties broken alphabetically (so the result is deterministic). | +| `entities` | sanitize | Token counts per type minted for this file, e.g. `{"EMAIL": 2, "PERSON": 1}`. Counts, never values. | +| `chunks` | chunk | How many windows the document produced (0 without the chunk stage). | +| `summary` | summarizer | Present only when the manifest asks for file summaries; `null` when there was no text. | +| `v` | | The card format version. | + +Only files with sanitized text get the text fields. A file without text still +gets a card: the host can see that it exists, its type and size, and why it +has no text. + +## The directory card + +`files`, `with_text`, `bytes`, `latest` (newest mtime), `types` (the top 6 +MIME types), `keywords` (the top 12 over the merged term counts), `entities` +(summed over the files), and `summary` when asked for. + +## What never enters the catalog + +- **Values.** Every field derived from text comes from the sanitized store. A + test searches the whole catalog for the fixture's personal data and fails on + any hit. +- **Placeholders as keywords.** `[PII_*_n]` tokens are removed before terms + are counted. +- **Anchors in names.** File and directory names are plaintext by nature, so + every segment of every path (parent directories included) goes through the + rules pass; an e-mail address, phone number, card or key in a name is masked + to `[PII_TYPE]`. The masking is conservative: the e-mail rule spans dots and + hyphens, so `invoice-m@example.com.pdf` becomes `[PII_EMAIL]`; `mime` and + `size` still say what it was. The summary counts masked segments as + `names_masked`. **A name like `Marisol_medical.pdf` is not caught**: that is + contextual personal data, the same limit as the rules-only `flag` mode. + +## The summarizer (a sensor) + +A manifest section `summarize: {model, per: [file, directory]}` turns it on +(it needs `summarize` in the mask). The model is reached through +`SUMMARIZE_URL` (default `http://127.0.0.1:11434`, a local Ollama), +`SUMMARIZE_API` (`ollama` or `openai`; unset means try Ollama's `/api/chat`, +then `/v1/chat/completions`) and `SUMMARIZE_TIMEOUT` (300 s): the +environment, never the manifest. + +It sees only sanitized text (the first 12,000 characters), is told that +placeholders are redactions to keep as they are, and runs at temperature 0. +Its reply is cleaned (`` blocks and surrounding quotes removed, +whitespace collapsed, at most 600 characters) into the `summary` string, the +only field it can touch. A directory's summary is built from its direct +children's summaries or heads (at most 60, 200 characters each). Summaries are +cached in `catalog.cache.json` by document, model and version, so a re-run +costs nothing for unchanged documents. + +The first failure stops summarizing: every card is still written, one card +carries `summary_error`, the summary says `stopped`, and the exit code is 1. + +## Tests + +`python3 -m unittest bastion_agent.test_catalog` covers title, head and +keywords and their determinism; name masking (e-mail, phone, plain names, +parent directories); summary cleaning; the whole stage with and without a +summarizer (card fields, entities, keywords free of placeholders, a card for +a file with no text, directory totals including the root, masked names +counted); the catalog containing none of the fixture's personal data; file and +directory summaries with the cache reused on the second run and the model +seeing only sanitized text; OpenAI-style detection; a summarizer failure +stopping after one call with every card still written; an unreachable +summarizer; and the guard that the earlier stages ran first. + +The stopword list is for prose: on source code, identifiers such as `self` +and `def` count as keywords. + +## Not in this stage + +Sealing (the next stage: the catalog encrypted to the owner and the +manifest's `seal_to` recipients) and any rendering of the cards on the host. diff --git a/docs/embed.md b/docs/embed.md new file mode 100644 index 0000000..92bbc3f --- /dev/null +++ b/docs/embed.md @@ -0,0 +1,85 @@ +# bastion agent `chunk` and `embed` stages, and `search` + +`chunk` splits the sanitized text into overlapping windows, `embed` turns each +window into a vector with a model on this machine, and `search` answers a +query against those vectors. They are `bastion_agent/embed.py`, standard +library only. + +## chunk + +`python3 -m bastion_agent chunk [--out DIR] [--require-signed]` +reads the **sanitized** store (never the extract store) and writes +`chunks.jsonl`: one row per window (`chunk_id`, `doc` (the sanitized text's +sha256), `idx`, `start` and `end` character offsets, `chars`, `tokens`, `v`), +sorted by `(doc, idx)`, plus `chunk.summary.json`. + +- Windows are `embed.chunk.size_tokens` long and advance by + `size - overlap` tokens; the last window may be shorter. With any overlap, + every character of the text is inside at least one window (with + `overlap_tokens: 0` the whitespace between two windows may fall outside + both). +- A "token" is a fixed regular-expression unit (a word or a punctuation mark), + not a model's tokenizer. The standard library has none, and the same text + must chunk the same way on every machine; the count is close enough to model + tokens to size windows by. +- `chunk_id = sha256(document sha256, NUL, window text)`. An unchanged + document keeps its chunk ids, and so its vectors, from one sweep to the + next. A changed document has a new sha256, so all of its windows get new ids + and are embedded again. Identical windows inside one document (a repeated + paragraph) share an id and are embedded once. +- Duplicate documents (the same sanitized content at several paths) are + chunked once. + +## embed + +`python3 -m bastion_agent embed [--out DIR] [--require-signed]` +writes `vectors.db` (SQLite) and `embed.summary.json`. + +**The endpoint is a sensor.** It comes from the environment, never the +manifest: `EMBED_URL` (default `http://127.0.0.1:11434`, a local Ollama), +`EMBED_API` (`ollama` or `openai`; unset means detect it with the probe), +`EMBED_BATCH` (32), `EMBED_TIMEOUT` (300 s). The manifest pins the **model +name and dimension**. Two wire formats: Ollama's `POST /api/embed` and the +OpenAI-style `POST /v1/embeddings` (llama.cpp server, TEI). + +**The probe is the gate.** Before the first real call the stage embeds a fixed +probe string. If the served dimension is not the manifest's, the stage stops +before opening the store, so a wrong or mislabelled model never writes a +vector. An unreachable endpoint stops it the same way. + +**Storage** (`vectors.db`): + +| Table | Role | +|---|---| +| `chunks(chunk_id PK, doc, idx, start, end, text)` | This sweep's windows, rebuilt on every run. Holds the sanitized window text, so a search can return it. | +| `vectors(chunk_id, model, dim, vec BLOB float32, created)`, PK `(chunk_id, model)` | Kept across runs; rows are only ever added. | + +Switching models **adds** rows for the new model; the old model's rows stay. +Vectors whose chunk no longer exists in this sweep also stay, and are counted +as `stale_vectors` in the summary: nothing is thrown away, so going back to an +earlier model or document costs nothing. A re-run embeds only chunk ids that +have no vector for the manifest's model (the rest are `reused`). + +## search + +`python3 -m bastion_agent search "" [-k N] [--out DIR]` +embeds the query through the same endpoint, scores cosine similarity against +every vector of the manifest's model, and prints the top k (default 5) +sanitized windows with their score, document id and index. It is a linear scan +in pure Python, which is fine to tens of thousands of chunks; an index +(HNSW or sqlite-vec) is the next step for a folder that outgrows it. The +query endpoint (`serve`, [serve.md](serve.md)) uses the same search. + +## Tests + +`python3 -m unittest bastion_agent.test_embed` runs against a loopback stub +that speaks both wire formats with a deterministic bag-of-words embedding: +window coverage, overlap and step; empty and short inputs; content-addressed +ids; the float32 packing and cosine edge cases; the chunk stage (duplicate +documents, sort order, determinism, the sanitize-first guard); the probe +refusing a wrong dimension before the store exists; an unreachable endpoint +failing closed; batching; reuse on a re-run; a second model adding rows while +the first model's rows stay; stale vectors kept and counted after a document +changes; OpenAI-style detection and a forced wrong API refused; and search +ranking the right document, with an empty result for a model that has no +vectors. diff --git a/docs/extract.md b/docs/extract.md new file mode 100644 index 0000000..91e7824 --- /dev/null +++ b/docs/extract.md @@ -0,0 +1,138 @@ +# bastion agent `extract` stage: text out of the walked files + +The extract stage turns every walked file it can read into plain text, with +built-in readers or with local tools when they are installed. It is +`bastion_agent/extract.py` and `bastion_agent/pdftext.py`, standard library +only. + +## What it produces + +`python3 -m bastion_agent extract [--out DIR] [--no-external] [--require-signed]` +reads `walk.jsonl` and `walk.summary.json` from `DIR` (the walk must have run; +the stage refuses otherwise) and writes: + +- **`extract//.txt`**: the text store, one file per distinct + content hash, UTF-8, normalised (CRLF and lone CR become LF, NUL removed, + NFC). Content addressing is the cache: two copies of a document are + extracted once, and an unchanged file costs nothing on the next sweep. +- **`extract.jsonl`**: one record per walked file, sorted by `(root, path)`: + `sha256`, `mime`, `status`, `extractor`, `chars`, `error`, `v` (the + extractor version), `reused` when a prior verdict was kept, and on + `unsupported` records the `tools` that were present, so the verdict is + retried when a tool appears. +- **`extract.summary.json`**: counts, total characters, the tools found, the + extractor version, the manifest digest, and the manifest digest the walk + recorded, which ties the walk map and the text store to one manifest. + +`DIR` defaults as for the walk (`/.agent/` for one root). This text +store is the plaintext of every file in scope, and it never leaves the machine; +see [runtime.md](runtime.md#where-an-agent-lives-rootagent). + +| `status` | Meaning | +|---|---| +| `ok` | Text written to the store. | +| `empty` | The reader ran and found no text (an empty file, a document with only pictures). | +| `unsupported` | No reader for this type on this machine; `error` names what would handle it. | +| `error` | The reader failed (a corrupt zip, bad XML, a tool failure); the stage continues. | +| `skipped` | The walk has no hash for it (too large, a read error, or a `--no-hash` walk). | + +A record whose earlier verdict was kept carries `reused: true` and is counted +under `reused` in the summary. + +Exit code: `0` clean; `1` the manifest is invalid, `extract` is not in its +stages, or there is no walk map; `2` completed with `error` records. + +## Readers + +Built in, no libraries: + +| Type | How | +|---|---| +| `text/*`, JSON, YAML, TOML, XML, SQL, SVG, PEM, code | Decoded, and the encoding recorded (`text:utf-8`, `text:cp1252`, …): a byte-order mark first, then BOM-less UTF-16 by the share of NUL bytes, then strict UTF-8, then cp1252. | +| `text/html` | `html.parser`: scripts, styles, `noscript` and `template` dropped; block tags become line breaks; entities resolved. | +| CSV / TSV | `csv` module; each row written as tab-separated values. | +| `message/rfc822` | `email` (modern policy): From/To/Cc/Date/Subject lines, then the plain body, or the HTML body through the HTML reader. | +| RTF | A small reader of its own: control words, `\'hh` (cp1252), `\uN` (one fallback character skipped), and the groups that hold no body text (`fonttbl`, `colortbl`, `stylesheet`, `info`, `pict`, `object`, `header`, `footer`, and others) discarded. | +| DOCX | Paragraphs of `word/document.xml` (runs, tabs, breaks, text boxes after the paragraph that holds them), plus headers and footers. | +| XLSX | Shared strings, inline strings and values, one tab-separated line per row, a `# sheetN` line per sheet; empty rows dropped. | +| PPTX | Every paragraph on every slide, slides in numeric order, a `# slideN` line per slide. | +| ODT / ODS / ODP | `content.xml`: paragraphs and headings; table rows tab-separated. | +| PDF | **`pdftotext -layout -enc UTF-8` when poppler is on PATH.** Otherwise the built-in reader (below). | +| Images (PNG, JPEG, TIFF, BMP, GIF, WebP) | **`tesseract` when it is on PATH**; otherwise `unsupported`. | + +Anything else (archives, executables, audio, video, unknown binaries) is +`unsupported`, with the MIME type named. + +### The built-in PDF reader + +Without poppler (on Windows, usually) PDFs are read by `pdftext.py`, which +walks the document far enough for text: the object table (a later definition +wins, so incremental updates are respected) and compressed object streams; +FlateDecode streams (a stream with any other filter is skipped); the page tree +in order with inherited resources; fonts through their `/ToUnicode` maps +(Type0 / Identity-H fonts, as browsers print them), or WinAnsi / MacRoman for +simple fonts without one; glyph widths, so a word placed by position still +gets its space; the text operators, and form XObjects. Pages are separated by +a form feed. + +Encrypted files are refused. Output below 90% printable characters is refused +too, so a font without a usable map comes back as `unsupported` (`extractor: +pdf-builtin`, `needs pdftotext (poppler) on this machine`), never as byte salad +in the catalog. + +It reads arbitrary files, so it reads on a budget per document: 60 s of wall +time, 256 MiB of decoded streams (64 MiB per stream), 5 million content +operators, 2,000 form invocations at most 6 deep, 200,000 CMap entries, +20 million tokens and 1 GiB lexed. Reaching a limit stops the read with the +pages read so far. Nothing in it raises out to the stage. + +## Rules the stage enforces + +- **External tools are sensors, not dependencies.** They are looked up on + PATH at run time, named in every record they produced (`extractor: + pdftotext`), and their absence is a recorded gap, never a failure. + `--no-external` (in a sweep, `FREEAGENT_NO_EXTERNAL=1`) uses the built-in + readers only. A manifest cannot name a tool or an address. +- **Nothing is written outside the output directory**, and every file is + written atomically. +- **Bounded reads.** One zip member is read to at most 64 MiB decompressed + (`ZIP_MEMBER_CAP`), so a hostile office file cannot balloon memory. Office + documents are parsed as a stream, each finished paragraph or row dropped + once read, and capped per document: 256 MiB of XML, 16 Mi characters of + text, 60 s, and 200,000 parts in one paragraph or row. Reaching a cap keeps + the text read so far and ends it with a line naming the cap. XML with a DTD + or an entity declaration is refused (Office files never carry one, and it is + how entity expansion gets in), as is XML that is neither UTF-8 nor UTF-16. + Tool calls time out at 300 s. +- **Errors are entries.** A corrupt document is one `error` record; the stage + finishes and reports the count. +- **Retry only when something changed.** An `unsupported` verdict is kept + while the machine's tool set is the same; install `pdftotext` and the next + sweep tries exactly those files again. `error` records are retried on every + sweep. Every record carries the extractor version (`v`, `EXTRACT_VERSION`, + now 2); raising it after a reader changes redoes every cached verdict and + text on the next sweep. + +## Tests + +`python3 -m unittest bastion_agent.test_extract bastion_agent.test_pdftext`. +`test_extract` runs every reader against a generated fixture (docx, xlsx, +pptx and odt built with `zipfile`, plain and Flate PDFs, RTF with hex and +Unicode escapes, plain and HTML e-mail, encodings), the Office streaming caps, +DTD refusal and UTF-16 input, the zip member cap, an oversized CSV field, the +whole stage over a mixed tree (statuses, counts, duplicates), reuse across +sweeps and its invalidation when content or the extractor version changes, +tool discovery and failure (mocked), and the walk-first guard. `test_pdftext` +builds browser-shaped PDFs with a small writer (Type0 fonts with ToUnicode +maps, per-run positioning, ligatures, inherited resources, form XObjects, +object streams) and hostile ones (reference cycles, deep nesting, malformed +tables and maps), and checks each budget. + +The external-tool paths are tested with mocks; a real scanned PDF or image +through `pdftotext` or `tesseract` is not part of the suite. + +## Not in this stage + +Sanitizing (the next stage: the text store goes through the PII sterilizer), +chunking, and change detection beyond the content hash. Audio and video are not +extracted. diff --git a/docs/manifest.md b/docs/manifest.md new file mode 100644 index 0000000..033d1a7 --- /dev/null +++ b/docs/manifest.md @@ -0,0 +1,291 @@ +# bastion agent job manifest: schema + +The manifest is the contract between the agent and whoever directs it: what to +read, what to do with it, and who may query the result. Every part of the +agent validates against it. The loader is `bastion_agent/manifest.py`; the +signature is checked by the runtime ([runtime.md](runtime.md)) with +`bastion_agent/x509gate.py`. + +## What a bastion agent is + +A bastion agent is a program that runs on the owner's own machine and does the +data work that must stay there: walk a folder, extract text, sterilize personal +data, chunk, embed with a local model, catalog, and seal. A model on the +machine may serve as a sensor (for recall of personal data, embeddings and +summaries), and that is all: the same rule as the PII sterilizer +([pii-sterilization.md](pii-sterilization.md)), a deterministic program owns +the control flow and every change, and a model proposes, never acts. + +The owner's host writes a **job manifest**, and it is signed (by the bastion, +in the managed install). The agent executes exactly that manifest and nothing +else. The output is a **catalog** (small, structured cards per file and per +directory) plus local vectors. The files themselves never move; a host +reasons over them by querying the agent. Sharing is a grant that lets another +host query this catalog: access moves, data does not. + +## Rules the schema enforces + +1. **Plaintext never leaves.** `sink.plaintext` is the constant `never`. Any + other value is a validation error, not a setting. +2. **Nothing runs that was not granted.** `mask` is the ceiling; every stage + must be covered by an op in the mask, or the manifest is refused before it + runs. +3. **Nothing leaves unsealed.** A catalog sent to the host, `seal_to` + recipients, or any grant requires the `seal` stage. +4. **No addresses.** The sanitizer and the embedder are found from the + machine's environment when the agent runs. The manifest never carries a + host:port. +5. **Signed by a CA-issued identity.** The runtime refuses a manifest whose + signing certificate does not chain to the root in `FREEAGENT_CA_ROOT`, and + refuses to judge at all without that root, because "signed by the key in + the certificate next to the manifest" proves nothing. The signing + certificate must carry exactly one CN. The loader checks the shape and + computes the signing payload; the runtime checks the signature and the + chain in the standard library (ECDSA P-256 / SHA-256), so the agent needs + neither openssl nor a third-party package on the machine. + +## File layout + +```yaml +freeagent: 1 +id: +owner: + host: # the authoring host's cert CN + +scope: + roots: [, ...] + exclude: [, ...] # optional + follow_symlinks: false # optional + max_file_bytes: 268435456 # optional + +mask: [read, search, index, summarize] # subset; 'read' is mandatory + +stages: [walk, extract, sanitize, chunk, embed, catalog, seal] # ordered subset + +sanitize: # required iff 'sanitize' in stages + mode: sterilize | flag + scope: # optional, default = id + verify: false # optional + +embed: # required iff 'embed' in stages + model: + dim: + chunk: + size_tokens: + overlap_tokens: + +summarize: # optional; needs 'summarize' in mask + model: + per: [file, directory] + +sweep: 15m | hourly | daily | manual + +sink: + catalog: local | host + vectors: local + plaintext: never + seal_to: [, ...] # optional + +grants: # optional, default [] + - host: + ops: [search, read_range] + until: + +signature: # optional in the file; runtime requires it + alg: ecdsa-p256-sha256 + cert: + value: +``` + +The first key, `freeagent: 1`, keeps the program's first name on purpose: the +bastion writes it into every manifest it generates, and renaming it would make +the agent refuse them. + +## Fields + +### Top level + +| Field | Required | Rules | +|---|---|---| +| `freeagent` | yes | Schema version. Integer `1`. | +| `id` | yes | Lowercase slug `[a-z0-9-]+`, 1–63 characters. Names the catalog and is the default token scope for personal data. | +| `owner.host` | yes | Slug, 1–63 characters. The authoring host's certificate CN; the runtime checks the signing certificate's CN against it. | + +Unknown keys anywhere are a validation error, so a misspelled `excludes:` +fails loudly instead of silently indexing everything. + +### `scope` + +| Field | Required | Rules | +|---|---|---| +| `roots` | yes | Non-empty list of paths. Each must be absolute (POSIX `/srv/a`, a Windows drive `C:\Users\me` or `C:/Users/me`, UNC `\\server\share`) or start with `~/` (or be exactly `~`), which expands to the running user's home on every OS. Relative paths are refused, so a manifest means the same thing wherever it lands. No duplicates. | +| `exclude` | no | List of glob strings, matched against the `/`-separated path relative to each root (`**` spans directories; a pattern with no `/` also matches any single path segment, so `node_modules` prunes it at any depth). Case-insensitive on Windows only. Default `[]`. See [walk.md](walk.md). | +| `follow_symlinks` | no | Boolean, default `false`. | +| `max_file_bytes` | no | Positive integer, default `268435456` (256 MiB). Larger files are catalogued by metadata only, never read. | + +### `mask`: the ceiling + +A non-empty list from `read`, `search`, `index`, `summarize`, with no +duplicates. + +- `read`: **mandatory**; every stage reads. +- `search`: the agent may answer `search` queries over the vectors. +- `index`: the agent may write vectors and catalog entries. Required by + `embed` and `catalog`. +- `summarize`: a local model may write summaries. Required by the `summarize` + section. + +There is no `write` and no `delete`: an agent never changes the files it +reads. Any other value is an error. + +### `stages`: what runs, in order + +A non-empty list, a subset of the canonical order, in canonical order: + +``` +walk → extract → sanitize → chunk → embed → catalog → seal +``` + +| Stage | Needs | Does | +|---|---|---| +| `walk` | (always first, always present) | The file system and metadata map: path, size, mtime, type, hash. The cheapest thing a host can plan from. | +| `extract` | `walk` | Text out of documents, code and (with a local OCR tool) images. | +| `sanitize` | `extract` | Personal data replaced by tokens in the text every later stage reads; never in the source files. | +| `chunk` | `sanitize` | Splits the sanitized text for embedding. | +| `embed` | `chunk`; mask `index` | Vectors from a local model, stored locally. | +| `catalog` | `sanitize`; mask `index` | Per-file and per-directory cards: what it is, when, entity counts, summary. **This is the curated dataset.** | +| `seal` | `catalog` | Encrypts the catalog to the owner's key and `sink.seal_to`. | + +Out-of-order, duplicate or unknown stage names are errors, as is a stage whose +prerequisite stage is absent. + +### `sanitize` + +Present **if and only if** `sanitize` is in `stages`. + +| Field | Required | Rules | +|---|---|---| +| `mode` | yes | `sterilize`: the full pipeline (rules, local-model recall, vault). `flag`: the rules pass only; deterministic, no model, and **not** "no personal data" (see [sanitize.md](sanitize.md)). | +| `scope` | no | Token scope, 1–63 characters. Default: the manifest `id`. The same (scope, type, value) gets the same token; a different scope gets a different token, so two catalogs cannot be linked through placeholders. | +| `verify` | no | Boolean, default `false`. A second, adversarial model pass (`sterilize` only; an error with `flag`). | + +Where the sterilizer runs is never in the manifest; the agent reads it from +its environment and fails closed if it is absent. + +### `embed` + +Present **if and only if** `embed` is in `stages`. + +| Field | Required | Rules | +|---|---|---| +| `model` | yes | Model name, 1–128 characters. | +| `dim` | yes | Integer 1–8192. The stage refuses to run if the local endpoint's vectors have another dimension. Vectors are stored per model, so a later model change never discards existing vectors. | +| `chunk.size_tokens` | yes | Integer 1–8192. | +| `chunk.overlap_tokens` | yes | Integer, 0 ≤ overlap < size. | + +There is no endpoint field. Embedding is local, always. + +### `summarize` + +Optional. Requires `summarize` in `mask` and `catalog` in `stages`. + +| Field | Required | Rules | +|---|---|---| +| `model` | yes | Local model name, 1–128 characters. | +| `per` | yes | Non-empty subset of `file`, `directory`. | + +### `sweep` + +One of `15m`, `hourly`, `daily`, `manual`. + +### `sink` + +| Field | Required | Rules | +|---|---|---| +| `catalog` | yes | `local`: the sealed catalog stays here, and the host queries it. `host`: the sealed catalog is sent to the owner's host, so it can plan while this machine is asleep. Requires the `seal` stage. | +| `vectors` | yes | `local`, the only value. | +| `plaintext` | yes | `never`, the only value, ever. It is there so the rule is written down in every manifest and can be searched for, not remembered. | +| `seal_to` | no | List of recipient keys, 1–128 characters each, no duplicates. Requires the `seal` stage. The owner's key is always added by the agent and need not be listed. | + +### `grants` + +An optional list, default `[]`. Each entry lets another host query this +catalog. Requires the `seal` stage. + +| Field | Required | Rules | +|---|---|---| +| `host` | yes | Slug, 1–63 characters: the grantee host's certificate CN. Must not equal `owner.host`. Unique across grants. | +| `ops` | yes | Non-empty subset of `search`, `read_range`. `search` requires `search` in the mask; `read_range` requires `read` (always present). | +| `until` | yes | `YYYY-MM-DD` expiry, a real calendar date. | + +Revoking is removing the entry and re-signing; `serve` stops serving that host +on its next request. The grantee's client certificate must also chain to +`FREEAGENT_CA_ROOT` ([serve.md](serve.md)). + +### `signature` + +Optional in the file, so a manifest can be written, validated and reviewed +before it is signed. **The runtime refuses an unsigned manifest.** +`python3 -m bastion_agent validate --require-signed` enforces that here. + +| Field | Required | Rules | +|---|---|---| +| `alg` | yes | `ecdsa-p256-sha256`. | +| `cert` | yes | Path to the signing host's PEM certificate, 1–256 characters, read relative to the manifest. | +| `value` | yes | Base64 signature over the signing payload. | + +**Signing payload** = the manifest with `signature` removed, serialized as +canonical JSON (sorted keys, no whitespace, UTF-8, dates as ISO strings). +`bastion_agent.signing_payload(doc)` produces it and `signing_digest(doc)` its +SHA-256 hex, so the signer and the runtime agree byte for byte. Any ECDSA P-256 +/ SHA-256 signer will do; the managed install signs in the bastion and records +the result with `sign --value`. By hand, with openssl: + +``` +python3 -m bastion_agent payload manifests/home-documents.yaml > payload.json +openssl dgst -sha256 -sign payload.json | base64 -w0 +python3 -m bastion_agent sign manifests/home-documents.yaml --value --cert +``` + +(`sign --key --cert ` does the same in one step.) + +The gate, in the standard library: exactly one CN, equal to `owner.host`; the +certificate inside its validity window and either identical to a certificate +in `FREEAGENT_CA_ROOT` (the managed install: its certificate is its own root, +and its self-signature must verify) or signed directly by one that is a CA +(basicConstraints, and keyCertSign when the root has a keyUsage extension); +and the signature over this payload. It is a one-level PKI only: longer chains +are not built. + +## Loader + +`bastion_agent/manifest.py` (Python 3, PyYAML). Library API: + +```python +from bastion_agent import load_manifest, signing_payload, signing_digest +m = load_manifest("manifests/home-documents.yaml") # raises ManifestError listing every problem +m.id, m.owner_host, m.scope.roots, m.mask, m.stages # typed, frozen +m.sanitize, m.embed, m.summarize # None when the section is absent +m.sink.catalog, m.grants, m.signed # grants: tuple of Grant +signing_payload(m.raw) # bytes the signature covers +``` + +CLI (from the repository root): + +``` +python3 -m bastion_agent validate [--require-signed] [...] +python3 -m bastion_agent show # stages, mask, grants, signed? +python3 -m bastion_agent payload # canonical signing payload to stdout +``` + +Validation collects **all** errors before failing, so one pass shows +everything wrong with the file. `python3 -m bastion_agent validate manifests/*.yaml` +validates every example. `python3 -m unittest bastion_agent.test_manifest` +runs the loader's tests, which also load `manifests/home-documents.yaml`, so +the example can never drift from the schema. + +## Deliberately absent + +- A `write` op. An agent reads; adding a way to change files is a decision of + its own, not a field. +- An endpoint or address field anywhere, for the reason in rule 4. diff --git a/docs/packaging.md b/docs/packaging.md new file mode 100644 index 0000000..b841692 --- /dev/null +++ b/docs/packaging.md @@ -0,0 +1,183 @@ +# bastion agent packaging: one file for three operating systems + +## The artifact + +`python3 -m bastion_agent.build --out dist` → `dist/bastion-agent-.pyz` +and `.sha256`. + +A **zipapp**, the standard library's own single-file format: one file of about +150 KB that runs on any Python 3.8 or later on Mac, Windows or Linux: + +``` +python3 bastion-agent-.pyz doctor +python3 bastion-agent-.pyz sweep manifest.yaml --once +python3 bastion-agent-.pyz serve manifest.yaml +``` + +Inside: the `bastion_agent` package (every stage, the scheduler, the +endpoint; no tests, and neither `build.py` nor `verify_release.py`, which are +tools for making and checking a release), the `pii` package (the sterilizer; no +tests, no Dockerfile), a vendored **pure-Python PyYAML** (MIT, its notice +included; the C accelerator is left out so one file runs everywhere), which is +the one import the manifest loader needs beyond the standard library, and the +repository's `LICENSE`, so the license travels with the program. The version +comes from `bastion_agent/VERSION`, read with `pkgutil` so it works from inside +the zip. + +`bastion_agent/VERSION` on `main` is the next release, not necessarily the one +bastions run. A merged change waits there until it is published, and the +published version is whatever +names at the moment: a bastion installs that version and no other. Between a +merge and a publish, `main` is ahead of the manifest; when a second release is +merged before the first is published, only the later one is published. + +**Reproducible.** Fixed member timestamps, sorted members, no bytecode, and +the vendored PyYAML pinned to one release (`PYYAML_VERSION`; the build refuses +any other): two builds of the same tree are byte-identical on any machine, so +the `.sha256` is a real statement about the file, and anyone can rebuild a +release and compare (`python3 -m bastion_agent.verify_release`, in the README). +`test_build` asserts identical bytes across two builds. + +`age` stays what it is, a single external binary, because the standard library +cannot do X25519 or ChaCha20. The manifest gate does **not** need `openssl`: it +reads the certificate and verifies the ECDSA P-256 signature in the standard +library (`x509gate.py`), so the agent runs on a machine without it (Windows +ships none). `openssl` is needed only to sign with `sign --key` and for +`SEAL_SIGN_KEY` seal signatures. `doctor` reports both. + +## doctor + +`bastion-agent doctor [--json]` is the preflight; it changes nothing: + +| Check | Used by | Required? | +|---|---|---| +| Python ≥ 3.8, PyYAML | everything | yes | +| `age` | seal | yes | +| `openssl` | `sign --key`, seal signatures | no: the gate verifies in the standard library | +| `pdftotext`, `tesseract` | extract | no: without them the gaps are recorded | +| `gliner` package | sanitize | no: the laptop recall backend | +| `EMBED_URL` (default `http://127.0.0.1:11434`), and `PII_ADDR` / `SUMMARIZE_URL` when set | embed / sanitize / catalog | no: reachable? Any HTTP answer counts, so a proxy that allows only the embedding path (and answers 403 on `/`) is reachable | +| `FREEAGENT_OWNER_RECIPIENT` set; `FREEAGENT_CA_ROOT`, `FREEAGENT_TLS_CERT`, `FREEAGENT_TLS_KEY` set and the files present | seal / sweep / serve | no, for the preflight: the stages that need them refuse without them | + +The exit code is 1 only when a required item is missing; optional gaps are +printed as `opt`. + +## Installers (a hand install) + +The bastion installs the agent itself from a grant; these are for running it +by hand. + +- `install.sh `: Linux and macOS, POSIX sh, no root, no downloads. It + checks for `python3` 3.8 or later, verifies the `.sha256` when it sits next + to the `.pyz` (`sha256sum` or `shasum`), copies to + `~/.local/bin/bastion-agent.pyz`, writes a `bastion-agent` wrapper (and + repoints an existing `freeagent` wrapper, so an earlier hand install keeps + working), runs `doctor`, and prints the next step. `BASTION_AGENT_BIN_DIR`, + or else `FREEAGENT_BIN_DIR`, overrides the directory. +- `install.ps1 `: Windows PowerShell, no administrator. The same checks + with `Get-FileHash`; installs to `%LOCALAPPDATA%\bastion-agent`, writes + `bastion-agent.cmd`, adds the directory to the user's PATH, and runs + `--version` and `doctor`. + +Neither installer fetches anything. Get the `.pyz` and its `.sha256` from the +download address, then run one command (Linux or macOS): + +``` +V= +curl -fsSLO https://privatae.ai/bastion-agent/dl/bastion-agent-$V.pyz && \ +curl -fsSLO https://privatae.ai/bastion-agent/dl/bastion-agent-$V.pyz.sha256 && \ +curl -fsSL https://privatae.ai/bastion-agent/install.sh | sh -s -- bastion-agent-$V.pyz +``` + +## The download address + +`bastion_agent/publish.sh` publishes to every edge server named in its +environment, each serving from its own disk, at `/bastion-agent/*`, with +backups and dotfiles refused. The legacy `/freeagent/*` path is retired (see +"The legacy name" below): + +``` +https://privatae.ai/bastion-agent/version.json version, built_from, python_floor, files (url, sha256, bytes), installers, notes +https://privatae.ai/bastion-agent/version.json..sig ed25519 signature; = first 16 hex of version.json's sha256 +https://privatae.ai/bastion-agent/dl/bastion-agent-.pyz the artifact +https://privatae.ai/bastion-agent/dl/bastion-agent-.pyz.sha256 +https://privatae.ai/bastion-agent/install.sh and install.ps1 +``` + +The manifest is signed with the bastion's release key, which stays on the key +holder's own computer. A publish therefore runs twice: the first run builds +and prints the `release-sign sign bastion-agent ` command to +run there, and the second, with `SIGNATURE=` and +`BASTION_TREE` pointing at a privatae-bastion checkout, checks the signature +against the keys that checkout pins and the keys the published bastion pins, +and only then publishes. What is signed is a short statement naming the kind +of release, its version and the manifest's sha256. A bastion from 0.41.0 on +fetches `version.json..sig`, refuses a manifest that does not verify, and +never replaces an installed agent with an older version. + +The script builds reproducibly, runs the built `.pyz` once (`--version`), +refuses to publish a version that is not newer than the one already served +(`ALLOW_REPUBLISH=1` overrides it deliberately), uploads to every edge, then +checks each edge's bytes against the manifest it serves. Where it publishes +is not written into the script: the edges, the ssh key, the web root and the +backup directory come from its environment, and it stops if one is missing. + +The manifest's `notes` link to the tag `v` on the public repository, +which marks the source of that release. `publish.sh` does not make the tag; it +is made when that release's source is published to the public repository. + +### The legacy name + +The agent was first published as freeagent, at `privatae.ai/freeagent/`, and +a bastion older than 0.20.0 knows only that address and the +`freeagent-.pyz` file name. Bastions from 0.20.0 read the new address first +and fall back to the legacy one; from 0.42.0 they read only the new one. + +The legacy address is retired: no edge has the `/freeagent/*` route, and +`https://privatae.ai/freeagent/version.json` answers 404. A bastion that +cannot fetch the manifest keeps the agent it has installed, so a bastion older +than 0.20.0 keeps the agent it has and gets no newer one. What would strand a +machine is the other case, a legacy manifest that answers but does not list +the legacy file name, and a retired address cannot produce it. + +`publish.sh` still checks on every run, on every edge (the route in the Caddy +configuration each edge has loaded) and at the public address. With no route +anywhere and a 404 in public it prints, per edge, "no /freeagent route" and +publishes the new file name only. It publishes both names again, the same +bytes under each, if any edge still has the route, if any edge cannot be +asked (ssh fails or times out, or the configuration Caddy has loaded cannot +be read), or if the public address answers anything but 404 (no answer +included), so that address could not serve a manifest without the name an +old bastion asks for. + +The names inside the agent that keep the word freeagent (the `FREEAGENT_*` +environment variables and the manifest's `freeagent: 1` field) are stable +names, not part of this retirement: the bastion sets and writes them, so +renaming one would have to change the bastion and the agent in step, for no +change in behaviour. + +## What this is not + +A native binary. PyInstaller or Nuitka make one per OS, but each has to be +built on that OS, and none is produced; it would be built from the same +`members()` list. Until then the `.pyz` and a system Python are the artifact. +Python 3.8 or later is on every Linux and current Mac (with the Xcode command +line tools or Homebrew) and one Store install away on Windows, and the floor +test keeps the code there. + +## Tests + +`python3 -m unittest bastion_agent.test_build` covers byte-identical builds; +the contents (the entry point, both packages, VERSION, the vendored YAML and +its notice; no tests, no `.so` or `.pyc`, no `build.py`, `verify_release.py` +or Dockerfile; every member at the fixed time; under 1 MB); the `.pyz` +answering `--version` and `doctor --json` and validating the example manifest; +**a full sweep from the `.pyz` in a scratch directory with `PYTHONPATH=""`** +(nothing from the repository on the path) producing a catalog with sanitized +heads; the `.agent` default output directory; and the doctor's checks against +unreachable and missing inputs. + +This repository has no CI. [CONTRIBUTING.md](../CONTRIBUTING.md) lists the +checks to run before proposing a change, and `publish.sh` builds the release +and runs it before anything is uploaded. The installers are not exercised by +the test suite. diff --git a/docs/pii-sterilization.md b/docs/pii-sterilization.md new file mode 100644 index 0000000..2bffc61 --- /dev/null +++ b/docs/pii-sterilization.md @@ -0,0 +1,226 @@ +# PII sterilization + +The `pii` package finds personal data in text and replaces it with +placeholder tokens, keeping the real values in a local vault. The agent's +`sanitize` stage runs it in process ([sanitize.md](sanitize.md)); it also +runs on its own as a small HTTP service. It is a deterministic program with +a model as one of its sensors: the program owns the control flow and every +change to the text, and the model only proposes. + +## Purpose + +Text that is going to leave the machine (a catalog, a chunk returned to a +host, context sent to a cloud model) first passes through `/sterilize`: +personal data is replaced with stable placeholder tokens, the real values stay +in a local vault, and `/rehydrate` puts them back, locally, on demand. A cloud +model can then work over the text without the values in it. Patterns alone +cannot do this: "my daughter goes to Lincoln Elementary" contains no pattern, +only meaning. + +## Pipeline (`pii/pipeline.py`) + +1. **Rules pass** (`detect.py`): deterministic detectors (below) for the + anchors a pattern can find. They never depend on a model. Overlaps are + resolved by type priority, then by length (an IP address is not a dotted + phone number). +2. **Recall pass** (`recall.py`): a model proposes verbatim spans with a type + from a closed vocabulary: names, addresses, employers, schools, medical + facts, credentials. Which model is chosen when the service starts (below). +3. **Grounding**: every proposed span must occur exactly in the input, be at + least two characters, have a known type and not contain a token or a + token's bare name (`PII_PERSON_1`), or it is dropped and counted. An invented span can never mint a token or touch the + text. +4. **Replacement**, by the program only: longest value first, every + occurrence. The model never rewrites the text, so a model that misbehaves + can cost recall but cannot corrupt the rest of the text. Replacement is by + exact substring, so a value is also replaced where it appears inside a + longer word ("Garcia" in "Garcias", "Marisol" in `Marisol_resume.pdf`): a + missed inflection would be a leak, and an extra match only costs + readability. No match may overlap a placeholder already in the text. + Matches that overlap each other, including a value that overlaps itself + ("aba" in "ababa"), become one placeholder over their union, typed by the + longer value, so no fragment of either is left in the clear. + Placeholder-shaped text in the input is first replaced by a placeholder of + its own, so rehydrating gives back the literal text. Documents the agent + reads are raw files, so every such literal in them is escaped. Text sent to + the service may be the service's own earlier output (a stored conversation + sent back on every save), so there a literal the vault holds for the scope + is taken to be that token and passes through unchanged, and only a literal + the vault does not hold (`[PII_FOO_99]`) is escaped. A caller can ask for + every literal to be escaped with `escape_held_literals: true`. The cost of + replacement grows with the number of occurrences times the number of + placeholders in the text. +5. **Residual loop**: the rules pass runs again on the output until it is + clean, at most three rounds; residuals are replaced, not only reported. +6. **Optional verify** (`verify: true`): a second, adversarial model pass over + the sterilized text; its grounded findings go through replacement too. + +If the recall pass fails, `/sterilize` answers 502: fail closed. Text without +the recall pass never leaves as "sterilized". + +## Tokens, scopes, vault + +- Tokens look like `[PII__]`: plain ASCII, they survive a round trip + through a cloud model, and the model can reason about `[PII_PERSON_1]` + across a conversation. +- **Stable within a scope**: the same (scope, type, value) always gets the + same token. **Isolated across scopes**: the same value in another scope gets + a different token, so two conversations or catalogs cannot be linked through + shared placeholders. Stability is per value as replaced, not per detection: + where a model's span overlaps a rule's hit (a phone number inside an address + the model proposed), the two become one placeholder for the combined text, + typed by the longer value. That placeholder differs from the one the phone + number gets on its own elsewhere. If a later run proposes the longer span + with a different type, the same bytes get a different placeholder. The + caller chooses the scope (default `default`; the agent uses the manifest's + `sanitize.scope`). +- The vault is SQLite, created 0600 under umask 077. Rows are never deleted: + the next number in a scope is the count of its rows plus one, so after a + delete that number either names a token that still exists (and the insert + fails) or, if the newest row was deleted, gives its token to a different + value. A delete feature would first need a separate sequence. In the service, every sterilize and rehydrate also + writes an audit row; `GET /scope/` lists tokens, types and creation + times, never values. + +## The service (`pii/service.py`) + +Standard library only: the process holds the most sensitive data on the +machine, and every dependency is attack surface. + +| Route | Auth | Does | +|---|---|---| +| `GET /health` | none | Liveness. | +| `GET /scope/` | key | The token inventory of a scope (no values). | +| `POST /sterilize {text, scope?, verify?, escape_held_literals?}` | key | The pipeline; returns `{text, findings, stats}`, with `stats.recall` naming the recall backend. `escape_held_literals: true` escapes every placeholder-shaped string in `text`, even one the vault holds; the agent sends it for documents. | +| `POST /rehydrate {text, scope?}` | key | Values back in place of known tokens; returns `{text, restored, unknown_tokens}`. | +| `POST /flag {text}` | key | The rules pass alone: `{flagged, counts}`. No model, no vault rows, no audit. | + +Request bodies are limited to 2 MiB. + +`stats` in a `/sterilize` answer counts what this call did: `rules` (rule +findings), `llm` (grounded model proposals), `llm_ungrounded` (proposals +dropped by grounding), `residual_tokenized` and `verify_tokenized` (values +those passes actually replaced), and `literal_tokens_escaped` +(placeholder-shaped strings in the input that were escaped, each distinct +string counted once per call). Escaped literals appear in neither `findings` +nor the per-type counts. `stats.recall` names the recall backend, and +`stats.escape_held_literals` echoes which literal rule was applied, so a +caller that asked for it can check. + +**A clean `/flag` does not mean "no personal data".** It sees only what the +rules find, never contextual data such as a name; anything that is kept or +sent on goes through `/sterilize`. + +Configuration, from the environment: + +| Variable | Default | Meaning | +|---|---|---| +| `PII_API_KEY` | (none) | Required. Without it the service refuses to start: `/rehydrate` returns real values, so there is no unauthenticated mode. (`PII_ALLOW_NOAUTH=1` exists for the unit tests only.) | +| `PII_BIND` | `0.0.0.0:8091` | The address the service listens on; this is its exposure control. | +| `PII_DB` | `/data/vault.db` | The vault. | +| `PII_RECALL` | `llm` | The recall backend (below). | +| `LLM_URL` | `http://llm:8080` | The model server for `llm`. | +| `PII_LLM_TIMEOUT` | `300` | Seconds per model call. | + +Two ways to run it, the same code: + +- **Container**: `pii/Dockerfile` (a pinned `python:*-slim` image, no pip + install; the vault on a mounted `/data`). +- **systemd**: `pii/pii.service`, with install steps in its header. It runs + with `DynamicUser`, a private 0700 state directory for the vault + (`/var/lib/privatae-pii/vault.db`), loopback bind (`127.0.0.1:8091`), and + the key in a 0600 environment file. + +The bastion can make a service on the machine reachable to the owner's host: +with `PRIVATAE_PII_ADDR` set to the service's address, it forwards to it +through its tunnel and reports in its heartbeat that the service is there and +whether it answers. The bastion passes requests through without reading them +and never holds the API key; the caller sends it. + +## Recall backends (`pii/recall.py`) + +The recall pass is a pluggable sensor, chosen by whoever runs the service +with `PII_RECALL` when it starts, never per request, so a request cannot +downgrade it. Whichever sensor proposes, grounding drops any span that is not +exactly in the input and the program makes every replacement, so swapping the +sensor changes recall, never the integrity rules. + +| `PII_RECALL` | What | Where it fits | +|---|---|---| +| `llm` (default) | A llama.cpp-compatible server's `/v1/chat/completions` at `LLM_URL`, with the output constrained by a JSON schema (`response_format: json_schema`), temperature 0, thinking turned off per request (`llmclient.py`). | A machine that can reach a model server. Best recall. | +| `gliner` | GLiNER (Apache-2.0), zero-shot named-entity recognition in this process, on CPU. Its labels are plain-English phrases mapped onto the closed type vocabulary (`recall.GLINER_LABELS`); `OTHER` is deliberately not a label. `GLINER_MODEL` (default `urchade/gliner_multi_pii-v1`), `GLINER_THRESHOLD` (0.5), `GLINER_VERIFY_THRESHOLD` (0.3, the more aggressive second pass). | A laptop with no GPU and no model server. | + +Fail closed when it starts: `PII_RECALL=gliner` without the `gliner` package, +a model that will not load, or an unknown backend name stops the service before +it serves a request. (The `llm` server is not probed at start; an unreachable +one fails each `/sterilize` with 502.) + +Why GLiNER and not Presidio: Presidio (MIT) is the usual framework, but it +brings spaCy and optionally torch into the process that holds the data. GLiNER +is one optional import, loaded only when chosen. What was taken from Presidio +instead is its list of recognizers, as deterministic rules. The cost is real: +the `gliner` package pulls in torch (a CPU build), hundreds of megabytes on +the machine. + +## The rules pass (`pii/detect.py`) + +Every detector is anchored by a checksum, a prefix or a fixed shape; none +depends on a model. + +- **E-mail, phone, SSN, IPv4/IPv6, MAC.** +- **Cards** → `CREDIT_CARD`, checked with Luhn. +- **IBAN** → `BANK_ACCOUNT`: ISO 13616 mod-97 plus a per-country length + table. It runs before the card detector, so an IBAN's digits cannot be + claimed as a card number. +- **US ABA routing number** → `BANK_ACCOUNT`: the weighted checksum and a + `routing`, `ABA`, `RTN` or `transit` cue within 40 characters before it + (one random 9-digit string in ten passes the checksum alone). +- **Bitcoin legacy and P2SH addresses** → `BANK_ACCOUNT`: base58check + (double SHA-256). Bech32 (`bc1…`) addresses are not detected. +- **Secrets** → `CREDENTIAL`: AWS access key ids, GitHub classic and + fine-grained tokens, Slack tokens, Google API keys, Stripe keys, GitLab + tokens, OpenAI-style `sk-` keys, JWTs, and whole PEM private-key blocks (one + finding per block, at the highest priority, so the digits inside are never + claimed piece by piece). + +## Portability + +The package parses at the Python 3.8 floor shared with the agent +(`test_syntax_floor.py`), with `typing` generics and no newer standard-library +calls, so the systemd form runs on older interpreters as it is. + +## Tests + +- `python3 -m unittest pii.test_pii`: detectors, vault stability and + isolation, grounding, longest match first, the HTTP wire format against a + stub llama.cpp server, auth, and fail-closed configuration. No model is + needed. +- `PII_API_KEY= pii/verify-live.sh [host:port]` runs eight checks + against a running service and its real model: rules-class sterilization, + contextual recall (school, address, doctor, medical), a round trip back to + the original, token stability, scope isolation, untouched non-personal text, + auth, and health. + +## Limits + +- **Sterilization narrows what is missed; it cannot guarantee nothing is.** The + recall pass is probabilistic. The layers (rules, model, residual loop, + optional verify) reduce misses but cannot make them impossible. Anything + that tells a person their data is protected has to say so. +- **The vault is not encrypted at rest.** The controls are the file's 0600 mode, + its directory, and the bind address. +- **Rows are never deleted** (see above), so there is no erasure yet. +- **Placeholders are plain tokens.** Surrogate values (invented names in place + of real ones, for fluency) are not done. +- **Through the service, a quoted placeholder that names a live token is read + as that token.** If text sent to `/sterilize` quotes `[PII_PERSON_1]` and the + vault holds `[PII_PERSON_1]` for the same scope, rehydrating gives the + vault's value in its place. The placeholder cannot be told apart from one + the service minted earlier, and treating it as a token is what lets + already-sterilized text be sterilized again without change. A quoted + placeholder the vault does not hold is escaped and comes back as written. + The agent's document stage is not affected: it asks for every literal to be + escaped. +- **One model call per `/sterilize`**, two with `verify`; the agent makes one + per 20,000-character chunk. That suits writes to context and memory; an + interactive path would want smaller calls or a smaller model. diff --git a/docs/runtime.md b/docs/runtime.md new file mode 100644 index 0000000..f9fa961 --- /dev/null +++ b/docs/runtime.md @@ -0,0 +1,131 @@ +# bastion agent runtime: the `sweep` scheduler and the manifest gate + +`sweep` is the agent proper: the loop that checks the manifest and runs its +stages on the manifest's cadence, on the owner's machine. It is +`bastion_agent/run.py`, with the signature check in `bastion_agent/x509gate.py`, +standard library only apart from the manifest loader's PyYAML. + +`python3 -m bastion_agent sweep [--out DIR] [--once] [--allow-unsigned]` + +Each cycle: + +1. **Read the manifest again.** A re-signed manifest takes effect on the next + sweep: a revoked grant, a new root, a changed cadence. +2. **The gate.** The manifest must carry a signature that verifies: ECDSA + P-256 over the canonical payload (`signing_payload`, the same bytes the + `payload` command prints), from a certificate with exactly one CN, equal to + `owner.host`, that chains to `FREEAGENT_CA_ROOT`. The root is required: + without one the gate refuses rather than trust the certificate beside the + manifest. Anything else (unsigned, the wrong key, edited after signing, + someone else's certificate, a certificate outside the CA, a certificate with + two CNs) and **nothing runs**. The refusal is appended to `run.jsonl` like + any other run; a manifest too broken to have an id is logged under + `build/freeagent/refused/` unless `--out` is given. `--allow-unsigned` is + for development only: the CLI help says so in capitals, and it is written + into every run record it touches. +3. **The stages, in the manifest's order.** The first failure stops the run, + so a later stage never consumes a half-finished earlier one. A stage that + completes but reports `stopped` (sanitize or catalog failing closed) counts + as failed. +4. **Record and wait.** One line is appended to `run.jsonl`, `run.state.json` + is rewritten, and the loop sleeps until the next run is due, or exits for + `manual` (and after `--once`). + +Signing, on the host's side: `python3 -m bastion_agent sign --cert --key ` +signs with openssl and writes (or replaces) the signature block; +`--value ` records a signature made elsewhere instead. The bastion +signs in its own process with the key it minted for the agent and records it +with `--value`. The certificate path is recorded as given and read relative to +the manifest. [manifest.md](manifest.md) has the details. + +## Where an agent lives: `/.agent/` + +One agent per folder, and the agent lives with its folder. For a manifest with +one root, the commands that run a single stage, `sweep` and `serve` default +their output directory to `/.agent/`: the signed manifest and every prepared output +(walk map, extract index, sanitized store, chunks, vectors, catalog, run log, +sealed bundles). Copy the folder and the agent comes with it; fork it and you +have a new one. The walker always excludes `.agent/`, so an agent never +indexes its own output. + +What is in `.agent/`: the **extract store is the plaintext of every file in +scope** (a PDF or DOCX becomes a flat `.txt`), and the sanitized store is that +text with personal data tokenized (only the rules-pass anchors, unless the +manifest uses `sterilize`). That is the same trust boundary as the folder +itself, since the source files are already there, but a sync provider now +receives a second, flattened, indexable copy. If that matters for a folder, +exclude `.agent` from sync or keep the output elsewhere (`--out`). + +The identity key is never in `.agent/`: it is wherever `FREEAGENT_TLS_KEY` +points. The PII vault (plaintext values) is in `.agent/sanitize/vault.db` for +an agent run by hand. A Documents folder is often cloud-synced, so set +`FREEAGENT_VAULT_DIR` to keep the vault out of it; the bastion always sets it +to its own state directory. The certificate is public and may sit anywhere. + +**One machine per folder.** The manifest is signed by the machine's own key, +and the gate proves "signed by this key", not "the newest". If a synced folder +is served from two machines, or a sync conflict restores an earlier signed +manifest, the endpoint serves the older terms until the manifest is rewritten. +Run one agent per folder, on one machine. + +## Cadence and backoff + +| `sweep` | Period | On failure | +|---|---|---| +| `15m` / `hourly` / `daily` | 900 / 3600 / 86400 s from the run's start | Retry after 60 s, 120 s, 240 s … capped at the period; `consecutive_failures` counts up and resets on success. | +| `manual` | | One run, then exit. | + +Waits are taken in naps of at most 30 s, so SIGTERM or SIGINT lands quickly; a +stop request lets the current run finish and never interrupts a stage. + +## The lock + +`/sweep.lock`, created atomically (`O_EXCL`) with the pid and time +inside. A second sweep on the same output directory is refused, with the holder +named. A lock is stale only when its holder is provably gone: on POSIX the pid +decides, and a live pid keeps its lock however long its sweep runs; only where +the pid cannot be checked (Windows) or the file cannot be read does age decide, +by the file's mtime, after 12 hours. A reclaimed lock is noted in the run +record. + +## Files + +| File | Content | +|---|---| +| `run.jsonl` | Append-only, one record per run: `v`, `started`, `finished`, `seconds`, `manifest`, `out`, `manifest_id`, `manifest_digest`, `gate` (`cn`, `cert`, `chained`, or `bypassed` with the reason), `stages[]` (`stage`, `status`, `seconds`, a few counts, `error`), `ok`, `error` (on failure), `allow_unsigned`, `lock_reclaimed` (when it happened). | +| `run.state.json` | The latest state: `last_started`, `last_finished`, `last_ok`, `last_error`, `consecutive_failures`, `sweep`, `next_due`, `pid`, `runs_this_process`. What a health check reads. | +| `sweep.lock` | Present only while a sweep runs. | + +The stages read their settings from the environment, never the manifest: +`FREEAGENT_OWNER_RECIPIENT`, `FREEAGENT_VAULT_DIR`, `FREEAGENT_CA_ROOT`, +`FREEAGENT_NO_EXTERNAL=1` (built-in readers only), `EMBED_URL` / `EMBED_API`, +`PII_ADDR` / `PII_API_KEY` / `PII_RECALL` / `LLM_URL`, `SUMMARIZE_URL`, and +`SEAL_SIGN_KEY`. + +## Running it as a service by hand + +The bastion runs the agent itself. To run one without a bastion on Linux, +`bastion_agent/bastion-agent.service` is a systemd **user** unit: the sweep +runs under the owner's account with their permissions, reads its environment +from `~/.config/bastion-agent/env` (create it 0600, as the unit's header +shows), writes to `~/.local/share/bastion-agent/out`, and runs with +`UMask=0077`, `NoNewPrivileges` and `PrivateTmp`, restarting 60 s after a +failure. There is no launchd or Scheduled Task file yet; on a Mac or Windows +machine run the same command from either. + +## Tests + +`python3 -m unittest bastion_agent.test_run bastion_agent.test_x509gate`. +`test_run` covers the gate (sign then verify; re-signing replaces the block; an +edit after signing refused; a CN mismatch refused; a certificate with two CNs +refused; the CA chain accepted against the right root and refused against +another; unsigned refused unless `--allow-unsigned`, and then recorded), runs +(stages in order with counts logged; a failing stage stops the run and later +stages never execute; an invalid manifest recorded as an error; the lock +refusing a second holder and reclaiming a dead one; `next_due` arithmetic +including the backoff cap), and the loop with a fake clock (`manual` runs +once; `15m` runs three times, sleeping exactly two periods in naps of at most +30 s, with `next_due` right; hourly failures retry after 60 s then 120 s with +the counter at 3). `test_x509gate` runs the gate with openssl removed from +PATH, through the CLI with no openssl, and against bit flips, expiry and a +foreign CA with the same name. diff --git a/docs/sanitize.md b/docs/sanitize.md new file mode 100644 index 0000000..2a16ed0 --- /dev/null +++ b/docs/sanitize.md @@ -0,0 +1,134 @@ +# bastion agent `sanitize` stage: personal data out of the text + +The sanitize stage runs the extracted text through the PII sterilizer +(`pii/`, described in [pii-sterilization.md](pii-sterilization.md)) and keeps +the result in a store of its own. Every later stage reads that store, never the +extracted text. It is `bastion_agent/sanitize.py`. + +## What it produces + +`python3 -m bastion_agent sanitize [--out DIR] [--require-signed]` +reads the extract index and store and writes: + +- **`sanitize//.txt`**: the sanitized text store. Personal data is + replaced by stable `[PII_TYPE_n]` tokens. **From this stage on, later + stages read only this store.** The extract store keeps the plaintext and + never leaves the machine. +- **`vault.db`**: the token-to-value map (in-process mode only), created 0600 + under umask 077. It is the only way back from a token to a value, and it + stays on the machine. It is `/sanitize/vault.db`, or + `$FREEAGENT_VAULT_DIR/vault.db` when that is set: the output folder may be + synced to a cloud drive, the vault must not be, so the bastion sets + `FREEAGENT_VAULT_DIR` to its own state directory. +- **`sanitize.jsonl`**: one record per extracted file: `status`, `mode`, + `stats` (counts from the pipeline, summed over the file's chunks: rule hits, + recall hits, dropped proposals, residuals and verify hits replaced, and + `literal_tokens_escaped`, so a literal that appears in two chunks counts + twice), `types` (token counts per type), `chars`, `reused`, + `error`. **Never a value.** A test searches the index and the summary for + the fixture's personal data and fails if any appears. +- **`sanitize.summary.json`**: counts, token totals per type, the sterilizer + that ran (`flag:rules-only`, `sterilize:llm=…`, `sterilize:gliner=…@0.50`, + `service:http://…`), and `stopped` when the stage failed closed. + +| `status` | Meaning | +|---|---| +| `ok` | Sanitized text written (or kept from an earlier run: then `reused` is true). | +| `skipped` | The extract stage had no text for it (unsupported, error, empty, too large). | +| `error` | A failure that is not the recall pass (the store unreadable, an HTTP error from the service other than 502). The stage continues. | +| `not_run` | The stage stopped (fail closed) at or before this file. | + +Exit code: `0` clean; `1` the manifest is invalid, `sanitize` is not in its +stages, there is no extract index, or the stage **stopped**; `2` completed with +`error` records. + +## Modes (manifest `sanitize.mode`) + +- **`sterilize`**: the full pipeline: the rules pass, the recall pass, + grounding, replacement by the program, the residual loop, and the optional + adversarial `verify` pass. Contextual personal data (names, schools, + employers, medical facts) is found by the recall pass. +- **`flag`**: the rules pass **only**: deterministic, no model, milliseconds. + It replaces the anchors a pattern can find: e-mail addresses, phone + numbers, SSNs, cards, IBANs, routing numbers, crypto addresses, IP and MAC + addresses, API keys, private-key blocks, JWTs. **It does not find a name.** + A flag-mode catalog is not free of personal data, and the manifest doc says + so too. + +## Where the sterilizer runs + +It is chosen from the environment, never the manifest (a manifest carries no +addresses). + +| Condition | Where | Vault | +|---|---|---| +| `mode: flag` | In this process, rules only. `PII_ADDR` is ignored. | `vault.db` as above | +| `mode: sterilize`, no `PII_ADDR` | In this process: `pii` inside the agent, the recall pass from `PII_RECALL` (`llm` → the server at `LLM_URL`; `gliner` → an in-process CPU model). The laptop case. | `vault.db` as above | +| `mode: sterilize`, `PII_ADDR` set | A running PII service over HTTP (`POST /sterilize`, `Authorization: Bearer $PII_API_KEY`). | The service's own | + +`PII_ADDR` without `PII_API_KEY` is refused: the service fails closed, and so +does its caller. + +## Rules the stage enforces + +- **Fail closed.** The first recall failure (the model unreachable, the service + answering 502 or not at all) stops the stage. So does a service that does + not confirm it escaped every placeholder-shaped string in the document + (`stats.escape_held_literals` missing or false: a service older than the + agent); upgrade the service. The file being sanitized and + every file not yet reached are recorded `not_run`; nothing half-sanitized is + written; the exit code is 1 and `summary.stopped` says why. A recall backend + that cannot start (for example `PII_RECALL=gliner` without the `gliner` + package) stops the stage before the first file, with exit code 1 and no + index written. +- **Tokens are stable within a scope.** `sanitize.scope` (default: the + manifest id) is the vault scope: the same e-mail address in two documents of + one catalog gets one token, so the catalog stays coherent, and a different + catalog gets different tokens, so two catalogs cannot be joined through + shared placeholders. +- **Long texts are split, not truncated.** The sterilizer is called per chunk + of at most 20,000 characters (`SANITIZE_CHUNK_CHARS`), split at paragraph + and then line boundaries (a single longer line is cut). The chunks join back + to the input exactly, and because tokens are stable a value that appears in + two chunks gets one token. A cut never falls inside a placeholder: it moves + back to before the placeholder, or just after it when the placeholder starts + the chunk (only possible when the placeholder is longer than the limit), so + that chunk runs over the limit by the rest of the placeholder. + Any text in a document that looks like a placeholder is escaped, whether or + not the vault holds that name, so a document quoting `[PII_EMAIL_1]` comes + back as written and never as another document's value. +- **What makes a file be done again**: a change in its content, the mode, the + scope, `verify`, where the sterilizer runs (in process or service), or + `SANITIZE_VERSION`. **Results are redone when the pipeline changes:** + `SANITIZE_VERSION` is raised whenever the same text would be sterilized + differently, so an upgrade that raises it re-sanitizes every file once. + Changing only the recall backend or its model does not redo files already + sanitized; raise `SANITIZE_VERSION`, or start from an empty output folder, + when that matters. +- **Duplicate content is sanitized once**, and every path that has it is + indexed. + +## Tests + +`python3 -m unittest bastion_agent.test_sanitize` covers flag mode with the +real pipeline and vault (anchors replaced, a name left alone: the limit is +asserted, not hidden), the index and summary carrying no values, the vault's +0600 mode and `FREEAGENT_VAULT_DIR`, reuse and its invalidation (content, +scope), token stability across files, sterilize mode with a fake recall +backend (grounding drops an invented span; stats reported), the fail-closed +stop after the first recall failure (`not_run`, no store file written, the +backend called once), a backend that cannot start, the chunking round trip +(one token across every chunk) and hard cuts, the service path over a stub +speaking the real wire format (auth header, scope, verify, `stats.recall`), a +service 502 failing closed, a service without a key refused, flag mode never +touching the service, and the extract-first guard. + +A `sterilize` run against a real recall model is not part of the suite; the +PII service's `verify-live.sh` does that against a running service. + +## Not in this stage + +Chunking for embedding (the next stage, which reads the sanitized store), +putting values back (the vault is local, and `/rehydrate` belongs to the PII +service), and encryption of the vault at rest (not done: the controls are the +file's 0600 mode and where it is kept). diff --git a/docs/seal.md b/docs/seal.md new file mode 100644 index 0000000..3086e93 --- /dev/null +++ b/docs/seal.md @@ -0,0 +1,89 @@ +# bastion agent `seal` stage: the catalog leaves, nothing else does + +`seal` is the last stage of a run and the one way a dataset leaves the machine: +an allowlisted bundle of the catalog, encrypted with age to the owner and the +manifest's named recipients. It is `bastion_agent/seal.py`. + +## What it does + +`python3 -m bastion_agent seal [--out DIR] [--require-signed]` +builds the bundle and encrypts it to the owner and the manifest's +`sink.seal_to` recipients: + +``` +/sealed/-.tar.age the envelope (0600) +/sealed/-.tar.age.sig optional detached signature +/sealed/latest.json pointer to the newest envelope +/seal.summary.json sizes, digests, recipients, members +``` + +`python3 -m bastion_agent unseal --identity --dest DIR` +is the owner's side: it decrypts, refuses unsafe paths, and checks every +member's SHA-256 against the inventory the sealer wrote inside the bundle. + +## What is in the bundle, and what cannot be + +| Member | Content | +|---|---| +| `manifest.json` | The manifest as executed (signature block removed). | +| `catalog.jsonl`, `catalog.dirs.jsonl` | The cards: sanitized text only. | +| `.summary.json` | The earlier stages' summaries: counts, digests, tool and model names. | +| `seal.manifest.json` | The inventory: sha256 and size per member, the manifest digest, the time of sealing. | + +The bundle is built from an **allowlist**: the two catalog files +(`BUNDLE_ALLOW`), the stage summaries (names matching `.summary.json`), +and the two files the sealer writes itself. There is no code path that adds +the extract store, the sanitized store, the vault, `vectors.db`, or any other +`*.jsonl` index. A test opens the bundle and checks exactly that, and searches +it for the fixture's personal data. The tar is built with fixed member times +and a fixed order, so the same catalog sealed at the same moment is the same +plaintext. + +## Recipients + +- **Owner**: `FREEAGENT_OWNER_RECIPIENT`, an `age1…` public key, from the + environment. Mandatory: no owner recipient, no seal. It is never in the + manifest, because the manifest is written by the host and the owner's key is + the machine's business. +- **`sink.seal_to`**: each entry is an `age1…` recipient or an `ssh-ed25519` + or `ssh-rsa` public-key line (age accepts both). Any other string is refused + before anything is written. +- Duplicates collapse; the owner is always first. +- **Grantees are not recipients.** A `grants` entry lets another host *query* + this agent; it never receives the sealed catalog. Sharing is access, not a + copy. + +## Why age + +age (X25519 and ChaCha20-Poly1305) is a small, widely reviewed format with a +single static binary for Mac, Windows and Linux. The envelope cannot be made +with the standard library (it has no X25519 and no ChaCha20-Poly1305), so, +unlike the extract stage's optional tools, **age is required here**: a catalog +that cannot be sealed does not leave, and the stage says so and exits 1. + +## Signature (optional) + +`SEAL_SIGN_KEY=` adds a detached +`openssl dgst -sha256` signature over the envelope, so a receiving host can +check who sealed it. The signature covers that one envelope (age encrypts +differently every time), not the plaintext. The stage only produces the +signature; checking it against a certificate is the receiver's job. A test +proves it verifies against the public half. + +## Tests + +`python3 -m unittest bastion_agent.test_seal` covers: no owner recipient, +refused with nothing written; malformed recipients refused and named; no `age` +on PATH, refused; `seal` not in the stages, refused; the bundle allowlist +(present members, forbidden members absent, no values, sanitized placeholders +present); reproducible plaintext; a round trip with two real recipients and a +stranger refused; owner and recipient de-duplication and an ECDSA P-256 +signature that verifies with openssl; and a tampered member refused on unseal. +The round-trip tests are skipped where `age` or `age-keygen` is not installed. + +## Where it sits + +walk → extract → sanitize → chunk → embed → catalog → seal, each usable on its +own and each with its own document here. The runtime around them (the `sweep` +scheduler and the manifest gate) is [runtime.md](runtime.md); the query +endpoint is [serve.md](serve.md). diff --git a/docs/serve.md b/docs/serve.md new file mode 100644 index 0000000..c89d3a9 --- /dev/null +++ b/docs/serve.md @@ -0,0 +1,97 @@ +# bastion agent query endpoint: `serve` + +`serve` is how a host reads what the agent prepared without the data moving: +a host (the owner's, or a grantee's) sends a query and gets **sanitized** +chunks and catalog cards back. Nothing else has a route: not the extract +store, not the vault, not the vectors table. It is `bastion_agent/serve.py`, +mutual TLS with the standard library's `ssl`. + +`python3 -m bastion_agent serve [--out DIR] [--bind HOST:PORT] [--port-file FILE] [--allow-unsigned]` + +## Identity: the client certificate + +TLS runs with `ssl.CERT_REQUIRED` against `FREEAGENT_CA_ROOT`, TLS 1.2 at +least. Each connection's handshake runs in that connection's own thread with a +10 s timeout, never on the accept loop, so a peer that connects and says +nothing costs its own thread and nothing else; an idle connection is closed +after 60 s. A client with no certificate, or one from another CA, never +reaches a handler: the handshake fails, one line goes to stderr, and there is +no audit row (it never became a request). For a client that did chain, the +certificate's CN is who they are: + +| CN | Gets | +|---|---| +| equal to the manifest's `owner.host` | `search` (if `search` is in the mask), `read_range`, `catalog`, `status`, `health` | +| a host in `manifest.grants`, on or before its `until` date | exactly that entry's `ops` (`search` and/or `read_range`), plus `health` | +| anyone else who chained | `health` only; everything else is 403, audited with the CN | + +There is no other credential: no token, no password, no header. + +## Routes + +| Route | Body | Returns | +|---|---|---| +| `POST /search` | `{query, k?}` (k defaults to 5 and is held to 1–50) | `{hits, k}`: the top k sanitized windows with score, document id, chunk id, index and offsets. The same search as the CLI, through the same `EMBED_URL` model ([embed.md](embed.md)); 502 if the model cannot be reached. | +| `POST /read_range` | `{doc, start?, end}` | Up to 65,536 characters of the **sanitized** text of that document, with `total`. | +| `GET /catalog`, `GET /catalog/dirs` | query `prefix`, `offset`, `limit` (default 500, at most 2,000) | The file cards or the directory cards (owner only), a page at a time: `{cards, kind, total, offset, limit, next_offset, prefix}`, `next_offset` being `null` on the last page. 404 until the first sweep has written a catalog. | +| `GET /status` | | `{state}`: `run.state.json` (owner only). | +| `GET /health` | | `{ok, id, serving}` for any client that chained. | + +`doc` is the document's content hash (64 lowercase hex), the same id the +extract and sanitized stores share. The route reads only the `sanitize/` +directory, so the plaintext for that id, which exists on disk, is never what +comes back; a test checks both halves of that sentence. + +## The manifest is the policy, live + +On each request the endpoint checks whether the manifest file +changed, and if so re-reads it and runs the same gate as the scheduler: the +signature over the canonical payload, the signing certificate's CN equal to +`owner.host`, and the chain to `FREEAGENT_CA_ROOT` (required). So: + +- **Revoke a grant**: delete the entry and re-sign. The next request from that + CN is 403. +- **Edit without re-signing**: the endpoint serves nothing but `/health` + (which reports `serving: false`), and answers everything else with 503 and + the gate's reason until the manifest verifies again. +- `--allow-unsigned` is for development only, as for `sweep`. + +## Audit + +Every request that reached a handler is one line in `/serve.audit.jsonl`: +`ts`, `cn`, `op`, `path`, `status`, `bytes`; for `search` the first 16 hex +characters of the query's sha256 (never the query) and the number of hits; for +`read_range` the start of the document id and the characters served; for the +catalog the page size and the total. A test checks that the query text is not +in the file. + +## Bind and certificates + +- `--bind` or `FREEAGENT_BIND`: `host:port` or `[v6]:port`, default + `[::]:8470`. `--bind` wins. An IPv6 address gets an `AF_INET6` socket. +- `--port-file FILE` writes the port actually bound. The bastion runs `serve` + on `127.0.0.1:0` and reads the port from this file, so the endpoint is not + reachable from the network; the bastion is its client. + Run by hand for another machine to reach, bind the interface it reaches + instead. +- `FREEAGENT_TLS_CERT` and `FREEAGENT_TLS_KEY`: the machine's certificate and + key. `FREEAGENT_CA_ROOT`: the CA that client certificates must chain to. All + three are required; the server refuses to start without them. + +## Tests + +`python3 -m unittest bastion_agent.test_serve` builds a small PKI per test +class (a CA root, a server certificate, owner, grantee, expired-grant and +unknown-CN client certificates, and a client certificate with the owner's CN +from a different CA) and checks: the owner gets every route and sees sanitized +heads in the catalog, paged; the grantee gets `search` and is 403 on +`read_range`, `catalog` and `status`; an expired grant and an unknown CN are +403; no certificate, or a foreign CA, fails the handshake and leaves no audit +row; a connection that never speaks does not stall the server; `read_range` is +capped, returns sanitized text for the shared id while the plaintext sits on +disk with no route, and gives 404 for an unknown id and 400 for malformed ids +and queries; the audit hashes the query; a grant revoked by re-signing is +refused on the next request; an edit without re-signing turns the endpoint +into 503 with `serving: false`, and re-signing restores it; an unreadable +manifest fails closed; and `--port-file` reports the bound port. Plus bind +parsing, CN extraction, and the required TLS inputs. diff --git a/docs/walk.md b/docs/walk.md new file mode 100644 index 0000000..d114ec6 --- /dev/null +++ b/docs/walk.md @@ -0,0 +1,116 @@ +# bastion agent `walk` stage: the filesystem and metadata map + +The walk is the first stage of every run and the cheapest thing a host can +plan from: every entry under the manifest's roots, with its size, time, type +and hash. It is `bastion_agent/walk.py`, standard library only. + +## What it produces + +`python3 -m bastion_agent walk [--out DIR] [--no-hash] [--require-signed]` +reads the manifest's `scope`, walks every root, and writes two files into +`DIR`. Without `--out`, `DIR` is `/.agent/` for a manifest with one +root, and `build/freeagent//` (under the current directory) for one with +several. The second keeps the program's first name on purpose: it is output a +person may already have, and renaming it would orphan it. Both files are +written atomically (a temporary file, then a rename), so a reader never sees a +half-written map. + +**`walk.jsonl`**: one JSON object per entry, sorted by `(root, path)`, keys +sorted, UTF-8. Identical trees produce byte-identical files. + +| Key | Present on | Meaning | +|---|---|---| +| `root` | all | Index into the manifest's `scope.roots`. | +| `path` | all | Path relative to that root, `/`-separated on every OS. | +| `kind` | all | `file`, `dir`, `symlink` (Windows junctions too), `other` (socket, fifo, device). | +| `size` | file | Bytes. | +| `mtime`, `mtime_ns` | file, dir | Modification time as UTC ISO-8601 and as integer nanoseconds (the cache key). | +| `mime` | file | Sniffed type: magic bytes first, then a fixed extension table, then a text/binary probe. `inode/x-empty` for empty files. The `mimetypes` module is not used, because it reads per-machine registries and would make the map depend on the machine. | +| `sha256` | file | Hex digest, or `null` when the file was too large, could not be read, or `--no-hash` was given. | +| `skip` | file | `too_large` when `size > scope.max_file_bytes` (default 256 MiB). The file is still sniffed and listed, never read in full. | +| `target` | symlink | The link text, `/`-separated. | +| `error` | any | `ExceptionName: message`. The entry is recorded and the walk continues. | + +**`walk.summary.json`**: `manifest_id`, `manifest_digest` (the signing +digest, which ties the map to the exact manifest that produced it), +`walked_at`, `platform`, `roots` (each with its `resolved` absolute path, and +an `error` when it is absent on this machine), `entries`, `bytes`, `counts` +(`files dirs symlinks other excluded errors too_large hashed hash_reused`) and +`hashing`. + +Exit code: `0` clean; `1` the manifest failed validation (or was unsigned +under `--require-signed`); `2` the walk completed with errors recorded +(a missing root, an unreadable directory). The map is written in every case +but the first. + +## Rules the walker enforces + +- **Excludes prune.** An excluded directory is not descended, so its subtree + costs nothing. Patterns (`scope.exclude`) are `/`-separated and relative to + the root. `**` spans directories; `*` and `?` stay within one segment; + `[..]` and `[!..]` classes work. A pattern with no `/` also matches any + single path segment, so `node_modules` or `*.key` prune at any depth. A + pattern ending in `/**` prunes the directory itself as well as its contents. + Backslashes in patterns are read as `/`. Matching is case-insensitive on + Windows only, as the file system is. +- **The agent's own folder is always excluded.** `.agent` is pruned at any + depth and in any letter case on every OS, whatever the manifest says, so an + agent never indexes its own output (on a case-insensitive Mac volume, + `.Agent` is the same folder). +- **Links are recorded, never followed**, unless `scope.follow_symlinks` is + true. This covers symlinks on every OS and junctions on Windows + (`is_junction()` on Python 3.12 and later, the reparse tag before that). + Following is opt-in because a link out of the root is the usual way a + scoped walk ends up reading the whole disk. +- **Large files are metadata only.** Over `max_file_bytes` the walker reads + 8 KiB to sniff the type and stops. +- **Errors are entries, not aborts.** Permission denied on a directory gives a + `dir` entry with `error`, and the walk moves on; a root that does not exist + on this machine is reported in the summary and skipped. The host can see + exactly what it could not see. +- **Hashes are reused.** A previous `walk.jsonl` in the output directory is a + cache keyed on `(root, path)` → `(size, mtime_ns)`. An unchanged file keeps + its digest without being read again, so a daily sweep of a large folder + costs one `stat` per file, not one read. `--no-hash` skips hashing (and the + cache) for a fast first survey. + +## Portability + +The stage runs on the owner's machine, whatever it is. + +- **Standard library only**: nothing to install on the machine that holds the + files. +- **Python 3.8 syntax floor**, enforced by `test_syntax_floor.py` at the + repository root, which parses every module in `bastion_agent/` and `pii/` at + 3.8 and denies the newer standard-library calls that parse on 3.8 and then + fail at run time. +- **Roots** may be POSIX absolute paths, `~` or `~/...` (expanded with + `os.path.expanduser`: `HOME` on Unix, `USERPROFILE` on Windows), Windows + drive paths (`C:\...` or `C:/...`) and UNC paths (`\\server\share`). +- **Paths in the map are always `/`-separated**, so a map made on Windows + reads the same as one made on a Mac, and exclude patterns mean the same thing + everywhere. +- **Unicode file names** are written as they are (`ensure_ascii=False`); + undecodable bytes in link targets go through `os.fsdecode`. +- **Timestamps** are nanosecond integers from `st_mtime_ns`, which exists on + all three platforms; the ISO form is derived from it. + +## What the stage does not do + +- It does not read file contents beyond the 8 KiB sniff and the hash stream. + Extraction is the next stage. +- It does not detect changes by itself. Two maps diff trivially because they + are sorted, and later stages find their work from the hashes. +- It does not treat mounts specially. A network share under a root is walked + like any directory; exclude it if that is not wanted. + +## Tests + +`python3 -m unittest bastion_agent.test_walk` covers the map's shape and sort +order, MIME sniffing (magic, extension, empty, binary), excludes (segment, +anchored, `/**` pruning, backslashes, Windows case folding, `.agent` in any +case), the too-large skip, `--no-hash`, hash reuse and its invalidation when a +file changes, byte-identical re-runs, a missing root, an unreadable directory, +Unicode names, `~` expansion, and symlinks recorded but not followed, or +followed when the manifest allows it. The symlink tests are skipped on Windows, +where creating a symlink needs a privilege. diff --git a/manifests/home-documents.yaml b/manifests/home-documents.yaml new file mode 100644 index 0000000..6e38633 --- /dev/null +++ b/manifests/home-documents.yaml @@ -0,0 +1,56 @@ +# bastion agent job manifest — example. Contract: docs/manifest.md. +# Authored by the owner's host, signed with the host's CA-issued key, executed +# verbatim by the bastion agent on this machine. +# Validate: python3 -m bastion_agent validate manifests/*.yaml +freeagent: 1 +id: home-documents +owner: + host: owner-host + +scope: + roots: + - ~/Documents + - /srv/share/family + exclude: + - "**/node_modules/**" + - "**/.git/**" + - "**/*.key" + - "**/*.pem" + follow_symlinks: false + max_file_bytes: 268435456 + +# the ceiling: every stage below must be covered here +mask: [read, search, index, summarize] + +stages: [walk, extract, sanitize, chunk, embed, catalog, seal] + +sanitize: + mode: sterilize # rules + local-model recall + vault; 'flag' = rules only + verify: false + +embed: + model: qwen3-embedding + dim: 1024 + chunk: + size_tokens: 512 + overlap_tokens: 64 + +summarize: + model: qwen3.6-35b-a3b + per: [file, directory] + +sweep: daily + +sink: + catalog: host # sealed catalog goes to the owner's host; corpus stays here + vectors: local + plaintext: never # invariant — the loader refuses any other value + +# access moves, data does not: another host may point its retrieval here +grants: + - host: partner-host + ops: [search] + until: 2026-12-31 + +# signature block added by the signing step (docs/manifest.md); +# the runtime refuses to run without it. diff --git a/pii/Dockerfile b/pii/Dockerfile new file mode 100644 index 0000000..bb8ca91 --- /dev/null +++ b/pii/Dockerfile @@ -0,0 +1,17 @@ +# PII sterilization service (docs/pii-sterilization.md). Runtime is the +# Python stdlib only — no pip install exists in this image on purpose; +# the vault holds the household's most sensitive data and every +# dependency is attack surface. +FROM python:3.13-slim@sha256:6771159cd4fa5d9bba1258caf0b82e6b73458c694d178ad97c5e925c2d0e1a91 + +WORKDIR /app +# every runtime module, named: a module missing here is an ImportError at +# boot (recall.py was, once) +COPY __init__.py detect.py llmclient.py pipeline.py recall.py service.py vault.py /app/pii/ + +# Root inside the container is deliberate: the vault db and its directory +# are created 0700/0600 (service sets umask 077) on the mounted /data, +# and the single-purpose process is bind-scoped by PII_BIND. Revisit if +# the service ever grows a second concern. +ENV PYTHONUNBUFFERED=1 +CMD ["python", "-m", "pii.service"] diff --git a/pii/__init__.py b/pii/__init__.py new file mode 100644 index 0000000..91ae805 --- /dev/null +++ b/pii/__init__.py @@ -0,0 +1,4 @@ +from . import detect, pipeline # noqa: F401 +from .detect import PII_TYPES, Finding # noqa: F401 +from .pipeline import TOKEN_RE, ground, rehydrate, replace_values, sterilize # noqa: F401 +from .vault import Vault # noqa: F401 diff --git a/pii/detect.py b/pii/detect.py new file mode 100644 index 0000000..adfa5d0 --- /dev/null +++ b/pii/detect.py @@ -0,0 +1,249 @@ +"""Deterministic PII detectors — the high-precision anchor pass. + +Two jobs: (1) anchor the sterilizer with detections that never depend on +model behavior, and (2) re-scan sterilized output as the residual check +(pipeline.py loops until this pass finds nothing). Recall for contextual +PII — names, addresses, employers, "my daughter's school" — is the LLM +pass's job (llmclient.py); here precision is deliberately favored over +recall, because a miss still has two more passes to be caught while a +false positive burns a vault row forever. + +Overlapping candidates are resolved by type priority then span length: +an IPv4 address must not be re-claimed as a dotted phone number, a MAC +must not be claimed as truncated IPv6. +""" + +import hashlib +import ipaddress +import re +from dataclasses import dataclass +from typing import List + +# Full type vocabulary — shared with the LLM extraction schema. Rules +# only ever emit the subset in _DETECTORS; the rest are LLM-only types. +PII_TYPES = [ + "PERSON", "EMAIL", "PHONE", "ADDRESS", "SSN", "DOB", + "CREDIT_CARD", "BANK_ACCOUNT", "IP_ADDRESS", "MAC_ADDRESS", + "USERNAME", "CREDENTIAL", "MEDICAL", "ID_NUMBER", "ORG", + "LOCATION", "OTHER", +] + + +@dataclass(frozen=True) +class Finding: + value: str + type: str + start: int + end: int + + +EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)*\.[A-Za-z]{2,}") + +MAC_RE = re.compile( + r"(? bool: + total = 0 + for i, ch in enumerate(reversed(digits)): + d = int(ch) + if i % 2 == 1: + d *= 2 + if d > 9: + d -= 9 + total += d + return total % 10 == 0 + + +def _find_emails(text): + for m in EMAIL_RE.finditer(text): + yield Finding(m.group(0), "EMAIL", m.start(), m.end()) + + +def _find_macs(text): + for m in MAC_RE.finditer(text): + yield Finding(m.group(0), "MAC_ADDRESS", m.start(), m.end()) + + +def _find_ips(text): + for m in IPV4_RE.finditer(text): + try: + ipaddress.IPv4Address(m.group(0)) + except ValueError: + continue + yield Finding(m.group(0), "IP_ADDRESS", m.start(), m.end()) + for m in IPV6_RE.finditer(text): + cand = m.group(0) + try: + ipaddress.IPv6Address(cand) + except ValueError: + continue + yield Finding(cand, "IP_ADDRESS", m.start(), m.end()) + + +def _find_ssns(text): + for m in SSN_RE.finditer(text): + area = int(m.group(1)) + if area == 0 or area == 666 or area >= 900: + continue + yield Finding(m.group(0), "SSN", m.start(), m.end()) + + +def _find_cards(text): + for m in CARD_RE.finditer(text): + digits = re.sub(r"\D", "", m.group(0)) + if 13 <= len(digits) <= 19 and _luhn_ok(digits): + yield Finding(m.group(0), "CREDIT_CARD", m.start(), m.end()) + + +def _find_phones(text): + for m in PHONE_RE.finditer(text): + digits = re.sub(r"\D", "", m.group(0)) + if not 10 <= len(digits) <= 15: + continue + yield Finding(m.group(0), "PHONE", m.start(), m.end()) + + +# Priority order: earlier detectors own their spans outright. Secrets go +# first (a PEM block or JWT contains digit runs that would otherwise be +# claimed piecemeal); IBAN before cards (an IBAN's digits can pass Luhn). +_DETECTORS = [ + _find_secrets, _find_emails, _find_macs, _find_ips, _find_ibans, _find_btc, + _find_ssns, _find_cards, _find_abas, _find_phones, +] + + +def detect(text: str) -> List[Finding]: + """All rule findings, non-overlapping, priority- then position-ordered.""" + candidates = [] + for prio, fn in enumerate(_DETECTORS): + for f in fn(text): + candidates.append((prio, -(f.end - f.start), f.start, f)) + accepted = [] # type: List[Finding] + for _, _, _, f in sorted(candidates, key=lambda c: (c[0], c[1], c[2])): + if all(f.end <= a.start or f.start >= a.end for a in accepted): + accepted.append(f) + return sorted(accepted, key=lambda f: f.start) diff --git a/pii/llmclient.py b/pii/llmclient.py new file mode 100644 index 0000000..9fc9dae --- /dev/null +++ b/pii/llmclient.py @@ -0,0 +1,112 @@ +"""LLM span extraction — the recall pass of the sterilizer. + +The model is a sensor, never an actor: it proposes verbatim substrings +plus a type from the closed vocabulary, grammar-constrained via +llama.cpp's json_schema response_format (off-schema output is +impossible). It never rewrites text — pipeline.py grounds every proposed +span against the input (verbatim occurrence or it is dropped and +counted) and performs all replacement itself, so the model cannot rewrite +the text whatever it outputs. + +Talks OpenAI-style chat completions to a llama.cpp-compatible server at +LLM_URL, with thinking disabled per request via chat_template_kwargs. +Stdlib urllib only. +""" + +import json +import urllib.request +from typing import List, Tuple + +from .detect import PII_TYPES + +Findings = List[Tuple[str, str]] + +_SCHEMA = { + "type": "object", + "properties": { + "findings": { + "type": "array", + "items": { + "type": "object", + "properties": { + "text": {"type": "string"}, + "type": {"type": "string", "enum": PII_TYPES}, + }, + "required": ["text", "type"], + }, + } + }, + "required": ["findings"], +} + +# The exact vocabulary MUST appear in the prompts. The grammar constrains +# type to this enum, but grammar is a backstop, not steering: a model that +# free-texts "School" gets prefix-coerced to "SSN" (seen with a real model). +# test_pii guards prompt/enum drift. +_TYPE_LIST = ", ".join(PII_TYPES) + +EXTRACT_SYSTEM = f"""You are a PII detector for a privacy gateway. List every substring of the user's text that could identify a person, household, account, or device. + +Label each finding with exactly one of these types (use the label verbatim): +{_TYPE_LIST} + +Find: personal names; email addresses; phone numbers; street or postal addresses; government/tax/license/passport ID numbers; credit card and bank account numbers; dates of birth; usernames and social handles; passwords, API keys, and other secrets; medical facts tied to a person; employers, schools, or organizations that identify someone; specific places that identify someone (their home, school, workplace); IP addresses; MAC addresses and device serial numbers. + +Rules: +- Copy each span VERBATIM, character for character, exactly as it appears in the text. Never paraphrase, correct, or reformat. +- Report the minimal identifying span, without surrounding punctuation or filler words. +- If the same value appears multiple times, report it once. +- Do not report placeholder tokens that look like [PII_TYPE_N] — those are already sterilized. +- Generic words, product names, and public facts are not PII. When a span does not identify anyone, leave it out. +- If there is no PII, return an empty findings list.""" + +VERIFY_SYSTEM = f"""You are auditing a sterilized document for residual PII. Placeholder tokens like [PII_EMAIL_1] are already sterilized — ignore them. Report ONLY remaining substrings that could still identify a person, household, account, or device, copied verbatim, each labeled with exactly one of these types (use the label verbatim): {_TYPE_LIST}. Be aggressive: a partial address, a first name, a nickname, an unusual job title can all identify someone. If nothing identifying remains, return an empty findings list.""" + + +class RecallError(RuntimeError): + """The recall pass could not run. Every backend raises this (or a + subclass); the service fails closed on it. Defined here so the older + LlmError keeps its name while sharing the contract.""" + + +class LlmError(RecallError): + pass + + +def _chat(llm_url: str, system: str, text: str, timeout: float) -> Findings: + body = json.dumps({ + "messages": [ + {"role": "system", "content": system}, + {"role": "user", "content": text}, + ], + "temperature": 0, + "chat_template_kwargs": {"enable_thinking": False}, + "response_format": { + "type": "json_schema", + "json_schema": {"name": "pii_findings", "schema": _SCHEMA}, + }, + }).encode() + req = urllib.request.Request( + llm_url.rstrip("/") + "/v1/chat/completions", + data=body, + headers={"Content-Type": "application/json"}, + ) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + payload = json.load(resp) + except OSError as e: + raise LlmError(f"llm unreachable: {e}") from e + try: + content = payload["choices"][0]["message"]["content"] + findings = json.loads(content)["findings"] + return [(f["text"], f["type"]) for f in findings] + except (KeyError, IndexError, TypeError, ValueError) as e: + raise LlmError(f"malformed llm response: {e}") from e + + +def extract(text: str, llm_url: str, timeout: float = 300.0) -> Findings: + return _chat(llm_url, EXTRACT_SYSTEM, text, timeout) + + +def verify(text: str, llm_url: str, timeout: float = 300.0) -> Findings: + return _chat(llm_url, VERIFY_SYSTEM, text, timeout) diff --git a/pii/pii.service b/pii/pii.service new file mode 100644 index 0000000..ae7170e --- /dev/null +++ b/pii/pii.service @@ -0,0 +1,51 @@ +# Standalone systemd packaging for the PII sterilization service, for a +# machine that does not run it as a container (pii/Dockerfile). The service +# is stdlib-only, so "packaging" is: a python3, the pii/ package, this unit, +# and an env file. +# +# Install (as root): +# install -d /opt/privatae-pii +# cp -r pii /opt/privatae-pii/pii # the package, tests and all +# install -m 600 /dev/null /etc/privatae-pii.env +# printf 'PII_API_KEY=%s\nLLM_URL=http://:8090\n' "$(openssl rand -hex 24)" >> /etc/privatae-pii.env +# # no model host (a laptop)? use the in-process CPU sensor instead of LLM_URL: +# # pip install gliner && echo 'PII_RECALL=gliner' >> /etc/privatae-pii.env +# cp /opt/privatae-pii/pii/pii.service /etc/systemd/system/privatae-pii.service +# systemctl daemon-reload && systemctl enable --now privatae-pii +# +# The vault lands in /var/lib/privatae-pii (StateDirectory, 0700, owned +# by the DynamicUser). Bind stays loopback: the bastion forwarder +# (PRIVATAE_PII_ADDR=127.0.0.1:8091) is the reachability story, not the bind. + +[Unit] +Description=privatae PII sterilization service +Documentation=https://git.privatae.ai/privatae/bastion-agent/src/branch/main/docs/pii-sterilization.md +After=network-online.target +Wants=network-online.target + +[Service] +Type=exec +WorkingDirectory=/opt/privatae-pii +Environment=PII_BIND=127.0.0.1:8091 +Environment=PII_DB=/var/lib/privatae-pii/vault.db +EnvironmentFile=/etc/privatae-pii.env +ExecStart=/usr/bin/python3 -m pii.service +Restart=always +RestartSec=2 + +# Containment: no shell, no privileges, private everything, one writable dir. +DynamicUser=yes +StateDirectory=privatae-pii +StateDirectoryMode=0700 +NoNewPrivileges=yes +ProtectSystem=strict +ProtectHome=yes +PrivateTmp=yes +PrivateDevices=yes +ProtectKernelTunables=yes +ProtectControlGroups=yes +RestrictSUIDSGID=yes +SystemCallArchitectures=native + +[Install] +WantedBy=multi-user.target diff --git a/pii/pipeline.py b/pii/pipeline.py new file mode 100644 index 0000000..cd7d888 --- /dev/null +++ b/pii/pipeline.py @@ -0,0 +1,217 @@ +"""Sterilization pipeline — deterministic control flow, LLM as sensor. + +Stages (docs/pii-sterilization.md): + 1. rules pass detect.detect() — precision anchors, model-free + 2. LLM pass llmclient.extract() — recall for contextual PII + 3. grounding every LLM span must occur verbatim in the input; + ungrounded spans are dropped and counted, never trusted + 4. replacement script-side, longest value first, every occurrence + (plain substring, inside longer words too); overlapping + matches merge into one token; never inside an existing + token; placeholder-shaped text in the input is escaped + first (all of it for raw documents, only what the vault + does not hold otherwise); tokens minted by the vault + (stable per scope) + 5. residual loop rules re-run on the output until clean (bounded); + any residual is tokenized, not just reported + 6. LLM verify optional second model pass over the sterilized text; + grounded hits go through replacement again + +The model proposes; this module disposes. Every mutation of the text +happens in replace_values(), which guarantees non-PII bytes are +byte-identical to the input. +""" + +import re + +from . import detect + +TOKEN_RE = re.compile(r"\[PII_([A-Z_]+)_(\d+)\]") +# a token's name without its brackets; as a value it could only ever come +# from the model reading an already-sterilized text +BARE_TOKEN_RE = re.compile(r"PII_[A-Z_]+_\d+") + +_MAX_RESIDUAL_ROUNDS = 3 + + +def _usable(value: str, typ: str) -> bool: + if typ not in detect.PII_TYPES: + return False + if len(value) < 2: + return False + if TOKEN_RE.search(value) or BARE_TOKEN_RE.search(value): + return False + return True + + +def ground(proposed, text): + """Split LLM proposals into (grounded, dropped_count).""" + grounded, dropped = [], 0 + for value, typ in proposed: + if value and value in text and _usable(value, typ): + grounded.append((value, typ)) + else: + dropped += 1 + return grounded, dropped + + +def _occurrences(value, text, blocked): + """Every occurrence of value, overlapping ones included ("aba" twice in + "ababa"), minus any that overlap a blocked span. Overlapping hits are + merged by the caller, so no fragment of a self-overlapping value is + left between two tokens.""" + spans = [] + i = text.find(value) + while i >= 0: + end = i + len(value) + if not any(i < be and end > bs for bs, be in blocked): + spans.append((i, end)) + i = text.find(value, i + 1) + return spans + + +def replace_values(text, values, scope, vault, block_tokens=True): + """Replace every occurrence of each value with its vault token. + + Plain substring matching: a value is replaced inside longer words too + ("Garcia" in "Garcias", "Marisol" in "Marisol_resume.pdf"), because a + missed inflection is a leak and an over-eager match only costs + readability. Longest value first, so "John Smith" is consumed before + "John" can split it. No match may overlap a token already in the text, + so a value like "PERSON" cannot break a placeholder apart. + + Two matches that partly overlap (neither contains the other) become one + token over their union: tokenizing one would leave the other's + remainder in the clear. It takes the longer value's type. + + Every byte outside a replaced span is copied unchanged. + + block_tokens=False is only for escaping placeholder-shaped text in the + input (see sterilize), where the literal itself is the value to replace. + """ + types, order = {}, {} + for value, typ in values: + if value and value not in types: + types[value] = typ + order[value] = len(order) + blocked = [m.span() for m in TOKEN_RE.finditer(text)] if block_tokens else [] + matches = [] + for value in sorted(types, key=lambda v: (-len(v), order[v])): + matches.extend((s, e, value) for s, e in _occurrences(value, text, blocked)) + + # union overlapping matches; the longest (then earliest-given) value names + # the group, and a group wider than it is a merged value of its own + groups = [] + for s, e, value in sorted(matches): + if groups and s < groups[-1][1]: + g = groups[-1] + g[1] = max(g[1], e) + g[2].append(value) + else: + groups.append([s, e, [value]]) + + spans = [] + for s, e, members in groups: + lead = min(members, key=lambda v: (-len(v), order[v])) + value = text[s:e] + if value not in types: + types[value] = types[lead] + order[value] = order[lead] + spans.append((s, e, value)) + + tokens, applied = {}, [] + for value in sorted({v for _, _, v in spans}, key=lambda v: (-len(v), order[v])): + tokens[value] = vault.token_for(scope, types[value], value) + applied.append({"token": tokens[value], "type": types[value]}) + + parts, pos = [], 0 + for s, e, value in spans: + parts.append(text[pos:s]) + parts.append(tokens[value]) + pos = e + parts.append(text[pos:]) + return "".join(parts), applied + + +def sterilize(text, scope, vault, llm_extract, llm_verify=None, escape_held_literals=False): + """Returns (sterilized_text, result_dict). + + escape_held_literals: escape every placeholder-shaped string in the + input, even one the vault holds for this scope. For raw documents, + which never contain this service's own output: a document that quotes + "[PII_EMAIL_1]" must not rehydrate to another document's email. Off for + callers that send already-sterilized text back (a stored conversation on + every save), where a held placeholder is that token and must stay it. + """ + rule_values = [(f.value, f.type) for f in detect.detect(text)] + + llm_values, ungrounded = ground(llm_extract(text), text) + + # Placeholder-shaped text that is not one of this scope's tokens is + # escaped as a value of its own, so rehydrate gives the literal back + # instead of leaving it to collide with a token minted later. Unless the + # caller asks for every literal to be escaped, one the vault holds passes + # through untouched: already-sterilized text is sterilized again (a + # stored conversation on every save), and its tokens must stay the same + # tokens. Each distinct literal is looked up once. + shapes = {m.group(0) for m in TOKEN_RE.finditer(text)} + if not escape_held_literals: + shapes = {lit for lit in shapes if vault.value_for(scope, lit) is None} + literals = [(lit, "OTHER") for lit in sorted(shapes)] + out, escaped = replace_values(text, literals, scope, vault, block_tokens=False) + + # Rules first: replace_values keeps the first type seen for a value, + # and on a duplicate the rule's type is the higher-precision one. + out, applied = replace_values(out, rule_values + llm_values, scope, vault) + + # every finding a later round tokenizes is counted, not only the last + # round's: the round that comes back clean must not reset the total + residual = 0 + for _ in range(_MAX_RESIDUAL_ROUNDS): + leftover = [(f.value, f.type) for f in detect.detect(out)] + if not leftover: + break + residual += len(leftover) + out, more = replace_values(out, leftover, scope, vault) + applied.extend(more) + + verify_hits = 0 + if llm_verify is not None: + vgrounded, _ = ground(llm_verify(out), out) + if vgrounded: + # count what was replaced, not what was proposed: a grounded value + # that only occurs inside an existing token replaces nothing + out, more = replace_values(out, vgrounded, scope, vault) + verify_hits = len(more) + applied.extend(more) + + return out, { + "findings": applied, + "stats": { + "rules": len(rule_values), + "llm": len(llm_values), + "llm_ungrounded": ungrounded, + "residual_tokenized": residual, + "verify_tokenized": verify_hits, + "literal_tokens_escaped": len(escaped), + }, + } + + +def rehydrate(text, scope, vault): + """Replace vault tokens with their original values. Unknown tokens are + left in place and reported — never guessed.""" + unknown = [] + restored = 0 + + def sub(m): + nonlocal restored + value = vault.value_for(scope, m.group(0)) + if value is None: + unknown.append(m.group(0)) + return m.group(0) + restored += 1 + return value + + out = TOKEN_RE.sub(sub, text) + return out, {"restored": restored, "unknown_tokens": sorted(set(unknown))} diff --git a/pii/recall.py b/pii/recall.py new file mode 100644 index 0000000..6f78ab4 --- /dev/null +++ b/pii/recall.py @@ -0,0 +1,157 @@ +"""Recall backends — the sensor behind the sterilizer's second pass. + +pipeline.py needs one thing from a recall backend: a callable that takes +text and returns [(verbatim_span, TYPE), ...] with TYPE from +detect.PII_TYPES. Everything downstream (grounding, replacement, residual +loop) is the same whichever sensor produced the proposals, so swapping the +sensor never changes the safety story: a span that is not byte-for-byte in +the input is dropped, and the script does every mutation. + +Two backends (docs/pii-sterilization.md, "Recall backends"): + + llm the llama.cpp chat-completions path (llmclient.py) — a model + server at LLM_URL, grammar-constrained. Best recall; needs a + machine that can reach one. + gliner an in-process zero-shot NER model (GLiNER, Apache-2.0) run on + CPU — the shape a member's laptop can carry. Optional import: + if PII_RECALL=gliner and the package is absent the service + refuses to START (fail closed at boot, not at first request). + +Selection is by env (PII_RECALL), never by the caller: a request cannot +downgrade the sensor. Stdlib-only here; the gliner package is the one +optional third-party import in the module and it is loaded lazily. +""" + +from typing import Callable, List, Tuple + +from . import llmclient +from .detect import PII_TYPES +from .llmclient import RecallError # noqa: F401 (re-exported: one except-clause for every backend) + +Findings = List[Tuple[str, str]] + + +BACKENDS = ("llm", "gliner") + +# GLiNER is zero-shot: the labels ARE the prompt. Plain-English phrases +# recall better than bare enum names; each maps back to the closed +# vocabulary the vault and the grammar use. "OTHER" is deliberately not a +# label — a catch-all label recalls everything and precision collapses. +GLINER_LABELS = { + "person name": "PERSON", + "email address": "EMAIL", + "phone number": "PHONE", + "street address": "ADDRESS", + "social security number": "SSN", + "date of birth": "DOB", + "credit card number": "CREDIT_CARD", + "bank account number": "BANK_ACCOUNT", + "ip address": "IP_ADDRESS", + "mac address": "MAC_ADDRESS", + "username": "USERNAME", + "password or api key or secret": "CREDENTIAL", + "medical condition or diagnosis": "MEDICAL", + "id number or passport or license number": "ID_NUMBER", + "employer or school or organization": "ORG", + "specific location that identifies a person": "LOCATION", +} +assert set(GLINER_LABELS.values()) <= set(PII_TYPES) + +DEFAULT_GLINER_MODEL = "urchade/gliner_multi_pii-v1" +DEFAULT_GLINER_THRESHOLD = 0.5 +DEFAULT_GLINER_VERIFY_THRESHOLD = 0.3 + + +class LlmBackend: + name = "llm" + + def __init__(self, llm_url, timeout): + self.llm_url = llm_url + self.timeout = timeout + + def extract(self, text): # type: (str) -> Findings + return llmclient.extract(text, llm_url=self.llm_url, timeout=self.timeout) + + def verify(self, text): # type: (str) -> Findings + return llmclient.verify(text, llm_url=self.llm_url, timeout=self.timeout) + + def describe(self): + return "llm=%s" % self.llm_url + + +class GlinerBackend: + """In-process GLiNER. The model object is created once at boot (that is + the fail-closed moment) and shared; predict_entities is called per + request with the label list above.""" + + name = "gliner" + + def __init__(self, model_name, threshold, verify_threshold, loader=None): + self.model_name = model_name + self.threshold = float(threshold) + self.verify_threshold = float(verify_threshold) + load = loader or _load_gliner + try: + self.model = load(model_name) + except ImportError as e: + raise RecallError( + "PII_RECALL=gliner but the 'gliner' package is not installed " + "(pip install gliner); refusing to start without a recall pass" + ) from e + except Exception as e: # model download / load failure + raise RecallError("gliner model %r failed to load: %s" % (model_name, e)) from e + + def _predict(self, text, threshold): # type: (str, float) -> Findings + try: + ents = self.model.predict_entities( + text, list(GLINER_LABELS), threshold=threshold) + except Exception as e: + raise RecallError("gliner prediction failed: %s" % e) from e + out = [] + seen = set() + for ent in ents: + try: + span, label = ent["text"], ent["label"] + except (KeyError, TypeError): + continue + typ = GLINER_LABELS.get(label) + if typ is None or not isinstance(span, str): + continue + key = (span, typ) + if key in seen: + continue + seen.add(key) + out.append(key) + return out + + def extract(self, text): + return self._predict(text, self.threshold) + + def verify(self, text): + return self._predict(text, self.verify_threshold) + + def describe(self): + return "gliner=%s@%.2f" % (self.model_name, self.threshold) + + +def _load_gliner(model_name): + from gliner import GLiNER # optional dependency, imported here only + return GLiNER.from_pretrained(model_name) + + +def build_backend(env, loader=None): + """Select and construct the recall backend from the environment. + Raises RecallError (fail closed) on an unknown backend or a backend + that cannot start.""" + kind = env.get("PII_RECALL", "llm").strip().lower() + if kind == "llm": + return LlmBackend(env.get("LLM_URL", "http://llm:8080"), + float(env.get("PII_LLM_TIMEOUT", "300"))) + if kind == "gliner": + return GlinerBackend( + env.get("GLINER_MODEL", DEFAULT_GLINER_MODEL), + env.get("GLINER_THRESHOLD", DEFAULT_GLINER_THRESHOLD), + env.get("GLINER_VERIFY_THRESHOLD", DEFAULT_GLINER_VERIFY_THRESHOLD), + loader=loader, + ) + raise RecallError("PII_RECALL=%r is not one of %s" % (kind, BACKENDS)) diff --git a/pii/service.py b/pii/service.py new file mode 100644 index 0000000..6e2a116 --- /dev/null +++ b/pii/service.py @@ -0,0 +1,178 @@ +"""PII sterilization service — HTTP surface over pipeline + vault. + +Stdlib-only by design: this process holds the household's most sensitive +data, and zero third-party runtime dependencies is the audit story +(every dependency is attack surface). + +Endpoints (JSON in/out): + GET /health liveness, no auth + GET /scope/ token inventory (types+tokens, no values) + POST /sterilize {text, scope?, verify?, escape_held_literals?} -> {text, findings, stats} + POST /rehydrate {text, scope?} -> {text, restored, unknown_tokens} + POST /flag {text} -> {flagged, counts} rules-only, no model + call, no vault writes — the milliseconds pre-inference check. A clean + /flag does NOT mean no PII (contextual PII needs /sterilize). + +Auth is fail-closed: PII_API_KEY must be set in the environment or the +service refuses to start. PII_ALLOW_NOAUTH=1 +exists for the unit-test harness only. /rehydrate returns REAL values — +callers are trusted surfaces only; every call lands in the vault's audit +table. + +Config (env): PII_BIND (host:port, default 0.0.0.0:8091), LLM_URL +(default http://llm:8080), PII_DB (default /data/vault.db), +PII_LLM_TIMEOUT seconds (default 300). +""" + +import hmac +import json +import os +import sys +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +from . import detect, pipeline, recall +from .vault import Vault + +_MAX_BODY = 2 * 1024 * 1024 + + +class Config: + def __init__(self, env): + self.bind = env.get("PII_BIND", "0.0.0.0:8091") + self.llm_url = env.get("LLM_URL", "http://llm:8080") + self.db = env.get("PII_DB", "/data/vault.db") + self.api_key = env.get("PII_API_KEY", "") + self.llm_timeout = float(env.get("PII_LLM_TIMEOUT", "300")) + if not self.api_key and env.get("PII_ALLOW_NOAUTH") != "1": + raise SystemExit( + "refusing to start: PII_API_KEY unset (fail-closed; " + "PII_ALLOW_NOAUTH=1 is for tests only)" + ) + # The recall sensor is chosen here, at boot, by the operator — never + # per request. A backend that cannot start stops the service. + try: + self.recall = recall.build_backend(env) + except recall.RecallError as e: + raise SystemExit("refusing to start: %s" % e) + + +class Handler(BaseHTTPRequestHandler): + server_version = "privatae-pii" + protocol_version = "HTTP/1.1" + + # injected by build_server + cfg: Config = None + vault: Vault = None + + def _send(self, code, obj): + body = json.dumps(obj).encode() + self.send_response(code) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def _authed(self) -> bool: + if not self.cfg.api_key: + return True # only reachable via PII_ALLOW_NOAUTH (tests) + got = self.headers.get("Authorization", "") + want = f"Bearer {self.cfg.api_key}" + return hmac.compare_digest(got.encode(), want.encode()) + + def do_GET(self): + if self.path == "/health": + self._send(200, {"ok": True}) + return + if not self._authed(): + self._send(401, {"error": "unauthorized"}) + return + if self.path.startswith("/scope/"): + scope = self.path[len("/scope/"):] + self._send(200, {"scope": scope, "tokens": self.vault.scope_tokens(scope)}) + return + self._send(404, {"error": "not found"}) + + def do_POST(self): + if not self._authed(): + self._send(401, {"error": "unauthorized"}) + return + try: + length = int(self.headers.get("Content-Length", "0")) + if length > _MAX_BODY: + self._send(413, {"error": "body too large"}) + return + req = json.loads(self.rfile.read(length)) + text = req["text"] + scope = req.get("scope", "default") + if not isinstance(text, str) or not isinstance(scope, str) or not scope: + raise ValueError("text must be a string, scope a non-empty string") + except (ValueError, KeyError, TypeError) as e: + self._send(400, {"error": f"bad request: {e}"}) + return + + if self.path == "/sterilize": + backend = self.cfg.recall + llm_verify = backend.verify if req.get("verify") else None + escape_all = req.get("escape_held_literals") is True + try: + out, result = pipeline.sterilize( + text, scope, self.vault, backend.extract, llm_verify, + escape_held_literals=escape_all, + ) + except recall.RecallError as e: + # Fail closed: no recall pass means no recall guarantee, so no + # partially-sterilized text ever leaves this process. + self._send(502, {"error": str(e)}) + return + result["stats"]["recall"] = backend.name + # echo the literal rule applied, so a caller that asked for + # escape_held_literals can tell it was honoured + result["stats"]["escape_held_literals"] = escape_all + self.vault.audit(scope, "sterilize", json.dumps(result["stats"])) + self._send(200, {"text": out, **result}) + return + + if self.path == "/flag": + # Rules pass only: deterministic, model-free, no vault rows, no + # audit (nothing minted, nothing recallable). Precision anchors + # only — callers must not treat a clean flag as PII-free. + counts = {} + for f in detect.detect(text): + counts[f.type] = counts.get(f.type, 0) + 1 + self._send(200, {"flagged": bool(counts), "counts": counts}) + return + + if self.path == "/rehydrate": + out, result = pipeline.rehydrate(text, scope, self.vault) + self.vault.audit( + scope, "rehydrate", + json.dumps({"restored": result["restored"], + "unknown": len(result["unknown_tokens"])}), + ) + self._send(200, {"text": out, **result}) + return + + self._send(404, {"error": "not found"}) + + def log_message(self, fmt, *args): + # No request bodies in logs, ever; method+path+status only. + sys.stderr.write("%s %s\n" % (self.address_string(), fmt % args)) + + +def build_server(cfg: Config) -> ThreadingHTTPServer: + os.umask(0o077) + handler = type("BoundHandler", (Handler,), {"cfg": cfg, "vault": Vault(cfg.db)}) + host, port = cfg.bind.rsplit(":", 1) + return ThreadingHTTPServer((host, int(port)), handler) + + +def main(): + cfg = Config(os.environ) + srv = build_server(cfg) + sys.stderr.write("pii service on %s, recall %s, db=%s\n" + % (cfg.bind, cfg.recall.describe(), cfg.db)) + srv.serve_forever() + + +if __name__ == "__main__": + main() diff --git a/pii/test_pii.py b/pii/test_pii.py new file mode 100644 index 0000000..01e75e4 --- /dev/null +++ b/pii/test_pii.py @@ -0,0 +1,687 @@ +"""Unit + wire tests for the PII sterilization service. + +Everything here is model-free: the LLM pass is either a fake callable +(pipeline tests) or a stub llama.cpp speaking the real chat-completions +wire format on loopback (service tests). Live recall quality against a +real model is verify-live.sh's job, against a running service. +""" + +import json +import os +import tempfile +import threading +import unittest +import urllib.error +import urllib.request +from unittest import mock +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +from . import detect, pipeline, service +from .vault import Vault + + +class DetectTests(unittest.TestCase): + def find(self, text): + return {(f.value, f.type) for f in detect.detect(text)} + + def test_email(self): + self.assertIn(("sam.r@example.co.uk", "EMAIL"), + self.find("mail sam.r@example.co.uk today")) + + def test_phone_forms(self): + for s in ["+1 415 555 0173", "(415) 555-0173", "415-555-0173", "415.555.0173"]: + self.assertIn((s, "PHONE"), self.find(f"call {s} now"), s) + + def test_date_is_not_phone(self): + self.assertEqual(self.find("released 2026-08-03 at noon"), set()) + + def test_ssn_valid_and_invalid_area(self): + self.assertIn(("123-45-6789", "SSN"), self.find("ssn 123-45-6789")) + self.assertEqual(self.find("ssn 666-45-6789"), set()) + self.assertEqual(self.find("ssn 000-45-6789"), set()) + + def test_credit_card_luhn(self): + self.assertIn(("4111 1111 1111 1111", "CREDIT_CARD"), + self.find("card 4111 1111 1111 1111 exp 12/28")) + # same shape, fails Luhn + self.assertEqual(self.find("ref 4111 1111 1111 1112"), set()) + + def test_ipv4_owns_dotted_span(self): + found = self.find("host at 192.168.68.54 responded") + self.assertIn(("192.168.68.54", "IP_ADDRESS"), found) + self.assertNotIn(("192.168.68.54", "PHONE"), found) + + def test_ipv4_invalid_octet(self): + self.assertNotIn(("999.168.68.541", "IP_ADDRESS"), self.find("x 999.168.68.541")) + + def test_ipv6_and_mac_disambiguation(self): + found = self.find("fe80::1ff:fe23:4567:890a from aa:bb:cc:dd:ee:ff") + self.assertIn(("fe80::1ff:fe23:4567:890a", "IP_ADDRESS"), found) + self.assertIn(("aa:bb:cc:dd:ee:ff", "MAC_ADDRESS"), found) + + def test_sentence_ending_full_stop(self): + self.assertIn(("415-555-0173", "PHONE"), self.find("Call me at 415-555-0173.")) + self.assertIn(("415.555.0173", "PHONE"), self.find("Call me at 415.555.0173.")) + self.assertIn(("10.1.2.3", "IP_ADDRESS"), self.find("the gateway was 10.1.2.3.")) + self.assertIn(("aa:bb:cc:dd:ee:ff", "MAC_ADDRESS"), self.find("from aa:bb:cc:dd:ee:ff.")) + + def test_full_stop_inside_a_longer_number_still_refuses(self): + self.assertEqual(self.find("version 10.1.2.3.4 shipped"), set()) + self.assertNotIn("PHONE", {t for _, t in self.find("ref 415-555-0173.5 end")}) + self.assertNotIn("PHONE", {t for _, t in self.find("file 415-555-0173.pdf")}) + + def test_full_stop_rule_is_the_same_for_ip_phone_and_mac(self): + # a full stop followed by a letter or digit is part of something longer + self.assertEqual(self.find("open 10.1.2.3.txt now"), set()) + self.assertIn(("10.1.2.3", "IP_ADDRESS"), self.find("the host was 10.1.2.3.")) + self.assertEqual(self.find("saved aa:bb:cc:dd:ee:ff.bak"), set()) + self.assertEqual(self.find("saved aa:bb:cc:dd:ee:ff.txt"), set()) + self.assertIn(("aa:bb:cc:dd:ee:ff", "MAC_ADDRESS"), self.find("it was aa:bb:cc:dd:ee:ff.")) + + # ── v1.2 checksum-anchored recognizers ────────────────────────────── + def test_iban_mod97(self): + for iban in ("GB82 WEST 1234 5698 7654 32", "DE89370400440532013000"): + self.assertIn((iban, "BANK_ACCOUNT"), self.find("pay to %s thanks" % iban), iban) + for bad in ("GB82 WEST 1234 5698 7654 33", # bad check digits + "ZZ82 WEST 1234 5698 7654 32"): # unknown country + self.assertNotIn("BANK_ACCOUNT", {t for _, t in self.find("pay to " + bad)}, bad) + + def test_iban_not_reclaimed_as_card(self): + found = self.find("iban DE89370400440532013000") + self.assertNotIn(("370400440532013000", "CREDIT_CARD"), found) + self.assertEqual({t for _, t in found}, {"BANK_ACCOUNT"}) + + def test_aba_needs_checksum_and_cue(self): + self.assertIn(("021000021", "BANK_ACCOUNT"), self.find("routing number 021000021")) + self.assertIn(("021000021", "BANK_ACCOUNT"), self.find("ABA: 021000021")) + self.assertEqual(self.find("order 021000021 shipped"), set()) # no cue + self.assertEqual(self.find("routing number 021000022"), set()) # bad checksum + + def test_bitcoin_base58check(self): + good = "1BvBMSEYstWetqTFn5Au4m4GFg7xJaNVN2" + self.assertIn((good, "BANK_ACCOUNT"), self.find("send to %s" % good)) + self.assertIn(("1A1zP1eP5QGefi2DMPTfTL5SLmv7DivfNa", "BANK_ACCOUNT"), + self.find("genesis 1A1zP1eP5QGefi2DMPTfTL5SLmv7DivfNa")) + self.assertEqual(self.find("send to 1BvBMSEYstWetqTFn5Au4m4GFg7xJaNVN3"), set()) + + def test_secrets_are_credentials(self): + cases = { + "AKIAIOSFODNN7EXAMPLE": "aws key AKIAIOSFODNN7EXAMPLE", + "ghp_" + "a" * 36: "token ghp_" + "a" * 36 + " end", + "xoxb-1234567890-abcdefghij": "slack xoxb-1234567890-abcdefghij", + "AIza" + "b" * 35: "gkey AIza" + "b" * 35, + "sk_live_" + "c" * 24: "stripe sk_live_" + "c" * 24, + "glpat-" + "d" * 20: "gl glpat-" + "d" * 20, + "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dozjgNryP4J3jVmNHl0w5N_XgL0n3I9PlFUP0THsR8U": + "bearer eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dozjgNryP4J3jVmNHl0w5N_XgL0n3I9PlFUP0THsR8U", + } + for value, text in cases.items(): + self.assertIn((value, "CREDENTIAL"), self.find(text), value) + + def test_pem_block_is_one_credential(self): + pem = "-----BEGIN EC PRIVATE KEY-----\nMHQCAQEEIF12 3456 7890 1234\n-----END EC PRIVATE KEY-----" + found = self.find("key:\n" + pem + "\nok") + self.assertEqual(found, {(pem, "CREDENTIAL")}) + + def test_secrets_do_not_fire_on_prose(self): + self.assertEqual(self.find("the skate park (AKA the bowl) opens at 9"), set()) + + +class VaultTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.vault = Vault(os.path.join(self.tmp.name, "v.db")) + + def tearDown(self): + self.tmp.cleanup() + + def test_stable_token_same_scope(self): + t1 = self.vault.token_for("s1", "EMAIL", "a@b.com") + t2 = self.vault.token_for("s1", "EMAIL", "a@b.com") + self.assertEqual(t1, t2) + self.assertEqual(t1, "[PII_EMAIL_1]") + + def test_scopes_isolated(self): + self.vault.token_for("s1", "EMAIL", "a@b.com") + self.vault.token_for("s1", "EMAIL", "c@d.com") + t_other = self.vault.token_for("s2", "EMAIL", "c@d.com") + self.assertEqual(t_other, "[PII_EMAIL_1]") # own sequence + self.assertIsNone(self.vault.value_for("s2", "[PII_EMAIL_2]")) + + def test_inventory_has_no_values(self): + self.vault.token_for("s1", "PERSON", "Ada Lovelace") + inv = json.dumps(self.vault.scope_tokens("s1")) + self.assertNotIn("Ada", inv) + + +class PipelineTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.vault = Vault(os.path.join(self.tmp.name, "v.db")) + + def tearDown(self): + self.tmp.cleanup() + + def test_grounding_drops_hallucinated_span(self): + text = "call me at 415-555-0173" + fake = lambda t: [("415-555-0173", "PHONE"), ("Bob Jones", "PERSON")] + out, res = pipeline.sterilize(text, "s", self.vault, fake) + self.assertEqual(res["stats"]["llm_ungrounded"], 1) + self.assertNotIn("Bob", out) + self.assertNotIn("415-555-0173", out) + + def test_longest_value_wins(self): + text = "John Smith emailed. John was brief." + fake = lambda t: [("John Smith", "PERSON"), ("John", "PERSON")] + out, _ = pipeline.sterilize(text, "s", self.vault, fake) + self.assertEqual(out, "[PII_PERSON_1] emailed. [PII_PERSON_2] was brief.") + + def test_non_pii_bytes_identical(self): + text = "The nftables ruleset, with odd spacing\tand café unicode, is fine." + out, res = pipeline.sterilize(text, "s", self.vault, lambda t: []) + self.assertEqual(out, text) + self.assertEqual(res["findings"], []) + + def test_rules_catch_what_llm_missed(self): + text = "ssn is 123-45-6789 ok" + out, res = pipeline.sterilize(text, "s", self.vault, lambda t: []) + self.assertEqual(out, "ssn is [PII_SSN_1] ok") + self.assertEqual(res["stats"]["rules"], 1) + + def test_round_trip(self): + text = "Sam Rivera (sam.r@example.com, 415-555-0173) lives at 4012 Maple Court." + fake = lambda t: [("Sam Rivera", "PERSON"), ("4012 Maple Court", "ADDRESS")] + out, _ = pipeline.sterilize(text, "s", self.vault, fake) + self.assertNotIn("Maple", out) + self.assertNotIn("sam.r", out) + back, res = pipeline.rehydrate(out, "s", self.vault) + self.assertEqual(back, text) + self.assertEqual(res["unknown_tokens"], []) + + def test_rehydrate_unknown_token_left_alone(self): + out, res = pipeline.rehydrate("hello [PII_EMAIL_9]", "s", self.vault) + self.assertEqual(out, "hello [PII_EMAIL_9]") + self.assertEqual(res["unknown_tokens"], ["[PII_EMAIL_9]"]) + + def test_stable_across_calls(self): + fake = lambda t: [] + out1, _ = pipeline.sterilize("mail a@b.com", "s", self.vault, fake) + out2, _ = pipeline.sterilize("again a@b.com!", "s", self.vault, fake) + self.assertIn("[PII_EMAIL_1]", out1) + self.assertIn("[PII_EMAIL_1]", out2) + + def test_rules_type_wins_on_duplicate_value(self): + # LLM mistyping a value the rules also caught must not change the + # minted type — rules are the higher-precision typer. + text = "reach me at 415-555-0173" + fake = lambda t: [("415-555-0173", "ID_NUMBER")] + _, res = pipeline.sterilize(text, "s", self.vault, fake) + self.assertEqual(res["findings"], [{"token": "[PII_PHONE_1]", "type": "PHONE"}]) + + def assertTokenizedAndRoundTrips(self, text, values, secret, verify=None, label=""): + vault = Vault(os.path.join(self.tmp.name, "case-%d.db" % self._n)) + self._n += 1 + out, res = pipeline.sterilize(text, "s", vault, lambda t: list(values), verify) + self.assertNotIn(secret, out, "%s leaked: %r" % (label, out)) + back, _ = pipeline.rehydrate(out, "s", vault) + self.assertEqual(back, text, "%s round trip: %r" % (label, out)) + return out, res + + # A value is replaced as a plain substring, inside longer words too: + # an inflection, a file name or a handle that keeps the value is still + # the value. Each case must tokenize and rehydrate byte-identically. + SUBSTRING_CASES = [ + ("possessive", "Marisol's file is here", [("Marisol", "PERSON")], "Marisol"), + ("possessive curly", "Marisol\u2019s file", [("Marisol", "PERSON")], "Marisol"), + ("hyphen", "Marisol-Garcia signed", [("Marisol", "PERSON")], "Marisol"), + ("punctuation", "Ask Marisol. Then Marisol, Marisol!", [("Marisol", "PERSON")], "Marisol"), + ("line start and end", "Marisol\nhi\nbye Marisol", [("Marisol", "PERSON")], "Marisol"), + ("email local part", "write marisol.garcia@acme.org or Marisol", [("Marisol", "PERSON")], "Marisol"), + ("url path", "see https://acme.org/team/Marisol/cv", [("Marisol", "PERSON")], "Marisol"), + ("underscore file name", "attached Marisol_Garcia_resume.pdf", [("Marisol", "PERSON")], "Marisol"), + ("underscore username", "login marisol_g from home", [("marisol", "USERNAME")], "marisol"), + ("digit suffix username", "user jsmith42 logged in", [("jsmith", "USERNAME")], "jsmith"), + ("camelCase", "folder MarisolGarcia/", [("Marisol", "PERSON")], "Marisol"), + ("plural surname", "the Garcias came over", [("Garcia", "PERSON")], "Garcia"), + ("inside another word", "Ann flew to Annapolis", [("Ann", "PERSON")], "Ann"), + ("german genitive", "Annas Auto ist rot", [("Anna", "PERSON")], "Anna"), + ("finnish case", "Kerro Marisolille", [("Marisol", "PERSON")], "Marisol"), + ("hungarian case", "Marisolnak adtam", [("Marisol", "PERSON")], "Marisol"), + ("hashtag and digits", "#Marisol2024 party", [("Marisol", "PERSON")], "Marisol"), + ("diacritics", "Jos\u00e9 and Zo\u00eb's", [("Jos\u00e9", "PERSON"), ("Zo\u00eb", "PERSON")], "Jos\u00e9"), + ("decomposed accent", "Jose\u0301 came", [("Jose", "PERSON")], "Jose"), + ("apostrophe name", "O'Brien's O'Brien", [("O'Brien", "PERSON")], "O'Brien"), + ("value with digit", "Room 101B and apt 101", [("101B", "ADDRESS")], "101B"), + ("next to a placeholder", "m@x.com Marisol m@x.comMarisol", [("Marisol", "PERSON")], "Marisol"), + ("chinese", "\u5f20\u4f1f\u4f4f\u5728\u5317\u4eac\uff0c\u5f20\u4f1f\u660e\u4e5f\u5728", + [("\u5f20\u4f1f", "PERSON")], "\u5f20\u4f1f"), + ("latin next to chinese", "Ann\u4f4f\u5728\u5317\u4eac Ann's", [("Ann", "PERSON")], "Ann"), + ("halfwidth katakana", "\uff80\uff85\uff76\u3055\u3093 \uff80\uff85\uff76\uff80\uff9b\uff73", + [("\uff80\uff85\uff76", "PERSON")], "\uff80\uff85\uff76"), + ("thai", "\u0e2a\u0e21\u0e0a\u0e32\u0e22\u0e44\u0e1b\u0e1a\u0e49\u0e32\u0e19", + [("\u0e2a\u0e21\u0e0a\u0e32\u0e22", "PERSON")], "\u0e2a\u0e21\u0e0a\u0e32\u0e22"), + ("korean particle", "\ubbfc\uc218\ub294 \ud559\uad50\uc5d0", [("\ubbfc\uc218", "PERSON")], "\ubbfc\uc218"), + ("devanagari suffix", "\u0930\u093e\u092e\u0915\u094b \u0918\u0930", + [("\u0930\u093e\u092e", "PERSON")], "\u0930\u093e\u092e"), + ("tai tham", "\u1a20\u1a21\u1a22\u1a23", [("\u1a20\u1a21", "PERSON")], "\u1a20\u1a21"), + ("model value partly overlaps a rule hit", "call 415-555-0173 Main St", + [("0173 Main St", "ADDRESS")], "Main St"), + ("model value covers a rule hit", "Marisol ", + [("Marisol ", "OTHER")], "m@example.com"), + # rule findings: every copy of the value goes, not only the detected one + ("repeated phone", "Call 415-555-0173 or text 415-555-0173.", [], "415-555-0173"), + ("repeated ip, sentence end", "ip 10.1.2.3 and gateway was 10.1.2.3.", [], "10.1.2.3"), + ("ssn copy with suffix", "ssn 123-45-6789 / ref 123-45-6789-A", [], "123-45-6789"), + ("phone copy glued to a word", "tel:415-555-0173 ph415-555-0173", [], "415-555-0173"), + ("phone at sentence end", "Call me at 415-555-0173.", [], "415-555-0173"), + ] + + def test_substring_cases_tokenize_and_round_trip(self): + self._n = 0 + for label, text, values, secret in self.SUBSTRING_CASES: + with self.subTest(label): + self.assertTokenizedAndRoundTrips(text, values, secret, label=label) + + def test_substring_match_is_counted_in_findings(self): + self._n = 0 + _, res = self.assertTokenizedAndRoundTrips( + "the Garcias came over", [("Garcia", "PERSON")], "Garcia") + self.assertEqual(res["findings"], [{"token": "[PII_PERSON_1]", "type": "PERSON"}]) + + def test_partial_overlap_becomes_one_token(self): + # the model value overlaps the rule hit from the right, then from the + # left; either way one token over the union, rehydrating to the bytes + cases = [ + ("call 415-555-0173 Main St", [("0173 Main St", "ADDRESS")], + "415-555-0173 Main St", "call [PII_PHONE_1]"), + ("at Main St 415-555-0173 today", [("Main St 415", "ADDRESS")], + "Main St 415-555-0173", "at [PII_PHONE_1] today"), + ] + for i, (text, values, merged, want) in enumerate(cases): + with self.subTest(text): + vault = Vault(os.path.join(self.tmp.name, "po-%d.db" % i)) + out, _ = pipeline.sterilize(text, "s", vault, lambda t, v=values: v) + self.assertEqual(out, want) + self.assertEqual(vault.value_for("s", "[PII_PHONE_1]"), merged) + back, res = pipeline.rehydrate(out, "s", vault) + self.assertEqual(back, text) + self.assertEqual((res["restored"], res["unknown_tokens"]), (1, [])) + + def test_self_overlapping_value_leaves_no_fragment(self): + for text, value, want in (("x ababa x", "aba", "x [PII_PERSON_1] x"), + ("id 121212 end", "1212", "id [PII_ID_NUMBER_1] end")): + with self.subTest(text): + vault = Vault(os.path.join(self.tmp.name, "so-%s.db" % value)) + typ = "ID_NUMBER" if value.isdigit() else "PERSON" + out, _ = pipeline.sterilize(text, "s", vault, lambda t, v=value, ty=typ: [(v, ty)]) + self.assertEqual(out, want) + back, _ = pipeline.rehydrate(out, "s", vault) + self.assertEqual(back, text) + + def test_never_minted_placeholder_in_the_source_survives_the_round_trip(self): + # placeholder-shaped text the vault does not hold is escaped, so it + # comes back as written, card-like digits inside it included + pipeline.sterilize("Marisol called", "s", self.vault, lambda t: [("Marisol", "PERSON")]) + text = "log: [PII_EMAIL_7], [PII_FOO_99] and [PII_A_4111111111111111], sent by Ana" + out, res = pipeline.sterilize(text, "s", self.vault, lambda t: [("Ana", "PERSON")]) + for lit in ("[PII_EMAIL_7]", "[PII_FOO_99]", "[PII_A_4111111111111111]"): + self.assertNotIn(lit, out) + self.assertEqual(res["stats"]["literal_tokens_escaped"], 3) + self.assertEqual(res["findings"], [{"token": "[PII_PERSON_2]", "type": "PERSON"}]) + back, info = pipeline.rehydrate(out, "s", self.vault) + self.assertEqual(back, text) + self.assertEqual(info["unknown_tokens"], []) + + def test_live_token_in_the_source_passes_through(self): + # a placeholder the vault holds for this scope is treated as the token + # it names: a quoted "[PII_PERSON_1]" rehydrates to the vault value. + # That is the documented limit; it is what keeps re-sterilizing + # already-sterilized text stable. + pipeline.sterilize("Marisol called", "s", self.vault, lambda t: [("Marisol", "PERSON")]) + text = "the log shows [PII_PERSON_1]" + out, res = pipeline.sterilize(text, "s", self.vault, lambda t: []) + self.assertEqual(out, text) + self.assertEqual(res["stats"]["literal_tokens_escaped"], 0) + self.assertEqual(pipeline.rehydrate(out, "s", self.vault)[0], "the log shows Marisol") + + def test_escape_held_literals_escapes_a_live_placeholder(self): + # raw documents: a quoted placeholder is never another text's value + pipeline.sterilize("mail ana@x.org", "s", self.vault, lambda t: []) + text = "quoted [PII_EMAIL_1] here" + kept, _ = pipeline.sterilize(text, "s", self.vault, lambda t: []) + self.assertEqual(kept, text) # default: the live token + out, res = pipeline.sterilize(text, "s", self.vault, lambda t: [], escape_held_literals=True) + self.assertNotIn("[PII_EMAIL_1]", out) + self.assertEqual(res["stats"]["literal_tokens_escaped"], 1) + self.assertEqual(pipeline.rehydrate(out, "s", self.vault)[0], text) + + def test_each_distinct_placeholder_is_looked_up_once(self): + text = " ".join(["[PII_EMAIL_%d]" % (i % 3) for i in range(300)]) + calls = [] + real = self.vault.value_for + + def counting(scope, token): + calls.append(token) + return real(scope, token) + + with mock.patch.object(self.vault, "value_for", counting): + pipeline.sterilize(text, "s", self.vault, lambda t: []) + self.assertEqual(sorted(calls), ["[PII_EMAIL_0]", "[PII_EMAIL_1]", "[PII_EMAIL_2]"]) + + def test_resterilizing_stored_text_is_stable(self): + # a stored conversation is sent back to sterilize on every save: + # three load/append/save rounds, then one rehydrate gives the original + scope = "thread:abc" + msgs = [{"role": "user", "content": "Email Marisol at marisol@acme.org"}] + model = lambda t: [("Marisol", "PERSON")] if "Marisol" in t else [] + stored, _ = pipeline.sterilize(json.dumps(msgs), scope, self.vault, model) + rows_after_first_save = len(self.vault.scope_tokens(scope)) + for i in range(3): + loaded = json.loads(stored) + loaded.append({"role": "assistant", "content": "ok %d" % i}) + msgs.append({"role": "assistant", "content": "ok %d" % i}) + stored, res = pipeline.sterilize(json.dumps(loaded), scope, self.vault, model) + self.assertEqual(res["stats"]["literal_tokens_escaped"], 0) + self.assertEqual(json.loads(stored)[0]["content"], "Email [PII_PERSON_1] at [PII_EMAIL_1]") + self.assertEqual(len(self.vault.scope_tokens(scope)), rows_after_first_save) + back, _ = pipeline.rehydrate(stored, scope, self.vault) + self.assertEqual(json.loads(back), msgs) + + def test_touching_matches_stay_two_tokens(self): + # adjacent is not overlapping: each value keeps its own token + text = "MarisolLopez and LopezMarisol" + out, res = pipeline.sterilize(text, "s", self.vault, + lambda t: [("Marisol", "PERSON"), ("Lopez", "PERSON")]) + self.assertEqual(out, "[PII_PERSON_1][PII_PERSON_2] and [PII_PERSON_2][PII_PERSON_1]") + self.assertEqual(len(res["findings"]), 2) + back, _ = pipeline.rehydrate(out, "s", self.vault) + self.assertEqual(back, text) + + def test_verify_value_cannot_break_a_token(self): + # a verify pass reading sterilized text may propose pieces of a + # placeholder; none may split one, and the round trip must hold + self._n = 0 + for piece in ("PII_PERSON_1", "PERSON", "PII", "_1"): + with self.subTest(piece): + out, _ = self.assertTokenizedAndRoundTrips( + "Marisol called", [("Marisol", "PERSON")], "Marisol", + verify=lambda t, piece=piece: [(piece, "OTHER")], label=piece) + self.assertEqual(out, "[PII_PERSON_1] called") + + def test_bare_token_name_is_not_usable(self): + grounded, dropped = pipeline.ground([("PII_PERSON_1", "OTHER"), ("x PII_EMAIL_2 y", "OTHER")], + "PII_PERSON_1 x PII_EMAIL_2 y") + self.assertEqual((grounded, dropped), ([], 2)) + + def test_residuals_are_counted(self): + # the first rules pass misses the email; the residual loop must + # tokenize it AND report it, even though the next round is clean + real = detect.detect + calls = [] + + def first_pass_blind(text): + calls.append(text) + return [] if len(calls) == 1 else real(text) + + with mock.patch.object(detect, "detect", first_pass_blind): + out, res = pipeline.sterilize("mail a@b.com and c@d.com", "s", self.vault, lambda t: []) + self.assertEqual(out, "mail [PII_EMAIL_1] and [PII_EMAIL_2]") + self.assertEqual(res["stats"]["residual_tokenized"], 2) + self.assertEqual(res["stats"]["rules"], 0) + + def test_residuals_summed_across_rounds(self): + # two residual rounds that each find something: the stat is their + # sum, not the last round's count + real = detect.detect + calls = [] + + def one_at_a_time(text): + calls.append(text) + if len(calls) == 1: + return [] + return real(text)[:1] + + with mock.patch.object(detect, "detect", one_at_a_time): + out, res = pipeline.sterilize("mail a@b.com and c@d.com", "s", self.vault, lambda t: []) + self.assertEqual(out, "mail [PII_EMAIL_1] and [PII_EMAIL_2]") + self.assertEqual(res["stats"]["residual_tokenized"], 2) + + def test_no_residuals_reports_zero(self): + _, res = pipeline.sterilize("ssn 123-45-6789", "s", self.vault, lambda t: []) + self.assertEqual(res["stats"]["residual_tokenized"], 0) + + def test_verify_pass_tokenizes_grounded_hits(self): + text = "the patient is Marisol" + fake_extract = lambda t: [] + fake_verify = lambda t: [("Marisol", "PERSON")] + out, res = pipeline.sterilize(text, "s", self.vault, fake_extract, fake_verify) + self.assertEqual(out, "the patient is [PII_PERSON_1]") + self.assertEqual(res["stats"]["verify_tokenized"], 1) + + def test_verify_counts_only_what_it_replaced(self): + # grounded but replacing nothing: a piece of an existing token, and a + # value already tokenized by the first pass + fake_verify = lambda t: [("PERSON", "OTHER"), ("Marisol", "PERSON"), ("Lopez", "PERSON")] + out, res = pipeline.sterilize("Marisol Lopez called", "s", self.vault, + lambda t: [("Marisol", "PERSON")], fake_verify) + self.assertEqual(out, "[PII_PERSON_1] [PII_PERSON_2] called") + self.assertEqual(res["stats"]["verify_tokenized"], 1) + + +class PromptContractTests(unittest.TestCase): + def test_prompts_state_full_type_vocabulary(self): + # The grammar prefix-coerces free-text labels to enum members + # ("School" -> SSN, observed live). The prompts must therefore + # state every legal label verbatim; this guards enum/prompt drift. + from . import llmclient + for prompt in (llmclient.EXTRACT_SYSTEM, llmclient.VERIFY_SYSTEM): + for typ in detect.PII_TYPES: + self.assertIn(typ, prompt) + + +class _FakeGliner: + """Stands in for gliner.GLiNER: returns canned entities, records labels.""" + entities = [] + calls = [] + + def predict_entities(self, text, labels, threshold=0.5): + _FakeGliner.calls.append((labels, threshold)) + return list(self.entities) + + +class RecallBackendTests(unittest.TestCase): + def setUp(self): + from . import recall + self.recall = recall + _FakeGliner.entities, _FakeGliner.calls = [], [] + + def test_default_is_llm(self): + b = self.recall.build_backend({"LLM_URL": "http://x:1"}) + self.assertEqual(b.name, "llm") + + def test_unknown_backend_refused(self): + with self.assertRaises(self.recall.RecallError): + self.recall.build_backend({"PII_RECALL": "cloud"}) + + def test_gliner_missing_package_fails_closed(self): + def loader(_): + raise ImportError("No module named 'gliner'") + with self.assertRaises(self.recall.RecallError) as ctx: + self.recall.build_backend({"PII_RECALL": "gliner"}, loader=loader) + self.assertIn("pip install gliner", str(ctx.exception)) + + def test_gliner_maps_labels_to_vocabulary_and_dedupes(self): + _FakeGliner.entities = [ + {"text": "Marisol", "label": "person name", "score": 0.9}, + {"text": "Marisol", "label": "person name", "score": 0.8}, + {"text": "Lincoln Elementary", "label": "employer or school or organization"}, + {"text": "junk", "label": "not a label we asked for"}, + ] + b = self.recall.build_backend( + {"PII_RECALL": "gliner", "GLINER_THRESHOLD": "0.6"}, loader=lambda _: _FakeGliner()) + out = b.extract("Marisol goes to Lincoln Elementary") + self.assertEqual(out, [("Marisol", "PERSON"), ("Lincoln Elementary", "ORG")]) + labels, threshold = _FakeGliner.calls[-1] + self.assertEqual(set(labels), set(self.recall.GLINER_LABELS)) + self.assertEqual(threshold, 0.6) + for typ in set(self.recall.GLINER_LABELS.values()): + self.assertIn(typ, detect.PII_TYPES) + + def test_gliner_verify_uses_lower_threshold(self): + b = self.recall.build_backend({"PII_RECALL": "gliner"}, loader=lambda _: _FakeGliner()) + b.verify("x") + self.assertEqual(_FakeGliner.calls[-1][1], self.recall.DEFAULT_GLINER_VERIFY_THRESHOLD) + + def test_gliner_prediction_failure_is_recall_error(self): + class Broken: + def predict_entities(self, *a, **k): + raise RuntimeError("cuda gone") + b = self.recall.build_backend({"PII_RECALL": "gliner"}, loader=lambda _: Broken()) + with self.assertRaises(self.recall.RecallError): + b.extract("x") + + def test_llm_error_is_a_recall_error(self): + from . import llmclient + self.assertTrue(issubclass(llmclient.LlmError, self.recall.RecallError)) + + def test_gliner_backend_through_pipeline(self): + _FakeGliner.entities = [{"text": "Marisol", "label": "person name"}, + {"text": "Ghost", "label": "person name"}] + b = self.recall.build_backend({"PII_RECALL": "gliner"}, loader=lambda _: _FakeGliner()) + tmp = tempfile.TemporaryDirectory() + try: + vault = Vault(os.path.join(tmp.name, "v.db")) + out, res = pipeline.sterilize("Marisol called 415-555-0173", "s", vault, b.extract) + finally: + tmp.cleanup() + self.assertEqual(out, "[PII_PERSON_1] called [PII_PHONE_1]") + self.assertEqual(res["stats"]["llm_ungrounded"], 1) # "Ghost" was not in the text + + +class _StubLlm(BaseHTTPRequestHandler): + """Speaks llama.cpp's chat-completions shape; extraction canned per test.""" + findings = [] + + def do_POST(self): + length = int(self.headers.get("Content-Length", "0")) + self.rfile.read(length) + content = json.dumps({"findings": self.findings}) + body = json.dumps( + {"choices": [{"message": {"content": content}}]} + ).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *a): + pass + + +class ServiceTests(unittest.TestCase): + KEY = "test-key-123" + + @classmethod + def setUpClass(cls): + cls.stub = ThreadingHTTPServer(("127.0.0.1", 0), _StubLlm) + threading.Thread(target=cls.stub.serve_forever, daemon=True).start() + cls.tmp = tempfile.TemporaryDirectory() + cfg = service.Config({ + "PII_BIND": "127.0.0.1:0", + "LLM_URL": f"http://127.0.0.1:{cls.stub.server_address[1]}", + "PII_DB": os.path.join(cls.tmp.name, "v.db"), + "PII_API_KEY": cls.KEY, + }) + cls.srv = service.build_server(cfg) + cls.port = cls.srv.server_address[1] + threading.Thread(target=cls.srv.serve_forever, daemon=True).start() + + @classmethod + def tearDownClass(cls): + cls.srv.shutdown() + cls.stub.shutdown() + cls.tmp.cleanup() + + def call(self, path, payload=None, key=None, method=None): + url = f"http://127.0.0.1:{self.port}{path}" + data = json.dumps(payload).encode() if payload is not None else None + req = urllib.request.Request(url, data=data, method=method) + req.add_header("Content-Type", "application/json") + if key: + req.add_header("Authorization", f"Bearer {key}") + with urllib.request.urlopen(req, timeout=10) as r: + return r.status, json.load(r) + + def test_health_no_auth(self): + status, body = self.call("/health") + self.assertEqual((status, body["ok"]), (200, True)) + + def test_auth_required(self): + with self.assertRaises(urllib.error.HTTPError) as ctx: + self.call("/sterilize", {"text": "x"}, key="wrong") + self.assertEqual(ctx.exception.code, 401) + + def test_sterilize_rehydrate_over_http(self): + _StubLlm.findings = [{"text": "Marisol", "type": "PERSON"}] + original = "Marisol's ssn is 123-45-6789, mail her at m@example.com" + _, st = self.call("/sterilize", {"text": original, "scope": "conv1"}, key=self.KEY) + for leak in ("Marisol", "123-45-6789", "m@example.com"): + self.assertNotIn(leak, st["text"]) + self.assertEqual(st["stats"]["rules"], 2) + _, back = self.call("/rehydrate", {"text": st["text"], "scope": "conv1"}, key=self.KEY) + self.assertEqual(back["text"], original) + # wrong scope must not resolve + _, other = self.call("/rehydrate", {"text": st["text"], "scope": "conv2"}, key=self.KEY) + self.assertEqual(other["restored"], 0) + + def test_flag_rules_only_no_vault_rows(self): + _StubLlm.findings = [{"text": "should never be called", "type": "PERSON"}] + _, body = self.call("/flag", {"text": "ssn 123-45-6789 and a@b.com"}, key=self.KEY) + self.assertTrue(body["flagged"]) + self.assertEqual(body["counts"], {"SSN": 1, "EMAIL": 1}) + _, clean = self.call("/flag", {"text": "nothing sensitive here"}, key=self.KEY) + self.assertEqual((clean["flagged"], clean["counts"]), (False, {})) + # no scope was touched, nothing minted + _, inv = self.call("/scope/default", key=self.KEY) + self.assertEqual(inv["tokens"], []) + + def test_escape_held_literals_over_http(self): + _StubLlm.findings = [] + self.call("/sterilize", {"text": "mail a@b.com", "scope": "esc"}, key=self.KEY) + quote = "quoted [PII_EMAIL_1]" + _, kept = self.call("/sterilize", {"text": quote, "scope": "esc"}, key=self.KEY) + self.assertEqual(kept["text"], quote) # default: a re-save keeps the token + _, esc = self.call("/sterilize", {"text": quote, "scope": "esc", + "escape_held_literals": True}, key=self.KEY) + self.assertNotIn("[PII_EMAIL_1]", esc["text"]) + _, back = self.call("/rehydrate", {"text": esc["text"], "scope": "esc"}, key=self.KEY) + self.assertEqual(back["text"], quote) + # the rule applied is echoed, so a caller can tell it was honoured + self.assertIs(kept["stats"]["escape_held_literals"], False) + self.assertIs(esc["stats"]["escape_held_literals"], True) + + def test_scope_inventory_no_values(self): + _StubLlm.findings = [] + self.call("/sterilize", {"text": "a@b.com", "scope": "inv"}, key=self.KEY) + _, body = self.call("/scope/inv", key=self.KEY) + self.assertEqual(body["tokens"][0]["token"], "[PII_EMAIL_1]") + self.assertNotIn("a@b.com", json.dumps(body)) + + def test_fail_closed_without_key(self): + with self.assertRaises(SystemExit): + service.Config({"PII_DB": "x"}) + + def test_fail_closed_on_unstartable_recall_backend(self): + with self.assertRaises(SystemExit) as ctx: + service.Config({"PII_DB": "x", "PII_API_KEY": "k", "PII_RECALL": "nope"}) + self.assertIn("PII_RECALL", str(ctx.exception)) + + def test_sterilize_reports_recall_backend(self): + _StubLlm.findings = [] + _, st = self.call("/sterilize", {"text": "a@b.com", "scope": "rb"}, key=self.KEY) + self.assertEqual(st["stats"]["recall"], "llm") + + +if __name__ == "__main__": + unittest.main() diff --git a/pii/vault.py b/pii/vault.py new file mode 100644 index 0000000..ea5311e --- /dev/null +++ b/pii/vault.py @@ -0,0 +1,100 @@ +"""Local PII vault — token <-> value mapping plus audit trail, SQLite. + +Tokens are stable per (scope, type, value): the same email in the same +scope always maps to the same token, so a cloud model can reason about +[PII_PERSON_1] coherently across a whole conversation. The same value in +a DIFFERENT scope gets a different token — scopes exist so two +conversations cannot be cross-linked through shared placeholders. + +Sequence numbers are per (scope, type) and rows are never deleted, so +COUNT()+1 is a safe next-sequence source under the module lock. After a +delete, COUNT()+1 names a token that still exists unless the deleted row +was the newest, and PRIMARY KEY (scope, token) then makes the insert fail; +if it was the newest, its token is minted again for a different value. +So erasure needs a separate sequence first. Values are plaintext in the file: the controls are the +file system (0600, umask set by the caller) and, for the service, its +bind address. Encryption at rest would be a design change of its own, not +something to improvise here. +""" + +import datetime +import os +import sqlite3 +import threading + +_SCHEMA = """ +CREATE TABLE IF NOT EXISTS pii ( + scope TEXT NOT NULL, + token TEXT NOT NULL, + type TEXT NOT NULL, + value TEXT NOT NULL, + created TEXT NOT NULL, + PRIMARY KEY (scope, token), + UNIQUE (scope, type, value) +); +CREATE TABLE IF NOT EXISTS audit ( + ts TEXT NOT NULL, + scope TEXT NOT NULL, + action TEXT NOT NULL, + detail TEXT NOT NULL +); +""" + + +def _now() -> str: + return datetime.datetime.now(datetime.timezone.utc).isoformat(timespec="seconds") + + +class Vault: + def __init__(self, path: str): + self.path = path + self._lock = threading.Lock() + d = os.path.dirname(path) + if d: + os.makedirs(d, mode=0o700, exist_ok=True) + with self._conn() as c: + c.executescript(_SCHEMA) + + def _conn(self) -> sqlite3.Connection: + return sqlite3.connect(self.path, timeout=30) + + def token_for(self, scope: str, typ: str, value: str) -> str: + with self._lock, self._conn() as c: + row = c.execute( + "SELECT token FROM pii WHERE scope=? AND type=? AND value=?", + (scope, typ, value), + ).fetchone() + if row: + return row[0] + n = c.execute( + "SELECT COUNT(*) FROM pii WHERE scope=? AND type=?", (scope, typ) + ).fetchone()[0] + 1 + token = f"[PII_{typ}_{n}]" + c.execute( + "INSERT INTO pii (scope, token, type, value, created) VALUES (?,?,?,?,?)", + (scope, token, typ, value, _now()), + ) + return token + + def value_for(self, scope: str, token: str): + with self._lock, self._conn() as c: + row = c.execute( + "SELECT value FROM pii WHERE scope=? AND token=?", (scope, token) + ).fetchone() + return row[0] if row else None + + def scope_tokens(self, scope: str) -> list: + """Token inventory for a scope — types and tokens only, never values.""" + with self._lock, self._conn() as c: + rows = c.execute( + "SELECT token, type, created FROM pii WHERE scope=? ORDER BY created, token", + (scope,), + ).fetchall() + return [{"token": t, "type": ty, "created": cr} for t, ty, cr in rows] + + def audit(self, scope: str, action: str, detail: str): + with self._lock, self._conn() as c: + c.execute( + "INSERT INTO audit (ts, scope, action, detail) VALUES (?,?,?,?)", + (_now(), scope, action, detail), + ) diff --git a/pii/verify-live.sh b/pii/verify-live.sh new file mode 100755 index 0000000..70701f3 --- /dev/null +++ b/pii/verify-live.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# Live verification of a running PII sterilization service and its real +# recall model (the unit tests use a stub; this is the check against the +# model itself): +# +# PII_API_KEY= pii/verify-live.sh [host:port] +# +# host:port defaults to 127.0.0.1:8091, where pii.service binds. +set -u +HOST="${1:-127.0.0.1:8091}" +KEY="${PII_API_KEY:-}" +[ -n "$KEY" ] || { echo "set PII_API_KEY to the service's key"; exit 1; } + +pass=0; fail=0 +ok() { pass=$((pass+1)); echo "ok $1"; } +bad() { fail=$((fail+1)); echo "FAIL $1"; } + +post() { # path json -> body + curl -sf -m 600 -H "Authorization: Bearer $KEY" -H 'Content-Type: application/json' \ + -d "$2" "http://$HOST$1" +} +jget() { # json pyexpr + python3 -c "import json,sys; d=json.load(sys.stdin); print($1)" +} + +# 1. health, no auth +curl -sf "http://$HOST/health" >/dev/null && ok "health" || bad "health" + +# 2. auth enforced +code=$(curl -s -o /dev/null -w '%{http_code}' -d '{"text":"x"}' "http://$HOST/sterilize") +[ "$code" = 401 ] && ok "auth fail-closed (401 without key)" || bad "auth: got $code" + +SCOPE="verify-$$" + +# 3. rules-class PII all tokenized +T1='Marisol Vega (SSN 123-45-6789) can be reached at marisol.vega@example.com or (415) 555-0173. Card on file: 4111 1111 1111 1111. Her laptop is 10.20.30.44 / aa:bb:cc:dd:ee:01.' +R1=$(post /sterilize "$(python3 -c "import json;print(json.dumps({'text':'''$T1''','scope':'$SCOPE'}))")") +leak=$(echo "$R1" | jget "sum(x in d['text'] for x in ['Marisol','123-45-6789','marisol.vega','555-0173','4111','10.20.30.44','aa:bb:cc:dd:ee:01'])") +[ "$leak" = 0 ] && ok "sterilize: no rules-class or name leakage" || bad "sterilize leaked $leak spans: $(echo "$R1" | jget "d['text']")" + +# 4. contextual PII a pattern cannot see (why the recall model is there) +T2='My daughter goes to Lincoln Elementary and we live at 4012 Maple Court. She sees Dr. Okafor for her asthma.' +R2=$(post /sterilize "$(python3 -c "import json;print(json.dumps({'text':'''$T2''','scope':'$SCOPE','verify':True}))")") +leak=$(echo "$R2" | jget "sum(x in d['text'] for x in ['Lincoln','4012','Maple','Okafor','asthma'])") +[ "$leak" = 0 ] && ok "contextual: school/address/doctor/medical caught" || bad "contextual leaked $leak: $(echo "$R2" | jget "d['text']")" + +# 5. round trip back to the original +B1=$(post /rehydrate "$(echo "$R1" | jget "json.dumps({'text': d['text'], 'scope': '$SCOPE'})")") +same=$(echo "$B1" | jget "d['text'] == '''$T1'''") +[ "$same" = True ] && ok "rehydrate: byte-identical round trip" || bad "rehydrate mismatch" + +# 6. token stability within scope +R3=$(post /sterilize "$(python3 -c "import json;print(json.dumps({'text':'Ping marisol.vega@example.com again.','scope':'$SCOPE'}))")") +tok1=$(echo "$R1" | jget "[f['token'] for f in d['findings'] if f['type']=='EMAIL'][0]") +stable=$(echo "$R3" | jget "'$tok1' in d['text']") +[ "$stable" = True ] && ok "stable token reuse in scope ($tok1)" || bad "token not stable" + +# 7. scope isolation on recall +B2=$(post /rehydrate "$(echo "$R1" | jget "json.dumps({'text': d['text'], 'scope': 'other-scope'})")") +n=$(echo "$B2" | jget "d['restored']") +[ "$n" = 0 ] && ok "scope isolation: foreign scope restores nothing" || bad "cross-scope restore: $n" + +# 8. non-PII technical text byte-identical +T4='Set net.core.rmem_max=26214400, then restart the collector; DSA ports stay tagged.' +R4=$(post /sterilize "$(python3 -c "import json;print(json.dumps({'text':'''$T4''','scope':'$SCOPE'}))")") +same=$(echo "$R4" | jget "d['text'] == '''$T4'''") +[ "$same" = True ] && ok "non-PII text untouched" || bad "non-PII mutated: $(echo "$R4" | jget "d['text']")" + +echo "pii live verify: $pass/$((pass+fail))" +[ "$fail" = 0 ] diff --git a/test_syntax_floor.py b/test_syntax_floor.py new file mode 100644 index 0000000..3e56f9a --- /dev/null +++ b/test_syntax_floor.py @@ -0,0 +1,46 @@ +"""Python 3.8 syntax floor for everything that runs on a member's machine. + +bastion agent stages and the PII service ship to whatever the owner runs — a +Mac or Windows box may lag years behind Debian stable. The parser enforces +the floor (`feature_version`), and a short denylist covers stdlib calls +that parse fine on 3.8 and then fail at runtime. Add a directory here when +a new module joins the portable set. Run: python3 -m unittest discover . +""" + +import ast +import os +import unittest + +HERE = os.path.dirname(os.path.abspath(__file__)) +PORTABLE = ("bastion_agent", "pii") +FLOOR = (3, 8) +BANNED_CALLS = ("removeprefix(", "removesuffix(", "file_digest(", "is_relative_to(", + "zoneinfo", "bisect_left(", "pairwise(") # bisect key= is 3.10 + + +class SyntaxFloorTests(unittest.TestCase): + def test_portable_modules_parse_at_floor(self): + checked = 0 + for pkg in PORTABLE: + for name in sorted(os.listdir(os.path.join(HERE, pkg))): + if not name.endswith(".py"): + continue + path = os.path.join(HERE, pkg, name) + with open(path, encoding="utf-8") as fh: + src = fh.read() + ast.parse(src, filename=path, feature_version=FLOOR) + # PEP 585 generics (list[str]) parse on 3.8 but raise at + # import time; catch them by inspection. + for node in ast.walk(ast.parse(src)): + if isinstance(node, ast.Subscript) and isinstance(node.value, ast.Name) \ + and node.value.id in ("list", "dict", "tuple", "set", "type"): + self.fail("%s: PEP 585 generic %s[...] needs typing.%s on 3.8" + % (path, node.value.id, node.value.id.capitalize())) + for banned in BANNED_CALLS: + self.assertNotIn(banned, src, "%s uses %s" % (path, banned)) + checked += 1 + self.assertGreater(checked, 5) + + +if __name__ == "__main__": + unittest.main()