bastion agent 0.1.14
This commit is contained in:
@@ -0,0 +1,17 @@
|
||||
# PII sterilization service (docs/pii-sterilization.md). Runtime is the
|
||||
# Python stdlib only — no pip install exists in this image on purpose;
|
||||
# the vault holds the household's most sensitive data and every
|
||||
# dependency is attack surface.
|
||||
FROM python:3.13-slim@sha256:6771159cd4fa5d9bba1258caf0b82e6b73458c694d178ad97c5e925c2d0e1a91
|
||||
|
||||
WORKDIR /app
|
||||
# every runtime module, named: a module missing here is an ImportError at
|
||||
# boot (recall.py was, once)
|
||||
COPY __init__.py detect.py llmclient.py pipeline.py recall.py service.py vault.py /app/pii/
|
||||
|
||||
# Root inside the container is deliberate: the vault db and its directory
|
||||
# are created 0700/0600 (service sets umask 077) on the mounted /data,
|
||||
# and the single-purpose process is bind-scoped by PII_BIND. Revisit if
|
||||
# the service ever grows a second concern.
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
CMD ["python", "-m", "pii.service"]
|
||||
@@ -0,0 +1,4 @@
|
||||
from . import detect, pipeline # noqa: F401
|
||||
from .detect import PII_TYPES, Finding # noqa: F401
|
||||
from .pipeline import TOKEN_RE, ground, rehydrate, replace_values, sterilize # noqa: F401
|
||||
from .vault import Vault # noqa: F401
|
||||
+249
@@ -0,0 +1,249 @@
|
||||
"""Deterministic PII detectors — the high-precision anchor pass.
|
||||
|
||||
Two jobs: (1) anchor the sterilizer with detections that never depend on
|
||||
model behavior, and (2) re-scan sterilized output as the residual check
|
||||
(pipeline.py loops until this pass finds nothing). Recall for contextual
|
||||
PII — names, addresses, employers, "my daughter's school" — is the LLM
|
||||
pass's job (llmclient.py); here precision is deliberately favored over
|
||||
recall, because a miss still has two more passes to be caught while a
|
||||
false positive burns a vault row forever.
|
||||
|
||||
Overlapping candidates are resolved by type priority then span length:
|
||||
an IPv4 address must not be re-claimed as a dotted phone number, a MAC
|
||||
must not be claimed as truncated IPv6.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import ipaddress
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from typing import List
|
||||
|
||||
# Full type vocabulary — shared with the LLM extraction schema. Rules
|
||||
# only ever emit the subset in _DETECTORS; the rest are LLM-only types.
|
||||
PII_TYPES = [
|
||||
"PERSON", "EMAIL", "PHONE", "ADDRESS", "SSN", "DOB",
|
||||
"CREDIT_CARD", "BANK_ACCOUNT", "IP_ADDRESS", "MAC_ADDRESS",
|
||||
"USERNAME", "CREDENTIAL", "MEDICAL", "ID_NUMBER", "ORG",
|
||||
"LOCATION", "OTHER",
|
||||
]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Finding:
|
||||
value: str
|
||||
type: str
|
||||
start: int
|
||||
end: int
|
||||
|
||||
|
||||
EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)*\.[A-Za-z]{2,}")
|
||||
|
||||
MAC_RE = re.compile(
|
||||
r"(?<![0-9A-Fa-f:.-])(?:[0-9A-Fa-f]{2}([:-]))(?:[0-9A-Fa-f]{2}\1){4}[0-9A-Fa-f]{2}"
|
||||
r"(?![0-9A-Fa-f:-])(?!\.[\w-])"
|
||||
)
|
||||
|
||||
# A full stop after the address ends a sentence; one followed by a letter or
|
||||
# digit makes it part of something longer (a dotted number, a file name).
|
||||
# The same rule applies to phone numbers and MAC addresses.
|
||||
IPV4_RE = re.compile(r"(?<![\d.])(?:\d{1,3}\.){3}\d{1,3}(?!\d)(?!\.[\w-])")
|
||||
|
||||
# Loose candidate; ipaddress.ip_address() is the validator (also rejects
|
||||
# MAC-shaped hex, which has 6 groups and no '::').
|
||||
IPV6_RE = re.compile(r"(?<![\w:.])(?:[0-9A-Fa-f]{0,4}:){2,7}[0-9A-Fa-f]{0,4}(?![\w:])")
|
||||
|
||||
SSN_RE = re.compile(r"(?<![\d-])(\d{3})-(\d{2})-(\d{4})(?![\d-])")
|
||||
|
||||
# 13–19 digits with optional single space/dash separators; Luhn decides.
|
||||
CARD_RE = re.compile(r"(?<![\d-])\d(?:[ -]?\d){12,18}(?![\d-])")
|
||||
|
||||
# Digit-count and shape checks happen in code; this only shapes candidates.
|
||||
PHONE_RE = re.compile(
|
||||
r"(?<![\w.])(?:\+\d{1,3}[\s.-]?)?(?:\(\d{1,4}\)[\s.-]?)?\d{2,4}(?:[\s.-]\d{2,4}){1,4}"
|
||||
r"(?![\w-])(?!\.[\w-])" # a sentence-ending full stop is not part of the number
|
||||
)
|
||||
|
||||
|
||||
# ── checksum-validated account identifiers (borrowed from Presidio's
|
||||
# recognizer list; only the ones a checksum can anchor are here) ──────────
|
||||
|
||||
# IBAN: country code + 2 check digits + 11-30 alphanumerics, optional
|
||||
# spaces every 4. mod-97 == 1 decides (ISO 13616).
|
||||
IBAN_RE = re.compile(r"(?<![A-Za-z0-9])[A-Z]{2}\d{2}(?:[ ]?[A-Z0-9]{4}){2,7}(?:[ ]?[A-Z0-9]{1,4})?(?![A-Za-z0-9])")
|
||||
IBAN_LENGTHS = {
|
||||
"AD": 24, "AE": 23, "AL": 28, "AT": 20, "AZ": 28, "BA": 20, "BE": 16, "BG": 22,
|
||||
"BH": 22, "BR": 29, "CH": 21, "CR": 22, "CY": 28, "CZ": 24, "DE": 22, "DK": 18,
|
||||
"DO": 28, "EE": 20, "EG": 29, "ES": 24, "FI": 18, "FO": 18, "FR": 27, "GB": 22,
|
||||
"GE": 22, "GI": 23, "GL": 18, "GR": 27, "GT": 28, "HR": 21, "HU": 28, "IE": 22,
|
||||
"IL": 23, "IS": 26, "IT": 27, "JO": 30, "KW": 30, "KZ": 20, "LB": 28, "LI": 21,
|
||||
"LT": 20, "LU": 20, "LV": 21, "MC": 27, "MD": 24, "ME": 22, "MK": 19, "MR": 27,
|
||||
"MT": 31, "MU": 30, "NL": 18, "NO": 15, "PK": 24, "PL": 28, "PS": 29, "PT": 25,
|
||||
"QA": 29, "RO": 24, "RS": 22, "SA": 24, "SE": 24, "SI": 19, "SK": 24, "SM": 27,
|
||||
"TN": 24, "TR": 26, "UA": 29, "VG": 24, "XK": 20,
|
||||
}
|
||||
|
||||
# US ABA routing number: 9 digits with a weighted checksum. One in ten
|
||||
# random 9-digit strings passes, so a routing/ABA/RTN cue within 40 chars
|
||||
# before the number is required — same precision stance as Presidio.
|
||||
ABA_RE = re.compile(r"(?<!\d)\d{9}(?!\d)")
|
||||
ABA_CUE_RE = re.compile(r"(?i)\b(?:routing|aba|rtn|transit)\b")
|
||||
|
||||
# Bitcoin: legacy/P2SH base58check (1.../3...). Bech32 (bc1...) has its own
|
||||
# checksum; kept out of v1.2 — its decode is more code than its recall buys.
|
||||
BTC_RE = re.compile(r"(?<![A-Za-z0-9])[13][a-km-zA-HJ-NP-Z1-9]{25,34}(?![A-Za-z0-9])")
|
||||
_B58 = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"
|
||||
|
||||
# Secrets: vendor-prefixed tokens whose prefix alone is near-unambiguous,
|
||||
# plus the two structural forms (PEM blocks, JWTs). Typed CREDENTIAL.
|
||||
SECRET_RES = [
|
||||
re.compile(r"(?<![A-Za-z0-9])(?:AKIA|ASIA)[0-9A-Z]{16}(?![A-Za-z0-9])"), # AWS access key id
|
||||
re.compile(r"(?<![A-Za-z0-9_])gh[pousr]_[A-Za-z0-9]{36,255}(?![A-Za-z0-9])"), # GitHub tokens
|
||||
re.compile(r"(?<![A-Za-z0-9_])github_pat_[A-Za-z0-9_]{22,255}(?![A-Za-z0-9])"), # GitHub fine-grained
|
||||
re.compile(r"(?<![A-Za-z0-9_])xox[abposr]-[A-Za-z0-9-]{10,}(?![A-Za-z0-9])"), # Slack
|
||||
re.compile(r"(?<![A-Za-z0-9_])AIza[0-9A-Za-z_-]{35}(?![A-Za-z0-9_-])"), # Google API key
|
||||
re.compile(r"(?<![A-Za-z0-9_])(?:sk|rk|pk)_(?:live|test)_[A-Za-z0-9]{16,}(?![A-Za-z0-9])"), # Stripe
|
||||
re.compile(r"(?<![A-Za-z0-9_])glpat-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])"), # GitLab PAT
|
||||
re.compile(r"(?<![A-Za-z0-9_])sk-(?:proj-)?[A-Za-z0-9_-]{32,}(?![A-Za-z0-9_-])"), # OpenAI-style
|
||||
re.compile(r"(?<![A-Za-z0-9_-])eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}(?![A-Za-z0-9_-])"), # JWT
|
||||
re.compile(r"-----BEGIN (?:[A-Z ]+ )?PRIVATE KEY-----[\s\S]*?-----END (?:[A-Z ]+ )?PRIVATE KEY-----"), # PEM
|
||||
]
|
||||
|
||||
|
||||
def _iban_ok(cand):
|
||||
s = cand.replace(" ", "").upper()
|
||||
if IBAN_LENGTHS.get(s[:2]) != len(s):
|
||||
return False
|
||||
rearranged = s[4:] + s[:4]
|
||||
num = "".join(str(ord(c) - 55) if c.isalpha() else c for c in rearranged)
|
||||
return int(num) % 97 == 1
|
||||
|
||||
|
||||
def _aba_ok(digits):
|
||||
w = (3, 7, 1, 3, 7, 1, 3, 7, 1)
|
||||
return sum(int(d) * k for d, k in zip(digits, w)) % 10 == 0
|
||||
|
||||
|
||||
def _b58check_ok(addr):
|
||||
n = 0
|
||||
for ch in addr:
|
||||
idx = _B58.find(ch)
|
||||
if idx < 0:
|
||||
return False
|
||||
n = n * 58 + idx
|
||||
raw = n.to_bytes((n.bit_length() + 7) // 8, "big") if n else b""
|
||||
raw = b"\x00" * (len(addr) - len(addr.lstrip("1"))) + raw # leading '1' = 0x00
|
||||
if len(raw) != 25:
|
||||
return False
|
||||
body, check = raw[:-4], raw[-4:]
|
||||
return hashlib.sha256(hashlib.sha256(body).digest()).digest()[:4] == check
|
||||
|
||||
|
||||
def _find_secrets(text):
|
||||
for rx in SECRET_RES:
|
||||
for m in rx.finditer(text):
|
||||
yield Finding(m.group(0), "CREDENTIAL", m.start(), m.end())
|
||||
|
||||
|
||||
def _find_ibans(text):
|
||||
for m in IBAN_RE.finditer(text):
|
||||
if _iban_ok(m.group(0)):
|
||||
yield Finding(m.group(0), "BANK_ACCOUNT", m.start(), m.end())
|
||||
|
||||
|
||||
def _find_abas(text):
|
||||
for m in ABA_RE.finditer(text):
|
||||
if not _aba_ok(m.group(0)):
|
||||
continue
|
||||
if not ABA_CUE_RE.search(text[max(0, m.start() - 40):m.start()]):
|
||||
continue
|
||||
yield Finding(m.group(0), "BANK_ACCOUNT", m.start(), m.end())
|
||||
|
||||
|
||||
def _find_btc(text):
|
||||
for m in BTC_RE.finditer(text):
|
||||
if _b58check_ok(m.group(0)):
|
||||
yield Finding(m.group(0), "BANK_ACCOUNT", m.start(), m.end())
|
||||
|
||||
|
||||
def _luhn_ok(digits: str) -> bool:
|
||||
total = 0
|
||||
for i, ch in enumerate(reversed(digits)):
|
||||
d = int(ch)
|
||||
if i % 2 == 1:
|
||||
d *= 2
|
||||
if d > 9:
|
||||
d -= 9
|
||||
total += d
|
||||
return total % 10 == 0
|
||||
|
||||
|
||||
def _find_emails(text):
|
||||
for m in EMAIL_RE.finditer(text):
|
||||
yield Finding(m.group(0), "EMAIL", m.start(), m.end())
|
||||
|
||||
|
||||
def _find_macs(text):
|
||||
for m in MAC_RE.finditer(text):
|
||||
yield Finding(m.group(0), "MAC_ADDRESS", m.start(), m.end())
|
||||
|
||||
|
||||
def _find_ips(text):
|
||||
for m in IPV4_RE.finditer(text):
|
||||
try:
|
||||
ipaddress.IPv4Address(m.group(0))
|
||||
except ValueError:
|
||||
continue
|
||||
yield Finding(m.group(0), "IP_ADDRESS", m.start(), m.end())
|
||||
for m in IPV6_RE.finditer(text):
|
||||
cand = m.group(0)
|
||||
try:
|
||||
ipaddress.IPv6Address(cand)
|
||||
except ValueError:
|
||||
continue
|
||||
yield Finding(cand, "IP_ADDRESS", m.start(), m.end())
|
||||
|
||||
|
||||
def _find_ssns(text):
|
||||
for m in SSN_RE.finditer(text):
|
||||
area = int(m.group(1))
|
||||
if area == 0 or area == 666 or area >= 900:
|
||||
continue
|
||||
yield Finding(m.group(0), "SSN", m.start(), m.end())
|
||||
|
||||
|
||||
def _find_cards(text):
|
||||
for m in CARD_RE.finditer(text):
|
||||
digits = re.sub(r"\D", "", m.group(0))
|
||||
if 13 <= len(digits) <= 19 and _luhn_ok(digits):
|
||||
yield Finding(m.group(0), "CREDIT_CARD", m.start(), m.end())
|
||||
|
||||
|
||||
def _find_phones(text):
|
||||
for m in PHONE_RE.finditer(text):
|
||||
digits = re.sub(r"\D", "", m.group(0))
|
||||
if not 10 <= len(digits) <= 15:
|
||||
continue
|
||||
yield Finding(m.group(0), "PHONE", m.start(), m.end())
|
||||
|
||||
|
||||
# Priority order: earlier detectors own their spans outright. Secrets go
|
||||
# first (a PEM block or JWT contains digit runs that would otherwise be
|
||||
# claimed piecemeal); IBAN before cards (an IBAN's digits can pass Luhn).
|
||||
_DETECTORS = [
|
||||
_find_secrets, _find_emails, _find_macs, _find_ips, _find_ibans, _find_btc,
|
||||
_find_ssns, _find_cards, _find_abas, _find_phones,
|
||||
]
|
||||
|
||||
|
||||
def detect(text: str) -> List[Finding]:
|
||||
"""All rule findings, non-overlapping, priority- then position-ordered."""
|
||||
candidates = []
|
||||
for prio, fn in enumerate(_DETECTORS):
|
||||
for f in fn(text):
|
||||
candidates.append((prio, -(f.end - f.start), f.start, f))
|
||||
accepted = [] # type: List[Finding]
|
||||
for _, _, _, f in sorted(candidates, key=lambda c: (c[0], c[1], c[2])):
|
||||
if all(f.end <= a.start or f.start >= a.end for a in accepted):
|
||||
accepted.append(f)
|
||||
return sorted(accepted, key=lambda f: f.start)
|
||||
@@ -0,0 +1,112 @@
|
||||
"""LLM span extraction — the recall pass of the sterilizer.
|
||||
|
||||
The model is a sensor, never an actor: it proposes verbatim substrings
|
||||
plus a type from the closed vocabulary, grammar-constrained via
|
||||
llama.cpp's json_schema response_format (off-schema output is
|
||||
impossible). It never rewrites text — pipeline.py grounds every proposed
|
||||
span against the input (verbatim occurrence or it is dropped and
|
||||
counted) and performs all replacement itself, so the model cannot rewrite
|
||||
the text whatever it outputs.
|
||||
|
||||
Talks OpenAI-style chat completions to a llama.cpp-compatible server at
|
||||
LLM_URL, with thinking disabled per request via chat_template_kwargs.
|
||||
Stdlib urllib only.
|
||||
"""
|
||||
|
||||
import json
|
||||
import urllib.request
|
||||
from typing import List, Tuple
|
||||
|
||||
from .detect import PII_TYPES
|
||||
|
||||
Findings = List[Tuple[str, str]]
|
||||
|
||||
_SCHEMA = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"findings": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"text": {"type": "string"},
|
||||
"type": {"type": "string", "enum": PII_TYPES},
|
||||
},
|
||||
"required": ["text", "type"],
|
||||
},
|
||||
}
|
||||
},
|
||||
"required": ["findings"],
|
||||
}
|
||||
|
||||
# The exact vocabulary MUST appear in the prompts. The grammar constrains
|
||||
# type to this enum, but grammar is a backstop, not steering: a model that
|
||||
# free-texts "School" gets prefix-coerced to "SSN" (seen with a real model).
|
||||
# test_pii guards prompt/enum drift.
|
||||
_TYPE_LIST = ", ".join(PII_TYPES)
|
||||
|
||||
EXTRACT_SYSTEM = f"""You are a PII detector for a privacy gateway. List every substring of the user's text that could identify a person, household, account, or device.
|
||||
|
||||
Label each finding with exactly one of these types (use the label verbatim):
|
||||
{_TYPE_LIST}
|
||||
|
||||
Find: personal names; email addresses; phone numbers; street or postal addresses; government/tax/license/passport ID numbers; credit card and bank account numbers; dates of birth; usernames and social handles; passwords, API keys, and other secrets; medical facts tied to a person; employers, schools, or organizations that identify someone; specific places that identify someone (their home, school, workplace); IP addresses; MAC addresses and device serial numbers.
|
||||
|
||||
Rules:
|
||||
- Copy each span VERBATIM, character for character, exactly as it appears in the text. Never paraphrase, correct, or reformat.
|
||||
- Report the minimal identifying span, without surrounding punctuation or filler words.
|
||||
- If the same value appears multiple times, report it once.
|
||||
- Do not report placeholder tokens that look like [PII_TYPE_N] — those are already sterilized.
|
||||
- Generic words, product names, and public facts are not PII. When a span does not identify anyone, leave it out.
|
||||
- If there is no PII, return an empty findings list."""
|
||||
|
||||
VERIFY_SYSTEM = f"""You are auditing a sterilized document for residual PII. Placeholder tokens like [PII_EMAIL_1] are already sterilized — ignore them. Report ONLY remaining substrings that could still identify a person, household, account, or device, copied verbatim, each labeled with exactly one of these types (use the label verbatim): {_TYPE_LIST}. Be aggressive: a partial address, a first name, a nickname, an unusual job title can all identify someone. If nothing identifying remains, return an empty findings list."""
|
||||
|
||||
|
||||
class RecallError(RuntimeError):
|
||||
"""The recall pass could not run. Every backend raises this (or a
|
||||
subclass); the service fails closed on it. Defined here so the older
|
||||
LlmError keeps its name while sharing the contract."""
|
||||
|
||||
|
||||
class LlmError(RecallError):
|
||||
pass
|
||||
|
||||
|
||||
def _chat(llm_url: str, system: str, text: str, timeout: float) -> Findings:
|
||||
body = json.dumps({
|
||||
"messages": [
|
||||
{"role": "system", "content": system},
|
||||
{"role": "user", "content": text},
|
||||
],
|
||||
"temperature": 0,
|
||||
"chat_template_kwargs": {"enable_thinking": False},
|
||||
"response_format": {
|
||||
"type": "json_schema",
|
||||
"json_schema": {"name": "pii_findings", "schema": _SCHEMA},
|
||||
},
|
||||
}).encode()
|
||||
req = urllib.request.Request(
|
||||
llm_url.rstrip("/") + "/v1/chat/completions",
|
||||
data=body,
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
payload = json.load(resp)
|
||||
except OSError as e:
|
||||
raise LlmError(f"llm unreachable: {e}") from e
|
||||
try:
|
||||
content = payload["choices"][0]["message"]["content"]
|
||||
findings = json.loads(content)["findings"]
|
||||
return [(f["text"], f["type"]) for f in findings]
|
||||
except (KeyError, IndexError, TypeError, ValueError) as e:
|
||||
raise LlmError(f"malformed llm response: {e}") from e
|
||||
|
||||
|
||||
def extract(text: str, llm_url: str, timeout: float = 300.0) -> Findings:
|
||||
return _chat(llm_url, EXTRACT_SYSTEM, text, timeout)
|
||||
|
||||
|
||||
def verify(text: str, llm_url: str, timeout: float = 300.0) -> Findings:
|
||||
return _chat(llm_url, VERIFY_SYSTEM, text, timeout)
|
||||
@@ -0,0 +1,51 @@
|
||||
# Standalone systemd packaging for the PII sterilization service, for a
|
||||
# machine that does not run it as a container (pii/Dockerfile). The service
|
||||
# is stdlib-only, so "packaging" is: a python3, the pii/ package, this unit,
|
||||
# and an env file.
|
||||
#
|
||||
# Install (as root):
|
||||
# install -d /opt/privatae-pii
|
||||
# cp -r pii /opt/privatae-pii/pii # the package, tests and all
|
||||
# install -m 600 /dev/null /etc/privatae-pii.env
|
||||
# printf 'PII_API_KEY=%s\nLLM_URL=http://<model-host>:8090\n' "$(openssl rand -hex 24)" >> /etc/privatae-pii.env
|
||||
# # no model host (a laptop)? use the in-process CPU sensor instead of LLM_URL:
|
||||
# # pip install gliner && echo 'PII_RECALL=gliner' >> /etc/privatae-pii.env
|
||||
# cp /opt/privatae-pii/pii/pii.service /etc/systemd/system/privatae-pii.service
|
||||
# systemctl daemon-reload && systemctl enable --now privatae-pii
|
||||
#
|
||||
# The vault lands in /var/lib/privatae-pii (StateDirectory, 0700, owned
|
||||
# by the DynamicUser). Bind stays loopback: the bastion forwarder
|
||||
# (PRIVATAE_PII_ADDR=127.0.0.1:8091) is the reachability story, not the bind.
|
||||
|
||||
[Unit]
|
||||
Description=privatae PII sterilization service
|
||||
Documentation=https://git.privatae.ai/privatae/bastion-agent/src/branch/main/docs/pii-sterilization.md
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=exec
|
||||
WorkingDirectory=/opt/privatae-pii
|
||||
Environment=PII_BIND=127.0.0.1:8091
|
||||
Environment=PII_DB=/var/lib/privatae-pii/vault.db
|
||||
EnvironmentFile=/etc/privatae-pii.env
|
||||
ExecStart=/usr/bin/python3 -m pii.service
|
||||
Restart=always
|
||||
RestartSec=2
|
||||
|
||||
# Containment: no shell, no privileges, private everything, one writable dir.
|
||||
DynamicUser=yes
|
||||
StateDirectory=privatae-pii
|
||||
StateDirectoryMode=0700
|
||||
NoNewPrivileges=yes
|
||||
ProtectSystem=strict
|
||||
ProtectHome=yes
|
||||
PrivateTmp=yes
|
||||
PrivateDevices=yes
|
||||
ProtectKernelTunables=yes
|
||||
ProtectControlGroups=yes
|
||||
RestrictSUIDSGID=yes
|
||||
SystemCallArchitectures=native
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
+217
@@ -0,0 +1,217 @@
|
||||
"""Sterilization pipeline — deterministic control flow, LLM as sensor.
|
||||
|
||||
Stages (docs/pii-sterilization.md):
|
||||
1. rules pass detect.detect() — precision anchors, model-free
|
||||
2. LLM pass llmclient.extract() — recall for contextual PII
|
||||
3. grounding every LLM span must occur verbatim in the input;
|
||||
ungrounded spans are dropped and counted, never trusted
|
||||
4. replacement script-side, longest value first, every occurrence
|
||||
(plain substring, inside longer words too); overlapping
|
||||
matches merge into one token; never inside an existing
|
||||
token; placeholder-shaped text in the input is escaped
|
||||
first (all of it for raw documents, only what the vault
|
||||
does not hold otherwise); tokens minted by the vault
|
||||
(stable per scope)
|
||||
5. residual loop rules re-run on the output until clean (bounded);
|
||||
any residual is tokenized, not just reported
|
||||
6. LLM verify optional second model pass over the sterilized text;
|
||||
grounded hits go through replacement again
|
||||
|
||||
The model proposes; this module disposes. Every mutation of the text
|
||||
happens in replace_values(), which guarantees non-PII bytes are
|
||||
byte-identical to the input.
|
||||
"""
|
||||
|
||||
import re
|
||||
|
||||
from . import detect
|
||||
|
||||
TOKEN_RE = re.compile(r"\[PII_([A-Z_]+)_(\d+)\]")
|
||||
# a token's name without its brackets; as a value it could only ever come
|
||||
# from the model reading an already-sterilized text
|
||||
BARE_TOKEN_RE = re.compile(r"PII_[A-Z_]+_\d+")
|
||||
|
||||
_MAX_RESIDUAL_ROUNDS = 3
|
||||
|
||||
|
||||
def _usable(value: str, typ: str) -> bool:
|
||||
if typ not in detect.PII_TYPES:
|
||||
return False
|
||||
if len(value) < 2:
|
||||
return False
|
||||
if TOKEN_RE.search(value) or BARE_TOKEN_RE.search(value):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def ground(proposed, text):
|
||||
"""Split LLM proposals into (grounded, dropped_count)."""
|
||||
grounded, dropped = [], 0
|
||||
for value, typ in proposed:
|
||||
if value and value in text and _usable(value, typ):
|
||||
grounded.append((value, typ))
|
||||
else:
|
||||
dropped += 1
|
||||
return grounded, dropped
|
||||
|
||||
|
||||
def _occurrences(value, text, blocked):
|
||||
"""Every occurrence of value, overlapping ones included ("aba" twice in
|
||||
"ababa"), minus any that overlap a blocked span. Overlapping hits are
|
||||
merged by the caller, so no fragment of a self-overlapping value is
|
||||
left between two tokens."""
|
||||
spans = []
|
||||
i = text.find(value)
|
||||
while i >= 0:
|
||||
end = i + len(value)
|
||||
if not any(i < be and end > bs for bs, be in blocked):
|
||||
spans.append((i, end))
|
||||
i = text.find(value, i + 1)
|
||||
return spans
|
||||
|
||||
|
||||
def replace_values(text, values, scope, vault, block_tokens=True):
|
||||
"""Replace every occurrence of each value with its vault token.
|
||||
|
||||
Plain substring matching: a value is replaced inside longer words too
|
||||
("Garcia" in "Garcias", "Marisol" in "Marisol_resume.pdf"), because a
|
||||
missed inflection is a leak and an over-eager match only costs
|
||||
readability. Longest value first, so "John Smith" is consumed before
|
||||
"John" can split it. No match may overlap a token already in the text,
|
||||
so a value like "PERSON" cannot break a placeholder apart.
|
||||
|
||||
Two matches that partly overlap (neither contains the other) become one
|
||||
token over their union: tokenizing one would leave the other's
|
||||
remainder in the clear. It takes the longer value's type.
|
||||
|
||||
Every byte outside a replaced span is copied unchanged.
|
||||
|
||||
block_tokens=False is only for escaping placeholder-shaped text in the
|
||||
input (see sterilize), where the literal itself is the value to replace.
|
||||
"""
|
||||
types, order = {}, {}
|
||||
for value, typ in values:
|
||||
if value and value not in types:
|
||||
types[value] = typ
|
||||
order[value] = len(order)
|
||||
blocked = [m.span() for m in TOKEN_RE.finditer(text)] if block_tokens else []
|
||||
matches = []
|
||||
for value in sorted(types, key=lambda v: (-len(v), order[v])):
|
||||
matches.extend((s, e, value) for s, e in _occurrences(value, text, blocked))
|
||||
|
||||
# union overlapping matches; the longest (then earliest-given) value names
|
||||
# the group, and a group wider than it is a merged value of its own
|
||||
groups = []
|
||||
for s, e, value in sorted(matches):
|
||||
if groups and s < groups[-1][1]:
|
||||
g = groups[-1]
|
||||
g[1] = max(g[1], e)
|
||||
g[2].append(value)
|
||||
else:
|
||||
groups.append([s, e, [value]])
|
||||
|
||||
spans = []
|
||||
for s, e, members in groups:
|
||||
lead = min(members, key=lambda v: (-len(v), order[v]))
|
||||
value = text[s:e]
|
||||
if value not in types:
|
||||
types[value] = types[lead]
|
||||
order[value] = order[lead]
|
||||
spans.append((s, e, value))
|
||||
|
||||
tokens, applied = {}, []
|
||||
for value in sorted({v for _, _, v in spans}, key=lambda v: (-len(v), order[v])):
|
||||
tokens[value] = vault.token_for(scope, types[value], value)
|
||||
applied.append({"token": tokens[value], "type": types[value]})
|
||||
|
||||
parts, pos = [], 0
|
||||
for s, e, value in spans:
|
||||
parts.append(text[pos:s])
|
||||
parts.append(tokens[value])
|
||||
pos = e
|
||||
parts.append(text[pos:])
|
||||
return "".join(parts), applied
|
||||
|
||||
|
||||
def sterilize(text, scope, vault, llm_extract, llm_verify=None, escape_held_literals=False):
|
||||
"""Returns (sterilized_text, result_dict).
|
||||
|
||||
escape_held_literals: escape every placeholder-shaped string in the
|
||||
input, even one the vault holds for this scope. For raw documents,
|
||||
which never contain this service's own output: a document that quotes
|
||||
"[PII_EMAIL_1]" must not rehydrate to another document's email. Off for
|
||||
callers that send already-sterilized text back (a stored conversation on
|
||||
every save), where a held placeholder is that token and must stay it.
|
||||
"""
|
||||
rule_values = [(f.value, f.type) for f in detect.detect(text)]
|
||||
|
||||
llm_values, ungrounded = ground(llm_extract(text), text)
|
||||
|
||||
# Placeholder-shaped text that is not one of this scope's tokens is
|
||||
# escaped as a value of its own, so rehydrate gives the literal back
|
||||
# instead of leaving it to collide with a token minted later. Unless the
|
||||
# caller asks for every literal to be escaped, one the vault holds passes
|
||||
# through untouched: already-sterilized text is sterilized again (a
|
||||
# stored conversation on every save), and its tokens must stay the same
|
||||
# tokens. Each distinct literal is looked up once.
|
||||
shapes = {m.group(0) for m in TOKEN_RE.finditer(text)}
|
||||
if not escape_held_literals:
|
||||
shapes = {lit for lit in shapes if vault.value_for(scope, lit) is None}
|
||||
literals = [(lit, "OTHER") for lit in sorted(shapes)]
|
||||
out, escaped = replace_values(text, literals, scope, vault, block_tokens=False)
|
||||
|
||||
# Rules first: replace_values keeps the first type seen for a value,
|
||||
# and on a duplicate the rule's type is the higher-precision one.
|
||||
out, applied = replace_values(out, rule_values + llm_values, scope, vault)
|
||||
|
||||
# every finding a later round tokenizes is counted, not only the last
|
||||
# round's: the round that comes back clean must not reset the total
|
||||
residual = 0
|
||||
for _ in range(_MAX_RESIDUAL_ROUNDS):
|
||||
leftover = [(f.value, f.type) for f in detect.detect(out)]
|
||||
if not leftover:
|
||||
break
|
||||
residual += len(leftover)
|
||||
out, more = replace_values(out, leftover, scope, vault)
|
||||
applied.extend(more)
|
||||
|
||||
verify_hits = 0
|
||||
if llm_verify is not None:
|
||||
vgrounded, _ = ground(llm_verify(out), out)
|
||||
if vgrounded:
|
||||
# count what was replaced, not what was proposed: a grounded value
|
||||
# that only occurs inside an existing token replaces nothing
|
||||
out, more = replace_values(out, vgrounded, scope, vault)
|
||||
verify_hits = len(more)
|
||||
applied.extend(more)
|
||||
|
||||
return out, {
|
||||
"findings": applied,
|
||||
"stats": {
|
||||
"rules": len(rule_values),
|
||||
"llm": len(llm_values),
|
||||
"llm_ungrounded": ungrounded,
|
||||
"residual_tokenized": residual,
|
||||
"verify_tokenized": verify_hits,
|
||||
"literal_tokens_escaped": len(escaped),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def rehydrate(text, scope, vault):
|
||||
"""Replace vault tokens with their original values. Unknown tokens are
|
||||
left in place and reported — never guessed."""
|
||||
unknown = []
|
||||
restored = 0
|
||||
|
||||
def sub(m):
|
||||
nonlocal restored
|
||||
value = vault.value_for(scope, m.group(0))
|
||||
if value is None:
|
||||
unknown.append(m.group(0))
|
||||
return m.group(0)
|
||||
restored += 1
|
||||
return value
|
||||
|
||||
out = TOKEN_RE.sub(sub, text)
|
||||
return out, {"restored": restored, "unknown_tokens": sorted(set(unknown))}
|
||||
+157
@@ -0,0 +1,157 @@
|
||||
"""Recall backends — the sensor behind the sterilizer's second pass.
|
||||
|
||||
pipeline.py needs one thing from a recall backend: a callable that takes
|
||||
text and returns [(verbatim_span, TYPE), ...] with TYPE from
|
||||
detect.PII_TYPES. Everything downstream (grounding, replacement, residual
|
||||
loop) is the same whichever sensor produced the proposals, so swapping the
|
||||
sensor never changes the safety story: a span that is not byte-for-byte in
|
||||
the input is dropped, and the script does every mutation.
|
||||
|
||||
Two backends (docs/pii-sterilization.md, "Recall backends"):
|
||||
|
||||
llm the llama.cpp chat-completions path (llmclient.py) — a model
|
||||
server at LLM_URL, grammar-constrained. Best recall; needs a
|
||||
machine that can reach one.
|
||||
gliner an in-process zero-shot NER model (GLiNER, Apache-2.0) run on
|
||||
CPU — the shape a member's laptop can carry. Optional import:
|
||||
if PII_RECALL=gliner and the package is absent the service
|
||||
refuses to START (fail closed at boot, not at first request).
|
||||
|
||||
Selection is by env (PII_RECALL), never by the caller: a request cannot
|
||||
downgrade the sensor. Stdlib-only here; the gliner package is the one
|
||||
optional third-party import in the module and it is loaded lazily.
|
||||
"""
|
||||
|
||||
from typing import Callable, List, Tuple
|
||||
|
||||
from . import llmclient
|
||||
from .detect import PII_TYPES
|
||||
from .llmclient import RecallError # noqa: F401 (re-exported: one except-clause for every backend)
|
||||
|
||||
Findings = List[Tuple[str, str]]
|
||||
|
||||
|
||||
BACKENDS = ("llm", "gliner")
|
||||
|
||||
# GLiNER is zero-shot: the labels ARE the prompt. Plain-English phrases
|
||||
# recall better than bare enum names; each maps back to the closed
|
||||
# vocabulary the vault and the grammar use. "OTHER" is deliberately not a
|
||||
# label — a catch-all label recalls everything and precision collapses.
|
||||
GLINER_LABELS = {
|
||||
"person name": "PERSON",
|
||||
"email address": "EMAIL",
|
||||
"phone number": "PHONE",
|
||||
"street address": "ADDRESS",
|
||||
"social security number": "SSN",
|
||||
"date of birth": "DOB",
|
||||
"credit card number": "CREDIT_CARD",
|
||||
"bank account number": "BANK_ACCOUNT",
|
||||
"ip address": "IP_ADDRESS",
|
||||
"mac address": "MAC_ADDRESS",
|
||||
"username": "USERNAME",
|
||||
"password or api key or secret": "CREDENTIAL",
|
||||
"medical condition or diagnosis": "MEDICAL",
|
||||
"id number or passport or license number": "ID_NUMBER",
|
||||
"employer or school or organization": "ORG",
|
||||
"specific location that identifies a person": "LOCATION",
|
||||
}
|
||||
assert set(GLINER_LABELS.values()) <= set(PII_TYPES)
|
||||
|
||||
DEFAULT_GLINER_MODEL = "urchade/gliner_multi_pii-v1"
|
||||
DEFAULT_GLINER_THRESHOLD = 0.5
|
||||
DEFAULT_GLINER_VERIFY_THRESHOLD = 0.3
|
||||
|
||||
|
||||
class LlmBackend:
|
||||
name = "llm"
|
||||
|
||||
def __init__(self, llm_url, timeout):
|
||||
self.llm_url = llm_url
|
||||
self.timeout = timeout
|
||||
|
||||
def extract(self, text): # type: (str) -> Findings
|
||||
return llmclient.extract(text, llm_url=self.llm_url, timeout=self.timeout)
|
||||
|
||||
def verify(self, text): # type: (str) -> Findings
|
||||
return llmclient.verify(text, llm_url=self.llm_url, timeout=self.timeout)
|
||||
|
||||
def describe(self):
|
||||
return "llm=%s" % self.llm_url
|
||||
|
||||
|
||||
class GlinerBackend:
|
||||
"""In-process GLiNER. The model object is created once at boot (that is
|
||||
the fail-closed moment) and shared; predict_entities is called per
|
||||
request with the label list above."""
|
||||
|
||||
name = "gliner"
|
||||
|
||||
def __init__(self, model_name, threshold, verify_threshold, loader=None):
|
||||
self.model_name = model_name
|
||||
self.threshold = float(threshold)
|
||||
self.verify_threshold = float(verify_threshold)
|
||||
load = loader or _load_gliner
|
||||
try:
|
||||
self.model = load(model_name)
|
||||
except ImportError as e:
|
||||
raise RecallError(
|
||||
"PII_RECALL=gliner but the 'gliner' package is not installed "
|
||||
"(pip install gliner); refusing to start without a recall pass"
|
||||
) from e
|
||||
except Exception as e: # model download / load failure
|
||||
raise RecallError("gliner model %r failed to load: %s" % (model_name, e)) from e
|
||||
|
||||
def _predict(self, text, threshold): # type: (str, float) -> Findings
|
||||
try:
|
||||
ents = self.model.predict_entities(
|
||||
text, list(GLINER_LABELS), threshold=threshold)
|
||||
except Exception as e:
|
||||
raise RecallError("gliner prediction failed: %s" % e) from e
|
||||
out = []
|
||||
seen = set()
|
||||
for ent in ents:
|
||||
try:
|
||||
span, label = ent["text"], ent["label"]
|
||||
except (KeyError, TypeError):
|
||||
continue
|
||||
typ = GLINER_LABELS.get(label)
|
||||
if typ is None or not isinstance(span, str):
|
||||
continue
|
||||
key = (span, typ)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
out.append(key)
|
||||
return out
|
||||
|
||||
def extract(self, text):
|
||||
return self._predict(text, self.threshold)
|
||||
|
||||
def verify(self, text):
|
||||
return self._predict(text, self.verify_threshold)
|
||||
|
||||
def describe(self):
|
||||
return "gliner=%s@%.2f" % (self.model_name, self.threshold)
|
||||
|
||||
|
||||
def _load_gliner(model_name):
|
||||
from gliner import GLiNER # optional dependency, imported here only
|
||||
return GLiNER.from_pretrained(model_name)
|
||||
|
||||
|
||||
def build_backend(env, loader=None):
|
||||
"""Select and construct the recall backend from the environment.
|
||||
Raises RecallError (fail closed) on an unknown backend or a backend
|
||||
that cannot start."""
|
||||
kind = env.get("PII_RECALL", "llm").strip().lower()
|
||||
if kind == "llm":
|
||||
return LlmBackend(env.get("LLM_URL", "http://llm:8080"),
|
||||
float(env.get("PII_LLM_TIMEOUT", "300")))
|
||||
if kind == "gliner":
|
||||
return GlinerBackend(
|
||||
env.get("GLINER_MODEL", DEFAULT_GLINER_MODEL),
|
||||
env.get("GLINER_THRESHOLD", DEFAULT_GLINER_THRESHOLD),
|
||||
env.get("GLINER_VERIFY_THRESHOLD", DEFAULT_GLINER_VERIFY_THRESHOLD),
|
||||
loader=loader,
|
||||
)
|
||||
raise RecallError("PII_RECALL=%r is not one of %s" % (kind, BACKENDS))
|
||||
+178
@@ -0,0 +1,178 @@
|
||||
"""PII sterilization service — HTTP surface over pipeline + vault.
|
||||
|
||||
Stdlib-only by design: this process holds the household's most sensitive
|
||||
data, and zero third-party runtime dependencies is the audit story
|
||||
(every dependency is attack surface).
|
||||
|
||||
Endpoints (JSON in/out):
|
||||
GET /health liveness, no auth
|
||||
GET /scope/<scope> token inventory (types+tokens, no values)
|
||||
POST /sterilize {text, scope?, verify?, escape_held_literals?} -> {text, findings, stats}
|
||||
POST /rehydrate {text, scope?} -> {text, restored, unknown_tokens}
|
||||
POST /flag {text} -> {flagged, counts} rules-only, no model
|
||||
call, no vault writes — the milliseconds pre-inference check. A clean
|
||||
/flag does NOT mean no PII (contextual PII needs /sterilize).
|
||||
|
||||
Auth is fail-closed: PII_API_KEY must be set in the environment or the
|
||||
service refuses to start. PII_ALLOW_NOAUTH=1
|
||||
exists for the unit-test harness only. /rehydrate returns REAL values —
|
||||
callers are trusted surfaces only; every call lands in the vault's audit
|
||||
table.
|
||||
|
||||
Config (env): PII_BIND (host:port, default 0.0.0.0:8091), LLM_URL
|
||||
(default http://llm:8080), PII_DB (default /data/vault.db),
|
||||
PII_LLM_TIMEOUT seconds (default 300).
|
||||
"""
|
||||
|
||||
import hmac
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
|
||||
from . import detect, pipeline, recall
|
||||
from .vault import Vault
|
||||
|
||||
_MAX_BODY = 2 * 1024 * 1024
|
||||
|
||||
|
||||
class Config:
|
||||
def __init__(self, env):
|
||||
self.bind = env.get("PII_BIND", "0.0.0.0:8091")
|
||||
self.llm_url = env.get("LLM_URL", "http://llm:8080")
|
||||
self.db = env.get("PII_DB", "/data/vault.db")
|
||||
self.api_key = env.get("PII_API_KEY", "")
|
||||
self.llm_timeout = float(env.get("PII_LLM_TIMEOUT", "300"))
|
||||
if not self.api_key and env.get("PII_ALLOW_NOAUTH") != "1":
|
||||
raise SystemExit(
|
||||
"refusing to start: PII_API_KEY unset (fail-closed; "
|
||||
"PII_ALLOW_NOAUTH=1 is for tests only)"
|
||||
)
|
||||
# The recall sensor is chosen here, at boot, by the operator — never
|
||||
# per request. A backend that cannot start stops the service.
|
||||
try:
|
||||
self.recall = recall.build_backend(env)
|
||||
except recall.RecallError as e:
|
||||
raise SystemExit("refusing to start: %s" % e)
|
||||
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
server_version = "privatae-pii"
|
||||
protocol_version = "HTTP/1.1"
|
||||
|
||||
# injected by build_server
|
||||
cfg: Config = None
|
||||
vault: Vault = None
|
||||
|
||||
def _send(self, code, obj):
|
||||
body = json.dumps(obj).encode()
|
||||
self.send_response(code)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def _authed(self) -> bool:
|
||||
if not self.cfg.api_key:
|
||||
return True # only reachable via PII_ALLOW_NOAUTH (tests)
|
||||
got = self.headers.get("Authorization", "")
|
||||
want = f"Bearer {self.cfg.api_key}"
|
||||
return hmac.compare_digest(got.encode(), want.encode())
|
||||
|
||||
def do_GET(self):
|
||||
if self.path == "/health":
|
||||
self._send(200, {"ok": True})
|
||||
return
|
||||
if not self._authed():
|
||||
self._send(401, {"error": "unauthorized"})
|
||||
return
|
||||
if self.path.startswith("/scope/"):
|
||||
scope = self.path[len("/scope/"):]
|
||||
self._send(200, {"scope": scope, "tokens": self.vault.scope_tokens(scope)})
|
||||
return
|
||||
self._send(404, {"error": "not found"})
|
||||
|
||||
def do_POST(self):
|
||||
if not self._authed():
|
||||
self._send(401, {"error": "unauthorized"})
|
||||
return
|
||||
try:
|
||||
length = int(self.headers.get("Content-Length", "0"))
|
||||
if length > _MAX_BODY:
|
||||
self._send(413, {"error": "body too large"})
|
||||
return
|
||||
req = json.loads(self.rfile.read(length))
|
||||
text = req["text"]
|
||||
scope = req.get("scope", "default")
|
||||
if not isinstance(text, str) or not isinstance(scope, str) or not scope:
|
||||
raise ValueError("text must be a string, scope a non-empty string")
|
||||
except (ValueError, KeyError, TypeError) as e:
|
||||
self._send(400, {"error": f"bad request: {e}"})
|
||||
return
|
||||
|
||||
if self.path == "/sterilize":
|
||||
backend = self.cfg.recall
|
||||
llm_verify = backend.verify if req.get("verify") else None
|
||||
escape_all = req.get("escape_held_literals") is True
|
||||
try:
|
||||
out, result = pipeline.sterilize(
|
||||
text, scope, self.vault, backend.extract, llm_verify,
|
||||
escape_held_literals=escape_all,
|
||||
)
|
||||
except recall.RecallError as e:
|
||||
# Fail closed: no recall pass means no recall guarantee, so no
|
||||
# partially-sterilized text ever leaves this process.
|
||||
self._send(502, {"error": str(e)})
|
||||
return
|
||||
result["stats"]["recall"] = backend.name
|
||||
# echo the literal rule applied, so a caller that asked for
|
||||
# escape_held_literals can tell it was honoured
|
||||
result["stats"]["escape_held_literals"] = escape_all
|
||||
self.vault.audit(scope, "sterilize", json.dumps(result["stats"]))
|
||||
self._send(200, {"text": out, **result})
|
||||
return
|
||||
|
||||
if self.path == "/flag":
|
||||
# Rules pass only: deterministic, model-free, no vault rows, no
|
||||
# audit (nothing minted, nothing recallable). Precision anchors
|
||||
# only — callers must not treat a clean flag as PII-free.
|
||||
counts = {}
|
||||
for f in detect.detect(text):
|
||||
counts[f.type] = counts.get(f.type, 0) + 1
|
||||
self._send(200, {"flagged": bool(counts), "counts": counts})
|
||||
return
|
||||
|
||||
if self.path == "/rehydrate":
|
||||
out, result = pipeline.rehydrate(text, scope, self.vault)
|
||||
self.vault.audit(
|
||||
scope, "rehydrate",
|
||||
json.dumps({"restored": result["restored"],
|
||||
"unknown": len(result["unknown_tokens"])}),
|
||||
)
|
||||
self._send(200, {"text": out, **result})
|
||||
return
|
||||
|
||||
self._send(404, {"error": "not found"})
|
||||
|
||||
def log_message(self, fmt, *args):
|
||||
# No request bodies in logs, ever; method+path+status only.
|
||||
sys.stderr.write("%s %s\n" % (self.address_string(), fmt % args))
|
||||
|
||||
|
||||
def build_server(cfg: Config) -> ThreadingHTTPServer:
|
||||
os.umask(0o077)
|
||||
handler = type("BoundHandler", (Handler,), {"cfg": cfg, "vault": Vault(cfg.db)})
|
||||
host, port = cfg.bind.rsplit(":", 1)
|
||||
return ThreadingHTTPServer((host, int(port)), handler)
|
||||
|
||||
|
||||
def main():
|
||||
cfg = Config(os.environ)
|
||||
srv = build_server(cfg)
|
||||
sys.stderr.write("pii service on %s, recall %s, db=%s\n"
|
||||
% (cfg.bind, cfg.recall.describe(), cfg.db))
|
||||
srv.serve_forever()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+687
@@ -0,0 +1,687 @@
|
||||
"""Unit + wire tests for the PII sterilization service.
|
||||
|
||||
Everything here is model-free: the LLM pass is either a fake callable
|
||||
(pipeline tests) or a stub llama.cpp speaking the real chat-completions
|
||||
wire format on loopback (service tests). Live recall quality against a
|
||||
real model is verify-live.sh's job, against a running service.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from unittest import mock
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
|
||||
from . import detect, pipeline, service
|
||||
from .vault import Vault
|
||||
|
||||
|
||||
class DetectTests(unittest.TestCase):
|
||||
def find(self, text):
|
||||
return {(f.value, f.type) for f in detect.detect(text)}
|
||||
|
||||
def test_email(self):
|
||||
self.assertIn(("sam.r@example.co.uk", "EMAIL"),
|
||||
self.find("mail sam.r@example.co.uk today"))
|
||||
|
||||
def test_phone_forms(self):
|
||||
for s in ["+1 415 555 0173", "(415) 555-0173", "415-555-0173", "415.555.0173"]:
|
||||
self.assertIn((s, "PHONE"), self.find(f"call {s} now"), s)
|
||||
|
||||
def test_date_is_not_phone(self):
|
||||
self.assertEqual(self.find("released 2026-08-03 at noon"), set())
|
||||
|
||||
def test_ssn_valid_and_invalid_area(self):
|
||||
self.assertIn(("123-45-6789", "SSN"), self.find("ssn 123-45-6789"))
|
||||
self.assertEqual(self.find("ssn 666-45-6789"), set())
|
||||
self.assertEqual(self.find("ssn 000-45-6789"), set())
|
||||
|
||||
def test_credit_card_luhn(self):
|
||||
self.assertIn(("4111 1111 1111 1111", "CREDIT_CARD"),
|
||||
self.find("card 4111 1111 1111 1111 exp 12/28"))
|
||||
# same shape, fails Luhn
|
||||
self.assertEqual(self.find("ref 4111 1111 1111 1112"), set())
|
||||
|
||||
def test_ipv4_owns_dotted_span(self):
|
||||
found = self.find("host at 192.168.68.54 responded")
|
||||
self.assertIn(("192.168.68.54", "IP_ADDRESS"), found)
|
||||
self.assertNotIn(("192.168.68.54", "PHONE"), found)
|
||||
|
||||
def test_ipv4_invalid_octet(self):
|
||||
self.assertNotIn(("999.168.68.541", "IP_ADDRESS"), self.find("x 999.168.68.541"))
|
||||
|
||||
def test_ipv6_and_mac_disambiguation(self):
|
||||
found = self.find("fe80::1ff:fe23:4567:890a from aa:bb:cc:dd:ee:ff")
|
||||
self.assertIn(("fe80::1ff:fe23:4567:890a", "IP_ADDRESS"), found)
|
||||
self.assertIn(("aa:bb:cc:dd:ee:ff", "MAC_ADDRESS"), found)
|
||||
|
||||
def test_sentence_ending_full_stop(self):
|
||||
self.assertIn(("415-555-0173", "PHONE"), self.find("Call me at 415-555-0173."))
|
||||
self.assertIn(("415.555.0173", "PHONE"), self.find("Call me at 415.555.0173."))
|
||||
self.assertIn(("10.1.2.3", "IP_ADDRESS"), self.find("the gateway was 10.1.2.3."))
|
||||
self.assertIn(("aa:bb:cc:dd:ee:ff", "MAC_ADDRESS"), self.find("from aa:bb:cc:dd:ee:ff."))
|
||||
|
||||
def test_full_stop_inside_a_longer_number_still_refuses(self):
|
||||
self.assertEqual(self.find("version 10.1.2.3.4 shipped"), set())
|
||||
self.assertNotIn("PHONE", {t for _, t in self.find("ref 415-555-0173.5 end")})
|
||||
self.assertNotIn("PHONE", {t for _, t in self.find("file 415-555-0173.pdf")})
|
||||
|
||||
def test_full_stop_rule_is_the_same_for_ip_phone_and_mac(self):
|
||||
# a full stop followed by a letter or digit is part of something longer
|
||||
self.assertEqual(self.find("open 10.1.2.3.txt now"), set())
|
||||
self.assertIn(("10.1.2.3", "IP_ADDRESS"), self.find("the host was 10.1.2.3."))
|
||||
self.assertEqual(self.find("saved aa:bb:cc:dd:ee:ff.bak"), set())
|
||||
self.assertEqual(self.find("saved aa:bb:cc:dd:ee:ff.txt"), set())
|
||||
self.assertIn(("aa:bb:cc:dd:ee:ff", "MAC_ADDRESS"), self.find("it was aa:bb:cc:dd:ee:ff."))
|
||||
|
||||
# ── v1.2 checksum-anchored recognizers ──────────────────────────────
|
||||
def test_iban_mod97(self):
|
||||
for iban in ("GB82 WEST 1234 5698 7654 32", "DE89370400440532013000"):
|
||||
self.assertIn((iban, "BANK_ACCOUNT"), self.find("pay to %s thanks" % iban), iban)
|
||||
for bad in ("GB82 WEST 1234 5698 7654 33", # bad check digits
|
||||
"ZZ82 WEST 1234 5698 7654 32"): # unknown country
|
||||
self.assertNotIn("BANK_ACCOUNT", {t for _, t in self.find("pay to " + bad)}, bad)
|
||||
|
||||
def test_iban_not_reclaimed_as_card(self):
|
||||
found = self.find("iban DE89370400440532013000")
|
||||
self.assertNotIn(("370400440532013000", "CREDIT_CARD"), found)
|
||||
self.assertEqual({t for _, t in found}, {"BANK_ACCOUNT"})
|
||||
|
||||
def test_aba_needs_checksum_and_cue(self):
|
||||
self.assertIn(("021000021", "BANK_ACCOUNT"), self.find("routing number 021000021"))
|
||||
self.assertIn(("021000021", "BANK_ACCOUNT"), self.find("ABA: 021000021"))
|
||||
self.assertEqual(self.find("order 021000021 shipped"), set()) # no cue
|
||||
self.assertEqual(self.find("routing number 021000022"), set()) # bad checksum
|
||||
|
||||
def test_bitcoin_base58check(self):
|
||||
good = "1BvBMSEYstWetqTFn5Au4m4GFg7xJaNVN2"
|
||||
self.assertIn((good, "BANK_ACCOUNT"), self.find("send to %s" % good))
|
||||
self.assertIn(("1A1zP1eP5QGefi2DMPTfTL5SLmv7DivfNa", "BANK_ACCOUNT"),
|
||||
self.find("genesis 1A1zP1eP5QGefi2DMPTfTL5SLmv7DivfNa"))
|
||||
self.assertEqual(self.find("send to 1BvBMSEYstWetqTFn5Au4m4GFg7xJaNVN3"), set())
|
||||
|
||||
def test_secrets_are_credentials(self):
|
||||
cases = {
|
||||
"AKIAIOSFODNN7EXAMPLE": "aws key AKIAIOSFODNN7EXAMPLE",
|
||||
"ghp_" + "a" * 36: "token ghp_" + "a" * 36 + " end",
|
||||
"xoxb-1234567890-abcdefghij": "slack xoxb-1234567890-abcdefghij",
|
||||
"AIza" + "b" * 35: "gkey AIza" + "b" * 35,
|
||||
"sk_live_" + "c" * 24: "stripe sk_live_" + "c" * 24,
|
||||
"glpat-" + "d" * 20: "gl glpat-" + "d" * 20,
|
||||
"eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dozjgNryP4J3jVmNHl0w5N_XgL0n3I9PlFUP0THsR8U":
|
||||
"bearer eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dozjgNryP4J3jVmNHl0w5N_XgL0n3I9PlFUP0THsR8U",
|
||||
}
|
||||
for value, text in cases.items():
|
||||
self.assertIn((value, "CREDENTIAL"), self.find(text), value)
|
||||
|
||||
def test_pem_block_is_one_credential(self):
|
||||
pem = "-----BEGIN EC PRIVATE KEY-----\nMHQCAQEEIF12 3456 7890 1234\n-----END EC PRIVATE KEY-----"
|
||||
found = self.find("key:\n" + pem + "\nok")
|
||||
self.assertEqual(found, {(pem, "CREDENTIAL")})
|
||||
|
||||
def test_secrets_do_not_fire_on_prose(self):
|
||||
self.assertEqual(self.find("the skate park (AKA the bowl) opens at 9"), set())
|
||||
|
||||
|
||||
class VaultTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.vault = Vault(os.path.join(self.tmp.name, "v.db"))
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def test_stable_token_same_scope(self):
|
||||
t1 = self.vault.token_for("s1", "EMAIL", "a@b.com")
|
||||
t2 = self.vault.token_for("s1", "EMAIL", "a@b.com")
|
||||
self.assertEqual(t1, t2)
|
||||
self.assertEqual(t1, "[PII_EMAIL_1]")
|
||||
|
||||
def test_scopes_isolated(self):
|
||||
self.vault.token_for("s1", "EMAIL", "a@b.com")
|
||||
self.vault.token_for("s1", "EMAIL", "c@d.com")
|
||||
t_other = self.vault.token_for("s2", "EMAIL", "c@d.com")
|
||||
self.assertEqual(t_other, "[PII_EMAIL_1]") # own sequence
|
||||
self.assertIsNone(self.vault.value_for("s2", "[PII_EMAIL_2]"))
|
||||
|
||||
def test_inventory_has_no_values(self):
|
||||
self.vault.token_for("s1", "PERSON", "Ada Lovelace")
|
||||
inv = json.dumps(self.vault.scope_tokens("s1"))
|
||||
self.assertNotIn("Ada", inv)
|
||||
|
||||
|
||||
class PipelineTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.vault = Vault(os.path.join(self.tmp.name, "v.db"))
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def test_grounding_drops_hallucinated_span(self):
|
||||
text = "call me at 415-555-0173"
|
||||
fake = lambda t: [("415-555-0173", "PHONE"), ("Bob Jones", "PERSON")]
|
||||
out, res = pipeline.sterilize(text, "s", self.vault, fake)
|
||||
self.assertEqual(res["stats"]["llm_ungrounded"], 1)
|
||||
self.assertNotIn("Bob", out)
|
||||
self.assertNotIn("415-555-0173", out)
|
||||
|
||||
def test_longest_value_wins(self):
|
||||
text = "John Smith emailed. John was brief."
|
||||
fake = lambda t: [("John Smith", "PERSON"), ("John", "PERSON")]
|
||||
out, _ = pipeline.sterilize(text, "s", self.vault, fake)
|
||||
self.assertEqual(out, "[PII_PERSON_1] emailed. [PII_PERSON_2] was brief.")
|
||||
|
||||
def test_non_pii_bytes_identical(self):
|
||||
text = "The nftables ruleset, with odd spacing\tand café unicode, is fine."
|
||||
out, res = pipeline.sterilize(text, "s", self.vault, lambda t: [])
|
||||
self.assertEqual(out, text)
|
||||
self.assertEqual(res["findings"], [])
|
||||
|
||||
def test_rules_catch_what_llm_missed(self):
|
||||
text = "ssn is 123-45-6789 ok"
|
||||
out, res = pipeline.sterilize(text, "s", self.vault, lambda t: [])
|
||||
self.assertEqual(out, "ssn is [PII_SSN_1] ok")
|
||||
self.assertEqual(res["stats"]["rules"], 1)
|
||||
|
||||
def test_round_trip(self):
|
||||
text = "Sam Rivera (sam.r@example.com, 415-555-0173) lives at 4012 Maple Court."
|
||||
fake = lambda t: [("Sam Rivera", "PERSON"), ("4012 Maple Court", "ADDRESS")]
|
||||
out, _ = pipeline.sterilize(text, "s", self.vault, fake)
|
||||
self.assertNotIn("Maple", out)
|
||||
self.assertNotIn("sam.r", out)
|
||||
back, res = pipeline.rehydrate(out, "s", self.vault)
|
||||
self.assertEqual(back, text)
|
||||
self.assertEqual(res["unknown_tokens"], [])
|
||||
|
||||
def test_rehydrate_unknown_token_left_alone(self):
|
||||
out, res = pipeline.rehydrate("hello [PII_EMAIL_9]", "s", self.vault)
|
||||
self.assertEqual(out, "hello [PII_EMAIL_9]")
|
||||
self.assertEqual(res["unknown_tokens"], ["[PII_EMAIL_9]"])
|
||||
|
||||
def test_stable_across_calls(self):
|
||||
fake = lambda t: []
|
||||
out1, _ = pipeline.sterilize("mail a@b.com", "s", self.vault, fake)
|
||||
out2, _ = pipeline.sterilize("again a@b.com!", "s", self.vault, fake)
|
||||
self.assertIn("[PII_EMAIL_1]", out1)
|
||||
self.assertIn("[PII_EMAIL_1]", out2)
|
||||
|
||||
def test_rules_type_wins_on_duplicate_value(self):
|
||||
# LLM mistyping a value the rules also caught must not change the
|
||||
# minted type — rules are the higher-precision typer.
|
||||
text = "reach me at 415-555-0173"
|
||||
fake = lambda t: [("415-555-0173", "ID_NUMBER")]
|
||||
_, res = pipeline.sterilize(text, "s", self.vault, fake)
|
||||
self.assertEqual(res["findings"], [{"token": "[PII_PHONE_1]", "type": "PHONE"}])
|
||||
|
||||
def assertTokenizedAndRoundTrips(self, text, values, secret, verify=None, label=""):
|
||||
vault = Vault(os.path.join(self.tmp.name, "case-%d.db" % self._n))
|
||||
self._n += 1
|
||||
out, res = pipeline.sterilize(text, "s", vault, lambda t: list(values), verify)
|
||||
self.assertNotIn(secret, out, "%s leaked: %r" % (label, out))
|
||||
back, _ = pipeline.rehydrate(out, "s", vault)
|
||||
self.assertEqual(back, text, "%s round trip: %r" % (label, out))
|
||||
return out, res
|
||||
|
||||
# A value is replaced as a plain substring, inside longer words too:
|
||||
# an inflection, a file name or a handle that keeps the value is still
|
||||
# the value. Each case must tokenize and rehydrate byte-identically.
|
||||
SUBSTRING_CASES = [
|
||||
("possessive", "Marisol's file is here", [("Marisol", "PERSON")], "Marisol"),
|
||||
("possessive curly", "Marisol\u2019s file", [("Marisol", "PERSON")], "Marisol"),
|
||||
("hyphen", "Marisol-Garcia signed", [("Marisol", "PERSON")], "Marisol"),
|
||||
("punctuation", "Ask Marisol. Then Marisol, Marisol!", [("Marisol", "PERSON")], "Marisol"),
|
||||
("line start and end", "Marisol\nhi\nbye Marisol", [("Marisol", "PERSON")], "Marisol"),
|
||||
("email local part", "write marisol.garcia@acme.org or Marisol", [("Marisol", "PERSON")], "Marisol"),
|
||||
("url path", "see https://acme.org/team/Marisol/cv", [("Marisol", "PERSON")], "Marisol"),
|
||||
("underscore file name", "attached Marisol_Garcia_resume.pdf", [("Marisol", "PERSON")], "Marisol"),
|
||||
("underscore username", "login marisol_g from home", [("marisol", "USERNAME")], "marisol"),
|
||||
("digit suffix username", "user jsmith42 logged in", [("jsmith", "USERNAME")], "jsmith"),
|
||||
("camelCase", "folder MarisolGarcia/", [("Marisol", "PERSON")], "Marisol"),
|
||||
("plural surname", "the Garcias came over", [("Garcia", "PERSON")], "Garcia"),
|
||||
("inside another word", "Ann flew to Annapolis", [("Ann", "PERSON")], "Ann"),
|
||||
("german genitive", "Annas Auto ist rot", [("Anna", "PERSON")], "Anna"),
|
||||
("finnish case", "Kerro Marisolille", [("Marisol", "PERSON")], "Marisol"),
|
||||
("hungarian case", "Marisolnak adtam", [("Marisol", "PERSON")], "Marisol"),
|
||||
("hashtag and digits", "#Marisol2024 party", [("Marisol", "PERSON")], "Marisol"),
|
||||
("diacritics", "Jos\u00e9 and Zo\u00eb's", [("Jos\u00e9", "PERSON"), ("Zo\u00eb", "PERSON")], "Jos\u00e9"),
|
||||
("decomposed accent", "Jose\u0301 came", [("Jose", "PERSON")], "Jose"),
|
||||
("apostrophe name", "O'Brien's O'Brien", [("O'Brien", "PERSON")], "O'Brien"),
|
||||
("value with digit", "Room 101B and apt 101", [("101B", "ADDRESS")], "101B"),
|
||||
("next to a placeholder", "m@x.com Marisol m@x.comMarisol", [("Marisol", "PERSON")], "Marisol"),
|
||||
("chinese", "\u5f20\u4f1f\u4f4f\u5728\u5317\u4eac\uff0c\u5f20\u4f1f\u660e\u4e5f\u5728",
|
||||
[("\u5f20\u4f1f", "PERSON")], "\u5f20\u4f1f"),
|
||||
("latin next to chinese", "Ann\u4f4f\u5728\u5317\u4eac Ann's", [("Ann", "PERSON")], "Ann"),
|
||||
("halfwidth katakana", "\uff80\uff85\uff76\u3055\u3093 \uff80\uff85\uff76\uff80\uff9b\uff73",
|
||||
[("\uff80\uff85\uff76", "PERSON")], "\uff80\uff85\uff76"),
|
||||
("thai", "\u0e2a\u0e21\u0e0a\u0e32\u0e22\u0e44\u0e1b\u0e1a\u0e49\u0e32\u0e19",
|
||||
[("\u0e2a\u0e21\u0e0a\u0e32\u0e22", "PERSON")], "\u0e2a\u0e21\u0e0a\u0e32\u0e22"),
|
||||
("korean particle", "\ubbfc\uc218\ub294 \ud559\uad50\uc5d0", [("\ubbfc\uc218", "PERSON")], "\ubbfc\uc218"),
|
||||
("devanagari suffix", "\u0930\u093e\u092e\u0915\u094b \u0918\u0930",
|
||||
[("\u0930\u093e\u092e", "PERSON")], "\u0930\u093e\u092e"),
|
||||
("tai tham", "\u1a20\u1a21\u1a22\u1a23", [("\u1a20\u1a21", "PERSON")], "\u1a20\u1a21"),
|
||||
("model value partly overlaps a rule hit", "call 415-555-0173 Main St",
|
||||
[("0173 Main St", "ADDRESS")], "Main St"),
|
||||
("model value covers a rule hit", "Marisol <m@example.com>",
|
||||
[("Marisol <m@example.com>", "OTHER")], "m@example.com"),
|
||||
# rule findings: every copy of the value goes, not only the detected one
|
||||
("repeated phone", "Call 415-555-0173 or text 415-555-0173.", [], "415-555-0173"),
|
||||
("repeated ip, sentence end", "ip 10.1.2.3 and gateway was 10.1.2.3.", [], "10.1.2.3"),
|
||||
("ssn copy with suffix", "ssn 123-45-6789 / ref 123-45-6789-A", [], "123-45-6789"),
|
||||
("phone copy glued to a word", "tel:415-555-0173 ph415-555-0173", [], "415-555-0173"),
|
||||
("phone at sentence end", "Call me at 415-555-0173.", [], "415-555-0173"),
|
||||
]
|
||||
|
||||
def test_substring_cases_tokenize_and_round_trip(self):
|
||||
self._n = 0
|
||||
for label, text, values, secret in self.SUBSTRING_CASES:
|
||||
with self.subTest(label):
|
||||
self.assertTokenizedAndRoundTrips(text, values, secret, label=label)
|
||||
|
||||
def test_substring_match_is_counted_in_findings(self):
|
||||
self._n = 0
|
||||
_, res = self.assertTokenizedAndRoundTrips(
|
||||
"the Garcias came over", [("Garcia", "PERSON")], "Garcia")
|
||||
self.assertEqual(res["findings"], [{"token": "[PII_PERSON_1]", "type": "PERSON"}])
|
||||
|
||||
def test_partial_overlap_becomes_one_token(self):
|
||||
# the model value overlaps the rule hit from the right, then from the
|
||||
# left; either way one token over the union, rehydrating to the bytes
|
||||
cases = [
|
||||
("call 415-555-0173 Main St", [("0173 Main St", "ADDRESS")],
|
||||
"415-555-0173 Main St", "call [PII_PHONE_1]"),
|
||||
("at Main St 415-555-0173 today", [("Main St 415", "ADDRESS")],
|
||||
"Main St 415-555-0173", "at [PII_PHONE_1] today"),
|
||||
]
|
||||
for i, (text, values, merged, want) in enumerate(cases):
|
||||
with self.subTest(text):
|
||||
vault = Vault(os.path.join(self.tmp.name, "po-%d.db" % i))
|
||||
out, _ = pipeline.sterilize(text, "s", vault, lambda t, v=values: v)
|
||||
self.assertEqual(out, want)
|
||||
self.assertEqual(vault.value_for("s", "[PII_PHONE_1]"), merged)
|
||||
back, res = pipeline.rehydrate(out, "s", vault)
|
||||
self.assertEqual(back, text)
|
||||
self.assertEqual((res["restored"], res["unknown_tokens"]), (1, []))
|
||||
|
||||
def test_self_overlapping_value_leaves_no_fragment(self):
|
||||
for text, value, want in (("x ababa x", "aba", "x [PII_PERSON_1] x"),
|
||||
("id 121212 end", "1212", "id [PII_ID_NUMBER_1] end")):
|
||||
with self.subTest(text):
|
||||
vault = Vault(os.path.join(self.tmp.name, "so-%s.db" % value))
|
||||
typ = "ID_NUMBER" if value.isdigit() else "PERSON"
|
||||
out, _ = pipeline.sterilize(text, "s", vault, lambda t, v=value, ty=typ: [(v, ty)])
|
||||
self.assertEqual(out, want)
|
||||
back, _ = pipeline.rehydrate(out, "s", vault)
|
||||
self.assertEqual(back, text)
|
||||
|
||||
def test_never_minted_placeholder_in_the_source_survives_the_round_trip(self):
|
||||
# placeholder-shaped text the vault does not hold is escaped, so it
|
||||
# comes back as written, card-like digits inside it included
|
||||
pipeline.sterilize("Marisol called", "s", self.vault, lambda t: [("Marisol", "PERSON")])
|
||||
text = "log: [PII_EMAIL_7], [PII_FOO_99] and [PII_A_4111111111111111], sent by Ana"
|
||||
out, res = pipeline.sterilize(text, "s", self.vault, lambda t: [("Ana", "PERSON")])
|
||||
for lit in ("[PII_EMAIL_7]", "[PII_FOO_99]", "[PII_A_4111111111111111]"):
|
||||
self.assertNotIn(lit, out)
|
||||
self.assertEqual(res["stats"]["literal_tokens_escaped"], 3)
|
||||
self.assertEqual(res["findings"], [{"token": "[PII_PERSON_2]", "type": "PERSON"}])
|
||||
back, info = pipeline.rehydrate(out, "s", self.vault)
|
||||
self.assertEqual(back, text)
|
||||
self.assertEqual(info["unknown_tokens"], [])
|
||||
|
||||
def test_live_token_in_the_source_passes_through(self):
|
||||
# a placeholder the vault holds for this scope is treated as the token
|
||||
# it names: a quoted "[PII_PERSON_1]" rehydrates to the vault value.
|
||||
# That is the documented limit; it is what keeps re-sterilizing
|
||||
# already-sterilized text stable.
|
||||
pipeline.sterilize("Marisol called", "s", self.vault, lambda t: [("Marisol", "PERSON")])
|
||||
text = "the log shows [PII_PERSON_1]"
|
||||
out, res = pipeline.sterilize(text, "s", self.vault, lambda t: [])
|
||||
self.assertEqual(out, text)
|
||||
self.assertEqual(res["stats"]["literal_tokens_escaped"], 0)
|
||||
self.assertEqual(pipeline.rehydrate(out, "s", self.vault)[0], "the log shows Marisol")
|
||||
|
||||
def test_escape_held_literals_escapes_a_live_placeholder(self):
|
||||
# raw documents: a quoted placeholder is never another text's value
|
||||
pipeline.sterilize("mail ana@x.org", "s", self.vault, lambda t: [])
|
||||
text = "quoted [PII_EMAIL_1] here"
|
||||
kept, _ = pipeline.sterilize(text, "s", self.vault, lambda t: [])
|
||||
self.assertEqual(kept, text) # default: the live token
|
||||
out, res = pipeline.sterilize(text, "s", self.vault, lambda t: [], escape_held_literals=True)
|
||||
self.assertNotIn("[PII_EMAIL_1]", out)
|
||||
self.assertEqual(res["stats"]["literal_tokens_escaped"], 1)
|
||||
self.assertEqual(pipeline.rehydrate(out, "s", self.vault)[0], text)
|
||||
|
||||
def test_each_distinct_placeholder_is_looked_up_once(self):
|
||||
text = " ".join(["[PII_EMAIL_%d]" % (i % 3) for i in range(300)])
|
||||
calls = []
|
||||
real = self.vault.value_for
|
||||
|
||||
def counting(scope, token):
|
||||
calls.append(token)
|
||||
return real(scope, token)
|
||||
|
||||
with mock.patch.object(self.vault, "value_for", counting):
|
||||
pipeline.sterilize(text, "s", self.vault, lambda t: [])
|
||||
self.assertEqual(sorted(calls), ["[PII_EMAIL_0]", "[PII_EMAIL_1]", "[PII_EMAIL_2]"])
|
||||
|
||||
def test_resterilizing_stored_text_is_stable(self):
|
||||
# a stored conversation is sent back to sterilize on every save:
|
||||
# three load/append/save rounds, then one rehydrate gives the original
|
||||
scope = "thread:abc"
|
||||
msgs = [{"role": "user", "content": "Email Marisol at marisol@acme.org"}]
|
||||
model = lambda t: [("Marisol", "PERSON")] if "Marisol" in t else []
|
||||
stored, _ = pipeline.sterilize(json.dumps(msgs), scope, self.vault, model)
|
||||
rows_after_first_save = len(self.vault.scope_tokens(scope))
|
||||
for i in range(3):
|
||||
loaded = json.loads(stored)
|
||||
loaded.append({"role": "assistant", "content": "ok %d" % i})
|
||||
msgs.append({"role": "assistant", "content": "ok %d" % i})
|
||||
stored, res = pipeline.sterilize(json.dumps(loaded), scope, self.vault, model)
|
||||
self.assertEqual(res["stats"]["literal_tokens_escaped"], 0)
|
||||
self.assertEqual(json.loads(stored)[0]["content"], "Email [PII_PERSON_1] at [PII_EMAIL_1]")
|
||||
self.assertEqual(len(self.vault.scope_tokens(scope)), rows_after_first_save)
|
||||
back, _ = pipeline.rehydrate(stored, scope, self.vault)
|
||||
self.assertEqual(json.loads(back), msgs)
|
||||
|
||||
def test_touching_matches_stay_two_tokens(self):
|
||||
# adjacent is not overlapping: each value keeps its own token
|
||||
text = "MarisolLopez and LopezMarisol"
|
||||
out, res = pipeline.sterilize(text, "s", self.vault,
|
||||
lambda t: [("Marisol", "PERSON"), ("Lopez", "PERSON")])
|
||||
self.assertEqual(out, "[PII_PERSON_1][PII_PERSON_2] and [PII_PERSON_2][PII_PERSON_1]")
|
||||
self.assertEqual(len(res["findings"]), 2)
|
||||
back, _ = pipeline.rehydrate(out, "s", self.vault)
|
||||
self.assertEqual(back, text)
|
||||
|
||||
def test_verify_value_cannot_break_a_token(self):
|
||||
# a verify pass reading sterilized text may propose pieces of a
|
||||
# placeholder; none may split one, and the round trip must hold
|
||||
self._n = 0
|
||||
for piece in ("PII_PERSON_1", "PERSON", "PII", "_1"):
|
||||
with self.subTest(piece):
|
||||
out, _ = self.assertTokenizedAndRoundTrips(
|
||||
"Marisol called", [("Marisol", "PERSON")], "Marisol",
|
||||
verify=lambda t, piece=piece: [(piece, "OTHER")], label=piece)
|
||||
self.assertEqual(out, "[PII_PERSON_1] called")
|
||||
|
||||
def test_bare_token_name_is_not_usable(self):
|
||||
grounded, dropped = pipeline.ground([("PII_PERSON_1", "OTHER"), ("x PII_EMAIL_2 y", "OTHER")],
|
||||
"PII_PERSON_1 x PII_EMAIL_2 y")
|
||||
self.assertEqual((grounded, dropped), ([], 2))
|
||||
|
||||
def test_residuals_are_counted(self):
|
||||
# the first rules pass misses the email; the residual loop must
|
||||
# tokenize it AND report it, even though the next round is clean
|
||||
real = detect.detect
|
||||
calls = []
|
||||
|
||||
def first_pass_blind(text):
|
||||
calls.append(text)
|
||||
return [] if len(calls) == 1 else real(text)
|
||||
|
||||
with mock.patch.object(detect, "detect", first_pass_blind):
|
||||
out, res = pipeline.sterilize("mail a@b.com and c@d.com", "s", self.vault, lambda t: [])
|
||||
self.assertEqual(out, "mail [PII_EMAIL_1] and [PII_EMAIL_2]")
|
||||
self.assertEqual(res["stats"]["residual_tokenized"], 2)
|
||||
self.assertEqual(res["stats"]["rules"], 0)
|
||||
|
||||
def test_residuals_summed_across_rounds(self):
|
||||
# two residual rounds that each find something: the stat is their
|
||||
# sum, not the last round's count
|
||||
real = detect.detect
|
||||
calls = []
|
||||
|
||||
def one_at_a_time(text):
|
||||
calls.append(text)
|
||||
if len(calls) == 1:
|
||||
return []
|
||||
return real(text)[:1]
|
||||
|
||||
with mock.patch.object(detect, "detect", one_at_a_time):
|
||||
out, res = pipeline.sterilize("mail a@b.com and c@d.com", "s", self.vault, lambda t: [])
|
||||
self.assertEqual(out, "mail [PII_EMAIL_1] and [PII_EMAIL_2]")
|
||||
self.assertEqual(res["stats"]["residual_tokenized"], 2)
|
||||
|
||||
def test_no_residuals_reports_zero(self):
|
||||
_, res = pipeline.sterilize("ssn 123-45-6789", "s", self.vault, lambda t: [])
|
||||
self.assertEqual(res["stats"]["residual_tokenized"], 0)
|
||||
|
||||
def test_verify_pass_tokenizes_grounded_hits(self):
|
||||
text = "the patient is Marisol"
|
||||
fake_extract = lambda t: []
|
||||
fake_verify = lambda t: [("Marisol", "PERSON")]
|
||||
out, res = pipeline.sterilize(text, "s", self.vault, fake_extract, fake_verify)
|
||||
self.assertEqual(out, "the patient is [PII_PERSON_1]")
|
||||
self.assertEqual(res["stats"]["verify_tokenized"], 1)
|
||||
|
||||
def test_verify_counts_only_what_it_replaced(self):
|
||||
# grounded but replacing nothing: a piece of an existing token, and a
|
||||
# value already tokenized by the first pass
|
||||
fake_verify = lambda t: [("PERSON", "OTHER"), ("Marisol", "PERSON"), ("Lopez", "PERSON")]
|
||||
out, res = pipeline.sterilize("Marisol Lopez called", "s", self.vault,
|
||||
lambda t: [("Marisol", "PERSON")], fake_verify)
|
||||
self.assertEqual(out, "[PII_PERSON_1] [PII_PERSON_2] called")
|
||||
self.assertEqual(res["stats"]["verify_tokenized"], 1)
|
||||
|
||||
|
||||
class PromptContractTests(unittest.TestCase):
|
||||
def test_prompts_state_full_type_vocabulary(self):
|
||||
# The grammar prefix-coerces free-text labels to enum members
|
||||
# ("School" -> SSN, observed live). The prompts must therefore
|
||||
# state every legal label verbatim; this guards enum/prompt drift.
|
||||
from . import llmclient
|
||||
for prompt in (llmclient.EXTRACT_SYSTEM, llmclient.VERIFY_SYSTEM):
|
||||
for typ in detect.PII_TYPES:
|
||||
self.assertIn(typ, prompt)
|
||||
|
||||
|
||||
class _FakeGliner:
|
||||
"""Stands in for gliner.GLiNER: returns canned entities, records labels."""
|
||||
entities = []
|
||||
calls = []
|
||||
|
||||
def predict_entities(self, text, labels, threshold=0.5):
|
||||
_FakeGliner.calls.append((labels, threshold))
|
||||
return list(self.entities)
|
||||
|
||||
|
||||
class RecallBackendTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
from . import recall
|
||||
self.recall = recall
|
||||
_FakeGliner.entities, _FakeGliner.calls = [], []
|
||||
|
||||
def test_default_is_llm(self):
|
||||
b = self.recall.build_backend({"LLM_URL": "http://x:1"})
|
||||
self.assertEqual(b.name, "llm")
|
||||
|
||||
def test_unknown_backend_refused(self):
|
||||
with self.assertRaises(self.recall.RecallError):
|
||||
self.recall.build_backend({"PII_RECALL": "cloud"})
|
||||
|
||||
def test_gliner_missing_package_fails_closed(self):
|
||||
def loader(_):
|
||||
raise ImportError("No module named 'gliner'")
|
||||
with self.assertRaises(self.recall.RecallError) as ctx:
|
||||
self.recall.build_backend({"PII_RECALL": "gliner"}, loader=loader)
|
||||
self.assertIn("pip install gliner", str(ctx.exception))
|
||||
|
||||
def test_gliner_maps_labels_to_vocabulary_and_dedupes(self):
|
||||
_FakeGliner.entities = [
|
||||
{"text": "Marisol", "label": "person name", "score": 0.9},
|
||||
{"text": "Marisol", "label": "person name", "score": 0.8},
|
||||
{"text": "Lincoln Elementary", "label": "employer or school or organization"},
|
||||
{"text": "junk", "label": "not a label we asked for"},
|
||||
]
|
||||
b = self.recall.build_backend(
|
||||
{"PII_RECALL": "gliner", "GLINER_THRESHOLD": "0.6"}, loader=lambda _: _FakeGliner())
|
||||
out = b.extract("Marisol goes to Lincoln Elementary")
|
||||
self.assertEqual(out, [("Marisol", "PERSON"), ("Lincoln Elementary", "ORG")])
|
||||
labels, threshold = _FakeGliner.calls[-1]
|
||||
self.assertEqual(set(labels), set(self.recall.GLINER_LABELS))
|
||||
self.assertEqual(threshold, 0.6)
|
||||
for typ in set(self.recall.GLINER_LABELS.values()):
|
||||
self.assertIn(typ, detect.PII_TYPES)
|
||||
|
||||
def test_gliner_verify_uses_lower_threshold(self):
|
||||
b = self.recall.build_backend({"PII_RECALL": "gliner"}, loader=lambda _: _FakeGliner())
|
||||
b.verify("x")
|
||||
self.assertEqual(_FakeGliner.calls[-1][1], self.recall.DEFAULT_GLINER_VERIFY_THRESHOLD)
|
||||
|
||||
def test_gliner_prediction_failure_is_recall_error(self):
|
||||
class Broken:
|
||||
def predict_entities(self, *a, **k):
|
||||
raise RuntimeError("cuda gone")
|
||||
b = self.recall.build_backend({"PII_RECALL": "gliner"}, loader=lambda _: Broken())
|
||||
with self.assertRaises(self.recall.RecallError):
|
||||
b.extract("x")
|
||||
|
||||
def test_llm_error_is_a_recall_error(self):
|
||||
from . import llmclient
|
||||
self.assertTrue(issubclass(llmclient.LlmError, self.recall.RecallError))
|
||||
|
||||
def test_gliner_backend_through_pipeline(self):
|
||||
_FakeGliner.entities = [{"text": "Marisol", "label": "person name"},
|
||||
{"text": "Ghost", "label": "person name"}]
|
||||
b = self.recall.build_backend({"PII_RECALL": "gliner"}, loader=lambda _: _FakeGliner())
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
try:
|
||||
vault = Vault(os.path.join(tmp.name, "v.db"))
|
||||
out, res = pipeline.sterilize("Marisol called 415-555-0173", "s", vault, b.extract)
|
||||
finally:
|
||||
tmp.cleanup()
|
||||
self.assertEqual(out, "[PII_PERSON_1] called [PII_PHONE_1]")
|
||||
self.assertEqual(res["stats"]["llm_ungrounded"], 1) # "Ghost" was not in the text
|
||||
|
||||
|
||||
class _StubLlm(BaseHTTPRequestHandler):
|
||||
"""Speaks llama.cpp's chat-completions shape; extraction canned per test."""
|
||||
findings = []
|
||||
|
||||
def do_POST(self):
|
||||
length = int(self.headers.get("Content-Length", "0"))
|
||||
self.rfile.read(length)
|
||||
content = json.dumps({"findings": self.findings})
|
||||
body = json.dumps(
|
||||
{"choices": [{"message": {"content": content}}]}
|
||||
).encode()
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def log_message(self, *a):
|
||||
pass
|
||||
|
||||
|
||||
class ServiceTests(unittest.TestCase):
|
||||
KEY = "test-key-123"
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.stub = ThreadingHTTPServer(("127.0.0.1", 0), _StubLlm)
|
||||
threading.Thread(target=cls.stub.serve_forever, daemon=True).start()
|
||||
cls.tmp = tempfile.TemporaryDirectory()
|
||||
cfg = service.Config({
|
||||
"PII_BIND": "127.0.0.1:0",
|
||||
"LLM_URL": f"http://127.0.0.1:{cls.stub.server_address[1]}",
|
||||
"PII_DB": os.path.join(cls.tmp.name, "v.db"),
|
||||
"PII_API_KEY": cls.KEY,
|
||||
})
|
||||
cls.srv = service.build_server(cfg)
|
||||
cls.port = cls.srv.server_address[1]
|
||||
threading.Thread(target=cls.srv.serve_forever, daemon=True).start()
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
cls.srv.shutdown()
|
||||
cls.stub.shutdown()
|
||||
cls.tmp.cleanup()
|
||||
|
||||
def call(self, path, payload=None, key=None, method=None):
|
||||
url = f"http://127.0.0.1:{self.port}{path}"
|
||||
data = json.dumps(payload).encode() if payload is not None else None
|
||||
req = urllib.request.Request(url, data=data, method=method)
|
||||
req.add_header("Content-Type", "application/json")
|
||||
if key:
|
||||
req.add_header("Authorization", f"Bearer {key}")
|
||||
with urllib.request.urlopen(req, timeout=10) as r:
|
||||
return r.status, json.load(r)
|
||||
|
||||
def test_health_no_auth(self):
|
||||
status, body = self.call("/health")
|
||||
self.assertEqual((status, body["ok"]), (200, True))
|
||||
|
||||
def test_auth_required(self):
|
||||
with self.assertRaises(urllib.error.HTTPError) as ctx:
|
||||
self.call("/sterilize", {"text": "x"}, key="wrong")
|
||||
self.assertEqual(ctx.exception.code, 401)
|
||||
|
||||
def test_sterilize_rehydrate_over_http(self):
|
||||
_StubLlm.findings = [{"text": "Marisol", "type": "PERSON"}]
|
||||
original = "Marisol's ssn is 123-45-6789, mail her at m@example.com"
|
||||
_, st = self.call("/sterilize", {"text": original, "scope": "conv1"}, key=self.KEY)
|
||||
for leak in ("Marisol", "123-45-6789", "m@example.com"):
|
||||
self.assertNotIn(leak, st["text"])
|
||||
self.assertEqual(st["stats"]["rules"], 2)
|
||||
_, back = self.call("/rehydrate", {"text": st["text"], "scope": "conv1"}, key=self.KEY)
|
||||
self.assertEqual(back["text"], original)
|
||||
# wrong scope must not resolve
|
||||
_, other = self.call("/rehydrate", {"text": st["text"], "scope": "conv2"}, key=self.KEY)
|
||||
self.assertEqual(other["restored"], 0)
|
||||
|
||||
def test_flag_rules_only_no_vault_rows(self):
|
||||
_StubLlm.findings = [{"text": "should never be called", "type": "PERSON"}]
|
||||
_, body = self.call("/flag", {"text": "ssn 123-45-6789 and a@b.com"}, key=self.KEY)
|
||||
self.assertTrue(body["flagged"])
|
||||
self.assertEqual(body["counts"], {"SSN": 1, "EMAIL": 1})
|
||||
_, clean = self.call("/flag", {"text": "nothing sensitive here"}, key=self.KEY)
|
||||
self.assertEqual((clean["flagged"], clean["counts"]), (False, {}))
|
||||
# no scope was touched, nothing minted
|
||||
_, inv = self.call("/scope/default", key=self.KEY)
|
||||
self.assertEqual(inv["tokens"], [])
|
||||
|
||||
def test_escape_held_literals_over_http(self):
|
||||
_StubLlm.findings = []
|
||||
self.call("/sterilize", {"text": "mail a@b.com", "scope": "esc"}, key=self.KEY)
|
||||
quote = "quoted [PII_EMAIL_1]"
|
||||
_, kept = self.call("/sterilize", {"text": quote, "scope": "esc"}, key=self.KEY)
|
||||
self.assertEqual(kept["text"], quote) # default: a re-save keeps the token
|
||||
_, esc = self.call("/sterilize", {"text": quote, "scope": "esc",
|
||||
"escape_held_literals": True}, key=self.KEY)
|
||||
self.assertNotIn("[PII_EMAIL_1]", esc["text"])
|
||||
_, back = self.call("/rehydrate", {"text": esc["text"], "scope": "esc"}, key=self.KEY)
|
||||
self.assertEqual(back["text"], quote)
|
||||
# the rule applied is echoed, so a caller can tell it was honoured
|
||||
self.assertIs(kept["stats"]["escape_held_literals"], False)
|
||||
self.assertIs(esc["stats"]["escape_held_literals"], True)
|
||||
|
||||
def test_scope_inventory_no_values(self):
|
||||
_StubLlm.findings = []
|
||||
self.call("/sterilize", {"text": "a@b.com", "scope": "inv"}, key=self.KEY)
|
||||
_, body = self.call("/scope/inv", key=self.KEY)
|
||||
self.assertEqual(body["tokens"][0]["token"], "[PII_EMAIL_1]")
|
||||
self.assertNotIn("a@b.com", json.dumps(body))
|
||||
|
||||
def test_fail_closed_without_key(self):
|
||||
with self.assertRaises(SystemExit):
|
||||
service.Config({"PII_DB": "x"})
|
||||
|
||||
def test_fail_closed_on_unstartable_recall_backend(self):
|
||||
with self.assertRaises(SystemExit) as ctx:
|
||||
service.Config({"PII_DB": "x", "PII_API_KEY": "k", "PII_RECALL": "nope"})
|
||||
self.assertIn("PII_RECALL", str(ctx.exception))
|
||||
|
||||
def test_sterilize_reports_recall_backend(self):
|
||||
_StubLlm.findings = []
|
||||
_, st = self.call("/sterilize", {"text": "a@b.com", "scope": "rb"}, key=self.KEY)
|
||||
self.assertEqual(st["stats"]["recall"], "llm")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
+100
@@ -0,0 +1,100 @@
|
||||
"""Local PII vault — token <-> value mapping plus audit trail, SQLite.
|
||||
|
||||
Tokens are stable per (scope, type, value): the same email in the same
|
||||
scope always maps to the same token, so a cloud model can reason about
|
||||
[PII_PERSON_1] coherently across a whole conversation. The same value in
|
||||
a DIFFERENT scope gets a different token — scopes exist so two
|
||||
conversations cannot be cross-linked through shared placeholders.
|
||||
|
||||
Sequence numbers are per (scope, type) and rows are never deleted, so
|
||||
COUNT()+1 is a safe next-sequence source under the module lock. After a
|
||||
delete, COUNT()+1 names a token that still exists unless the deleted row
|
||||
was the newest, and PRIMARY KEY (scope, token) then makes the insert fail;
|
||||
if it was the newest, its token is minted again for a different value.
|
||||
So erasure needs a separate sequence first. Values are plaintext in the file: the controls are the
|
||||
file system (0600, umask set by the caller) and, for the service, its
|
||||
bind address. Encryption at rest would be a design change of its own, not
|
||||
something to improvise here.
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import os
|
||||
import sqlite3
|
||||
import threading
|
||||
|
||||
_SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS pii (
|
||||
scope TEXT NOT NULL,
|
||||
token TEXT NOT NULL,
|
||||
type TEXT NOT NULL,
|
||||
value TEXT NOT NULL,
|
||||
created TEXT NOT NULL,
|
||||
PRIMARY KEY (scope, token),
|
||||
UNIQUE (scope, type, value)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS audit (
|
||||
ts TEXT NOT NULL,
|
||||
scope TEXT NOT NULL,
|
||||
action TEXT NOT NULL,
|
||||
detail TEXT NOT NULL
|
||||
);
|
||||
"""
|
||||
|
||||
|
||||
def _now() -> str:
|
||||
return datetime.datetime.now(datetime.timezone.utc).isoformat(timespec="seconds")
|
||||
|
||||
|
||||
class Vault:
|
||||
def __init__(self, path: str):
|
||||
self.path = path
|
||||
self._lock = threading.Lock()
|
||||
d = os.path.dirname(path)
|
||||
if d:
|
||||
os.makedirs(d, mode=0o700, exist_ok=True)
|
||||
with self._conn() as c:
|
||||
c.executescript(_SCHEMA)
|
||||
|
||||
def _conn(self) -> sqlite3.Connection:
|
||||
return sqlite3.connect(self.path, timeout=30)
|
||||
|
||||
def token_for(self, scope: str, typ: str, value: str) -> str:
|
||||
with self._lock, self._conn() as c:
|
||||
row = c.execute(
|
||||
"SELECT token FROM pii WHERE scope=? AND type=? AND value=?",
|
||||
(scope, typ, value),
|
||||
).fetchone()
|
||||
if row:
|
||||
return row[0]
|
||||
n = c.execute(
|
||||
"SELECT COUNT(*) FROM pii WHERE scope=? AND type=?", (scope, typ)
|
||||
).fetchone()[0] + 1
|
||||
token = f"[PII_{typ}_{n}]"
|
||||
c.execute(
|
||||
"INSERT INTO pii (scope, token, type, value, created) VALUES (?,?,?,?,?)",
|
||||
(scope, token, typ, value, _now()),
|
||||
)
|
||||
return token
|
||||
|
||||
def value_for(self, scope: str, token: str):
|
||||
with self._lock, self._conn() as c:
|
||||
row = c.execute(
|
||||
"SELECT value FROM pii WHERE scope=? AND token=?", (scope, token)
|
||||
).fetchone()
|
||||
return row[0] if row else None
|
||||
|
||||
def scope_tokens(self, scope: str) -> list:
|
||||
"""Token inventory for a scope — types and tokens only, never values."""
|
||||
with self._lock, self._conn() as c:
|
||||
rows = c.execute(
|
||||
"SELECT token, type, created FROM pii WHERE scope=? ORDER BY created, token",
|
||||
(scope,),
|
||||
).fetchall()
|
||||
return [{"token": t, "type": ty, "created": cr} for t, ty, cr in rows]
|
||||
|
||||
def audit(self, scope: str, action: str, detail: str):
|
||||
with self._lock, self._conn() as c:
|
||||
c.execute(
|
||||
"INSERT INTO audit (ts, scope, action, detail) VALUES (?,?,?,?)",
|
||||
(_now(), scope, action, detail),
|
||||
)
|
||||
Executable
+70
@@ -0,0 +1,70 @@
|
||||
#!/usr/bin/env bash
|
||||
# Live verification of a running PII sterilization service and its real
|
||||
# recall model (the unit tests use a stub; this is the check against the
|
||||
# model itself):
|
||||
#
|
||||
# PII_API_KEY=<the service's key> pii/verify-live.sh [host:port]
|
||||
#
|
||||
# host:port defaults to 127.0.0.1:8091, where pii.service binds.
|
||||
set -u
|
||||
HOST="${1:-127.0.0.1:8091}"
|
||||
KEY="${PII_API_KEY:-}"
|
||||
[ -n "$KEY" ] || { echo "set PII_API_KEY to the service's key"; exit 1; }
|
||||
|
||||
pass=0; fail=0
|
||||
ok() { pass=$((pass+1)); echo "ok $1"; }
|
||||
bad() { fail=$((fail+1)); echo "FAIL $1"; }
|
||||
|
||||
post() { # path json -> body
|
||||
curl -sf -m 600 -H "Authorization: Bearer $KEY" -H 'Content-Type: application/json' \
|
||||
-d "$2" "http://$HOST$1"
|
||||
}
|
||||
jget() { # json pyexpr
|
||||
python3 -c "import json,sys; d=json.load(sys.stdin); print($1)"
|
||||
}
|
||||
|
||||
# 1. health, no auth
|
||||
curl -sf "http://$HOST/health" >/dev/null && ok "health" || bad "health"
|
||||
|
||||
# 2. auth enforced
|
||||
code=$(curl -s -o /dev/null -w '%{http_code}' -d '{"text":"x"}' "http://$HOST/sterilize")
|
||||
[ "$code" = 401 ] && ok "auth fail-closed (401 without key)" || bad "auth: got $code"
|
||||
|
||||
SCOPE="verify-$$"
|
||||
|
||||
# 3. rules-class PII all tokenized
|
||||
T1='Marisol Vega (SSN 123-45-6789) can be reached at marisol.vega@example.com or (415) 555-0173. Card on file: 4111 1111 1111 1111. Her laptop is 10.20.30.44 / aa:bb:cc:dd:ee:01.'
|
||||
R1=$(post /sterilize "$(python3 -c "import json;print(json.dumps({'text':'''$T1''','scope':'$SCOPE'}))")")
|
||||
leak=$(echo "$R1" | jget "sum(x in d['text'] for x in ['Marisol','123-45-6789','marisol.vega','555-0173','4111','10.20.30.44','aa:bb:cc:dd:ee:01'])")
|
||||
[ "$leak" = 0 ] && ok "sterilize: no rules-class or name leakage" || bad "sterilize leaked $leak spans: $(echo "$R1" | jget "d['text']")"
|
||||
|
||||
# 4. contextual PII a pattern cannot see (why the recall model is there)
|
||||
T2='My daughter goes to Lincoln Elementary and we live at 4012 Maple Court. She sees Dr. Okafor for her asthma.'
|
||||
R2=$(post /sterilize "$(python3 -c "import json;print(json.dumps({'text':'''$T2''','scope':'$SCOPE','verify':True}))")")
|
||||
leak=$(echo "$R2" | jget "sum(x in d['text'] for x in ['Lincoln','4012','Maple','Okafor','asthma'])")
|
||||
[ "$leak" = 0 ] && ok "contextual: school/address/doctor/medical caught" || bad "contextual leaked $leak: $(echo "$R2" | jget "d['text']")"
|
||||
|
||||
# 5. round trip back to the original
|
||||
B1=$(post /rehydrate "$(echo "$R1" | jget "json.dumps({'text': d['text'], 'scope': '$SCOPE'})")")
|
||||
same=$(echo "$B1" | jget "d['text'] == '''$T1'''")
|
||||
[ "$same" = True ] && ok "rehydrate: byte-identical round trip" || bad "rehydrate mismatch"
|
||||
|
||||
# 6. token stability within scope
|
||||
R3=$(post /sterilize "$(python3 -c "import json;print(json.dumps({'text':'Ping marisol.vega@example.com again.','scope':'$SCOPE'}))")")
|
||||
tok1=$(echo "$R1" | jget "[f['token'] for f in d['findings'] if f['type']=='EMAIL'][0]")
|
||||
stable=$(echo "$R3" | jget "'$tok1' in d['text']")
|
||||
[ "$stable" = True ] && ok "stable token reuse in scope ($tok1)" || bad "token not stable"
|
||||
|
||||
# 7. scope isolation on recall
|
||||
B2=$(post /rehydrate "$(echo "$R1" | jget "json.dumps({'text': d['text'], 'scope': 'other-scope'})")")
|
||||
n=$(echo "$B2" | jget "d['restored']")
|
||||
[ "$n" = 0 ] && ok "scope isolation: foreign scope restores nothing" || bad "cross-scope restore: $n"
|
||||
|
||||
# 8. non-PII technical text byte-identical
|
||||
T4='Set net.core.rmem_max=26214400, then restart the collector; DSA ports stay tagged.'
|
||||
R4=$(post /sterilize "$(python3 -c "import json;print(json.dumps({'text':'''$T4''','scope':'$SCOPE'}))")")
|
||||
same=$(echo "$R4" | jget "d['text'] == '''$T4'''")
|
||||
[ "$same" = True ] && ok "non-PII text untouched" || bad "non-PII mutated: $(echo "$R4" | jget "d['text']")"
|
||||
|
||||
echo "pii live verify: $pass/$((pass+fail))"
|
||||
[ "$fail" = 0 ]
|
||||
Reference in New Issue
Block a user