"""Deterministic PII detectors — the high-precision anchor pass. Two jobs: (1) anchor the sterilizer with detections that never depend on model behavior, and (2) re-scan sterilized output as the residual check (pipeline.py loops until this pass finds nothing). Recall for contextual PII — names, addresses, employers, "my daughter's school" — is the LLM pass's job (llmclient.py); here precision is deliberately favored over recall, because a miss still has two more passes to be caught while a false positive burns a vault row forever. Overlapping candidates are resolved by type priority then span length: an IPv4 address must not be re-claimed as a dotted phone number, a MAC must not be claimed as truncated IPv6. """ import hashlib import ipaddress import re from dataclasses import dataclass from typing import List # Full type vocabulary — shared with the LLM extraction schema. Rules # only ever emit the subset in _DETECTORS; the rest are LLM-only types. PII_TYPES = [ "PERSON", "EMAIL", "PHONE", "ADDRESS", "SSN", "DOB", "CREDIT_CARD", "BANK_ACCOUNT", "IP_ADDRESS", "MAC_ADDRESS", "USERNAME", "CREDENTIAL", "MEDICAL", "ID_NUMBER", "ORG", "LOCATION", "OTHER", ] @dataclass(frozen=True) class Finding: value: str type: str start: int end: int EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)*\.[A-Za-z]{2,}") MAC_RE = re.compile( r"(? bool: total = 0 for i, ch in enumerate(reversed(digits)): d = int(ch) if i % 2 == 1: d *= 2 if d > 9: d -= 9 total += d return total % 10 == 0 def _find_emails(text): for m in EMAIL_RE.finditer(text): yield Finding(m.group(0), "EMAIL", m.start(), m.end()) def _find_macs(text): for m in MAC_RE.finditer(text): yield Finding(m.group(0), "MAC_ADDRESS", m.start(), m.end()) def _find_ips(text): for m in IPV4_RE.finditer(text): try: ipaddress.IPv4Address(m.group(0)) except ValueError: continue yield Finding(m.group(0), "IP_ADDRESS", m.start(), m.end()) for m in IPV6_RE.finditer(text): cand = m.group(0) try: ipaddress.IPv6Address(cand) except ValueError: continue yield Finding(cand, "IP_ADDRESS", m.start(), m.end()) def _find_ssns(text): for m in SSN_RE.finditer(text): area = int(m.group(1)) if area == 0 or area == 666 or area >= 900: continue yield Finding(m.group(0), "SSN", m.start(), m.end()) def _find_cards(text): for m in CARD_RE.finditer(text): digits = re.sub(r"\D", "", m.group(0)) if 13 <= len(digits) <= 19 and _luhn_ok(digits): yield Finding(m.group(0), "CREDIT_CARD", m.start(), m.end()) def _find_phones(text): for m in PHONE_RE.finditer(text): digits = re.sub(r"\D", "", m.group(0)) if not 10 <= len(digits) <= 15: continue yield Finding(m.group(0), "PHONE", m.start(), m.end()) # Priority order: earlier detectors own their spans outright. Secrets go # first (a PEM block or JWT contains digit runs that would otherwise be # claimed piecemeal); IBAN before cards (an IBAN's digits can pass Luhn). _DETECTORS = [ _find_secrets, _find_emails, _find_macs, _find_ips, _find_ibans, _find_btc, _find_ssns, _find_cards, _find_abas, _find_phones, ] def detect(text: str) -> List[Finding]: """All rule findings, non-overlapping, priority- then position-ordered.""" candidates = [] for prio, fn in enumerate(_DETECTORS): for f in fn(text): candidates.append((prio, -(f.end - f.start), f.start, f)) accepted = [] # type: List[Finding] for _, _, _, f in sorted(candidates, key=lambda c: (c[0], c[1], c[2])): if all(f.end <= a.start or f.start >= a.end for a in accepted): accepted.append(f) return sorted(accepted, key=lambda f: f.start)