diff --git a/src/lib.rs b/src/lib.rs index fd825d0..8f77bcf 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -17,7 +17,12 @@ //! * `report` — a JSON document listing every redaction and a //! per-category counter, plus the full `original` text so a //! downstream verify-step can confirm the redaction set is -//! exhaustive (GDPR-mapping note in `module.yaml`). +//! exhaustive (GDPR-mapping note in `module.yaml`). The report +//! also carries a `coverage` object that honestly declares what +//! the module does NOT catch (no NER, no address detection) and +//! a `warning` string noting that the report itself embeds the +//! original PII — storing or forwarding it is a processing +//! activity in its own right. //! //! # Categories //! @@ -47,9 +52,15 @@ //! - `IPV4` — four 1..3-digit octets, dot-separated. Skips //! matches where the value would be > 255 to avoid eating //! "1.2.3.4-style version numbers" of >255. -//! - `GERMAN_TAX_ID` — the 11-digit Steueridentifikationsnummer. -//! Matched as a standalone token surrounded by word -//! boundaries. +//! - `GERMAN_TAX_ID` — the 11-digit Steueridentifikationsnummer, +//! in either spelling: contiguous (`12345678901`) or the official +//! 2-3-3-3 grouping printed on tax office letters +//! (`12 345 678 901`, dot separators too). Matched structurally +//! with word boundaries; the checksum is deliberately NOT computed +//! and separators are NOT required, so any 11-digit token is +//! redacted. That over-redacts (a random 11-digit order number is +//! caught too) rather than risk missing a real Steuer-ID — an +//! honest, deliberate tradeoff, also stated in `module.yaml`. //! - `CUSTOM` — every entry from `custom_terms`, matched as a //! whole word (Unicode-case-insensitive). //! @@ -106,10 +117,61 @@ struct Redaction { offset: usize, } +/// Honest declaration of what this module does and does NOT catch. +/// Clerks read the `report` output as ground truth, so this object +/// exists to keep that trust from being misplaced. It describes the +/// module's capability, not the individual run. +#[derive(Debug, Serialize)] +struct Coverage { + /// Person / organisation names: only the operator-supplied + /// `custom_terms` list is matched. There is no NER model, so any + /// free-text name not on that list passes through unredacted. + names: &'static str, + /// Postal addresses are not detected at all. + addresses: &'static str, + /// The structured categories that ARE checked on every run. + categories_checked: Vec<&'static str>, + /// The tax-id rule is structural: any 11-digit token (grouped or + /// contiguous) is redacted with no checksum validation. It + /// over-redacts rather than risk missing a real Steuer-ID. + tax_id_matching: &'static str, +} + +/// Plain-language warning attached to every report. The report +/// embeds the original PII (`redactions[].original` plus offsets), +/// so the report is itself a copy of the sensitive data. +const REPORT_WARNING: &str = "This report contains the original PII in the \ +`redactions[].original` and `offset` fields. Storing, logging, or forwarding \ +this report is itself a processing activity that re-exposes the redacted data \ +— protect it exactly like the source document, or discard it once the \ +verify-pass is done."; + +/// Build the static coverage declaration. Kept as a function so the +/// category list stays a single source of truth next to the patterns. +fn build_coverage() -> Coverage { + Coverage { + names: "list-only (custom_terms); no NER model", + addresses: "not detected", + categories_checked: vec![ + "EMAIL", + "PHONE", + "IBAN", + "BIC", + "IPV4", + "GERMAN_TAX_ID", + "CUSTOM", + ], + tax_id_matching: "structural: any 11-digit token (2-3-3-3 grouped or \ +contiguous), no checksum — also redacts non-tax-id 11-digit numbers", + } +} + #[derive(Debug, Serialize)] struct Report { redactions: Vec, counts: HashMap, + coverage: Coverage, + warning: &'static str, } #[fai_module] @@ -126,7 +188,12 @@ pub fn invoke(_ctx: Context, inputs: Inputs) -> Result { for r in &redactions { *counts.entry(r.kind.clone()).or_insert(0) += 1; } - let report = Report { redactions, counts }; + let report = Report { + redactions, + counts, + coverage: build_coverage(), + warning: REPORT_WARNING, + }; Outputs::new() .with_text("anonymized", anonymized) @@ -205,10 +272,18 @@ fn anonymize(text: &str, custom_raw: &str) -> Result<(String, Vec), M ), ( "GERMAN_TAX_ID", - // 11 consecutive digits, word-bounded. The official - // checksum we deliberately don't compute — partial - // tokens still warrant redaction in raw text. - Regex::new(r"(?x)\b\d{11}\b").map_err(re_err)?, + // The 11-digit Steueridentifikationsnummer in either + // spelling: contiguous (`12345678901`) or the official + // 2-3-3-3 grouping printed on tax office letters + // (`12 345 678 901`, dot separators too). Separators are + // optional between the groups, so one pattern covers + // both. Structural only: the checksum is deliberately not + // computed and separators are not required, so any + // 11-digit token is redacted (honest false-positive note + // in module.yaml). Not written in (?x) verbose mode + // because the literal space in the `[ .]` class would be + // easy to misread there. + Regex::new(r"\b\d{2}[ .]?\d{3}[ .]?\d{3}[ .]?\d{3}\b").map_err(re_err)?, ), ]; @@ -432,4 +507,92 @@ mod tests { assert_eq!(phones.len(), 1); assert_eq!(phones[0].token, "\u{27E6}PHONE_1\u{27E7}"); } + + #[test] + fn redacts_tax_id_contiguous() { + // The plain 11-digit spelling must still be caught. + let (out, reds) = + anonymize("Steuer-ID: 12345678901 im Bescheid.", "").unwrap(); + assert!(out.contains("\u{27E6}GERMAN_TAX_ID_1\u{27E7}")); + let hits: Vec<&Redaction> = + reds.iter().filter(|r| r.kind == "GERMAN_TAX_ID").collect(); + assert_eq!(hits.len(), 1); + assert_eq!(hits[0].original, "12345678901"); + } + + #[test] + fn redacts_tax_id_grouped() { + // The official 2-3-3-3 grouping on tax office letters, with + // spaces and with dots — both must be caught, including the + // separators as part of the redacted span. + for raw in ["12 345 678 901", "12.345.678.901"] { + let src = format!("Ihre IdNr. lautet {raw} laut Schreiben."); + let (out, reds) = anonymize(&src, "").unwrap(); + assert!( + out.contains("\u{27E6}GERMAN_TAX_ID_1\u{27E7}"), + "grouped tax id not redacted in: {src}" + ); + let hits: Vec<&Redaction> = + reds.iter().filter(|r| r.kind == "GERMAN_TAX_ID").collect(); + assert_eq!(hits.len(), 1, "expected one tax-id hit for: {src}"); + assert_eq!(hits[0].original, raw); + } + } + + #[test] + fn tax_id_matching_is_structural_over_redacts() { + // Documented tradeoff: matching is structural, not + // checksum-validated, so a random 11-digit number that is NOT + // a real Steuer-ID is still redacted. This test pins that + // behaviour so a future "tighten it up" change is a conscious + // decision, not an accident. + let (out, reds) = + anonymize("Auftragsnummer 98765432100 im System.", "").unwrap(); + assert!(out.contains("\u{27E6}GERMAN_TAX_ID_1\u{27E7}")); + assert_eq!( + reds.iter().filter(|r| r.kind == "GERMAN_TAX_ID").count(), + 1 + ); + } + + #[test] + fn report_declares_coverage_and_warning() { + // The JSON report is trusted by clerks, so it must state its + // own limits and warn that it embeds the original PII. + let text = "kontakt: a@b.de"; + let (_, redactions) = anonymize(text, "").unwrap(); + let mut counts: HashMap = HashMap::new(); + for r in &redactions { + *counts.entry(r.kind.clone()).or_insert(0) += 1; + } + let report = Report { + redactions, + counts, + coverage: build_coverage(), + warning: REPORT_WARNING, + }; + let v = serde_json::to_value(&report).unwrap(); + + // Coverage object is present and honest about the gaps. + assert!(v["coverage"]["names"].as_str().unwrap().contains("no NER")); + assert_eq!(v["coverage"]["addresses"], "not detected"); + let cats: Vec<&str> = v["coverage"]["categories_checked"] + .as_array() + .unwrap() + .iter() + .map(|c| c.as_str().unwrap()) + .collect(); + for expected in ["EMAIL", "PHONE", "IBAN", "BIC", "IPV4", "GERMAN_TAX_ID", "CUSTOM"] { + assert!(cats.contains(&expected), "missing category {expected}"); + } + assert!(v["coverage"]["tax_id_matching"] + .as_str() + .unwrap() + .contains("no checksum")); + + // Warning field is present and names the risk. + let warning = v["warning"].as_str().unwrap(); + assert!(warning.contains("original PII")); + assert!(warning.contains("processing activity")); + } }