From 734d8f6e6f187d7f28d4d1633d7d4897b75d8110 Mon Sep 17 00:00:00 2001 From: flemming-it Date: Mon, 25 May 2026 20:53:24 +0200 Subject: [PATCH] =?UTF-8?q?feat:=20text.anonymize=20v0.1.0=20=E2=80=94=20r?= =?UTF-8?q?egex-based=20PII=20redaction?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First release. Pure-Rust, in-WASM, regex-only, declares no permissions. Suitable as a first redaction pass after text.extract before any cloud-LLM step. Detection categories: EMAIL RFC-5321-ish local@domain, IDN-aware. PHONE International (+CC …) and DE national (030 …, 0151-…) shapes, 7..20 raw digits. IBAN Word-bounded [A-Z]{2}\d{2}[A-Z0-9]{11,30}. Structural only — MOD-97 checksum deliberately skipped so partial / truncated tokens in running text still get redacted. BIC 8 or 11 uppercase alnum. IPV4 Four 0..255 octets, dot-separated. GERMAN_TAX_ID 11 consecutive digits, word-bounded. CUSTOM Operator-supplied bare terms from the newline-separated `custom_terms` input, matched whole-word case-insensitive. Token shape: ⟦TYPE_N⟧ — U+27E6 / U+27E7 mathematical white square brackets. Distinct from any plain ASCII `[…]` already present in source text (Markdown links, legal citations, code blocks) so a reviewer never has to guess which `[…]` is a redaction. Outputs: anonymized text Input with PII replaced by ⟦TYPE_N⟧. Counter restarts at 1 per type so the tokens stay operator-readable. report json { redactions: [{type, token, original, offset}…], counts: { TYPE: n, … } }. Full original-text reconstruction is possible from this — the GDPR Art. 32(1)(a) "ability to undo" requirement. Quality bar (7 unit tests): * email round-trip * IBAN + BIC don't eat each other * three phone-number shapes redact * IPv4 only matches valid 0..255 octets * custom_terms case-insensitive * no double-redaction on overlapping patterns * per-category counter resets correctly Built artefact: target/wasm32-wasip2/release/text_anonymize.wasm (~180 KiB stripped). NER for free-text names / organisations / locations is the v0.2.0 plan once a benchmarked ONNX model is selected; the operator's `custom_terms` field is the v0.1.0 escape hatch. Signed-off-by: flemming-it --- .gitignore | 4 + Cargo.lock | 482 ++++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 37 ++++ MODULE.de.md | 74 +++++++ MODULE.md | 75 +++++++ module.yaml | 53 +++++ rust-toolchain.toml | 4 + src/lib.rs | 435 +++++++++++++++++++++++++++++++++++++++ 8 files changed, 1164 insertions(+) create mode 100644 .gitignore create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 MODULE.de.md create mode 100644 MODULE.md create mode 100644 module.yaml create mode 100644 rust-toolchain.toml create mode 100644 src/lib.rs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..fcc55a5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +target/ +**/*.rs.bk +.DS_Store +module.wasm diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..9f802e8 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,482 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "bitflags" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "fai-module-sdk" +version = "0.1.2" +source = "git+https://git.flemming.ai/fai/module-sdk.git?branch=main#f209fbc86f531b53e576f3cdbdf75750eef3b07a" +dependencies = [ + "fai-module-sdk-macros", + "serde", + "serde_json", + "thiserror", + "wit-bindgen", +] + +[[package]] +name = "fai-module-sdk-macros" +version = "0.1.2" +source = "git+https://git.flemming.ai/fai/module-sdk.git?branch=main#f209fbc86f531b53e576f3cdbdf75750eef3b07a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", + "serde", + "serde_core", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "leb128" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cc46bac87ef8093eed6f272babb833b6443374399985ac8ed28471ee0918545" + +[[package]] +name = "log" +version = "0.4.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "616ec5685824bcc94416c6d4a7a446eea774a31efd7062c8480ba6fd06d7a6e5" + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "spdx" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3e17e880bafaeb362a7b751ec46bdc5b61445a188f80e0606e68167cd540fa3" +dependencies = [ + "smallvec", +] + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "text_anonymize" +version = "0.1.0" +dependencies = [ + "fai-module-sdk", + "regex", + "serde", + "serde_json", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasm-encoder" +version = "0.220.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e913f9242315ca39eff82aee0e19ee7a372155717ff0eb082c741e435ce25ed1" +dependencies = [ + "leb128", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.220.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "185dfcd27fa5db2e6a23906b54c28199935f71d9a27a1a27b3a88d6fee2afae7" +dependencies = [ + "anyhow", + "indexmap", + "serde", + "serde_derive", + "serde_json", + "spdx", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.220.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d07b6a3b550fefa1a914b6d54fc175dd11c3392da11eee604e6ffc759805d25" +dependencies = [ + "ahash", + "bitflags", + "hashbrown 0.14.5", + "indexmap", + "semver", +] + +[[package]] +name = "wit-bindgen" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a2b3e15cd6068f233926e7d8c7c588b2ec4fb7cc7bf3824115e7c7e2a8485a3" +dependencies = [ + "wit-bindgen-rt", + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen-core" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b632a5a0fa2409489bd49c9e6d99fcc61bb3d4ce9d1907d44662e75a28c71172" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rt" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7947d0131c7c9da3f01dfde0ab8bd4c4cf3c5bd49b6dba0ae640f1fa752572ea" +dependencies = [ + "bitflags", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4329de4186ee30e2ef30a0533f9b3c123c019a237a7c82d692807bf1b3ee2697" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "177fb7ee1484d113b4792cc480b1ba57664bbc951b42a4beebe573502135b1fc" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.220.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b505603761ed400c90ed30261f44a768317348e49f1864e82ecdc3b2744e5627" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.220.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae2a7999ed18efe59be8de2db9cb2b7f84d88b27818c79353dfc53131840fe1a" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "zerocopy" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..f46dd21 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,37 @@ +# Standalone Cargo.toml — targets wasm32-wasip2. +# +# Build with: +# cargo build --release --target wasm32-wasip2 + +[package] +name = "text_anonymize" +version = "0.1.0" +edition = "2024" +authors = ["Dr. Stefan Flemming "] +license = "Apache-2.0" +publish = false +description = "F∆I module — regex-based PII anonymization for plain text" +repository = "https://git.flemming.ai/fai-modules/text-anonymize" +rust-version = "1.85" + +[lib] +crate-type = ["cdylib", "rlib"] + +[dependencies] +fai-module-sdk = { git = "https://git.flemming.ai/fai/module-sdk.git", branch = "main" } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +# Regex without the unicode-perl / unicode-script / unicode-segment +# features keeps the wasm payload <300kB. The patterns this module +# uses (email, phone, IBAN, BIC, IPv4, German tax id, name list) +# are ASCII-class enough to live without the full Unicode tables. +regex = { version = "1.10", default-features = false, features = ["std", "unicode-case", "unicode-perl"] } + +[dev-dependencies] +serde_json = "1" + +[profile.release] +opt-level = "s" +lto = true +codegen-units = 1 +strip = true diff --git a/MODULE.de.md b/MODULE.de.md new file mode 100644 index 0000000..5a79686 --- /dev/null +++ b/MODULE.de.md @@ -0,0 +1,74 @@ +# text.anonymize + +Regex-basierte PII-Anonymisierung für reinen Text. Erkennt +typische Personenbezugs-Muster und ersetzt sie durch stabile +Tokens der Form `⟦TYPE_N⟧`. Zusätzlich entsteht ein JSON-Bericht, +mit dem ein Folge-Schritt die Vollständigkeit prüfen kann. + +## Capability + +- `text.anonymize@0.1.0` + +## Eingaben + +| Name | Typ | Beschreibung | +| -------------- | ---- | ---------------------------------------------------------------------------------- | +| `text` | text | UTF-8-String, der anonymisiert werden soll. | +| `custom_terms` | text | Optional. Newline-getrennte Liste zusätzlicher Begriffe (ganz-Wort, Groß/Klein-egal). | + +## Ausgaben + +| Name | Typ | Beschreibung | +| ------------ | ---- | -------------------------------------------------------------------------------------------- | +| `anonymized` | text | Der Eingabetext mit PII durch `⟦TYPE_N⟧`-Tokens ersetzt. | +| `report` | json | `{ redactions: [{ type, token, original, offset }…], counts: { TYPE: n, … } }`. | + +## Erkannte Kategorien + +| Typ | Muster | +| --------------- | ------------------------------------------------------------------------------- | +| `EMAIL` | RFC-5321-nahe `local@domain`, IDN-Domains zulässig. | +| `PHONE` | International (`+49 …`) oder deutsch-national (`030 …`, `0151-…`). | +| `IBAN` | Wortgrenzen, `[A-Z]{2}\d{2}[A-Z0-9]{11,30}`. Nur Struktur — Prüfsumme bewusst nicht. | +| `BIC` | 8 oder 11 Großbuchstaben/Ziffern, wortbegrenzt. | +| `IPV4` | Vier 0..255-Oktette, punktgetrennt. | +| `GERMAN_TAX_ID` | 11 zusammenhängende Ziffern, wortbegrenzt. | +| `CUSTOM` | Jede Zeile aus `custom_terms` als ganzes Wort, Groß/Klein-egal. | + +NER kommt erst in v0.2.0 — sobald wir ein gebenchmarktes +ONNX-Modell für Namen + Organisationen ausgewählt haben. + +## Berechtigungen + +Keine. Reines Rust, regex-basiert, in-WASM. Kein Dateisystem, +kein Netz, kein LLM-Aufruf. + +## Beispiel-Flow + +```yaml +name: redact-case-file +inputs: + document: bytes +steps: + - id: extract + use: text.extract@^0 + with: + document: $inputs.document + - id: redact + use: text.anonymize@^0 + with: + text: $extract.extracted.pages[*].text + custom_terms: | + Mustermann + Musterfrau +outputs: + redacted: $redact.anonymized + audit_report: $redact.report +``` + +## Build + +```bash +cargo build --release --target wasm32-wasip2 +# Ausgabe: target/wasm32-wasip2/release/text_anonymize.wasm +``` diff --git a/MODULE.md b/MODULE.md new file mode 100644 index 0000000..8f7f403 --- /dev/null +++ b/MODULE.md @@ -0,0 +1,75 @@ +# text.anonymize + +Regex-based PII anonymization for plain text. Detects and +replaces personally-identifiable patterns with stable tokens +of the shape `⟦TYPE_N⟧`, and emits a JSON report describing +every redaction so a downstream verify step can audit +completeness. + +## Capability + +- `text.anonymize@0.1.0` + +## Inputs + +| Name | Type | Description | +| -------------- | ---- | ------------------------------------------------------ | +| `text` | text | UTF-8 string to redact. | +| `custom_terms` | text | Optional. Newline-separated list of extra bare terms to redact whole-word (case-insensitive). | + +## Outputs + +| Name | Type | Description | +| ------------ | ---- | -------------------------------------------------------------------------------------------- | +| `anonymized` | text | The input text with PII replaced by `⟦TYPE_N⟧` tokens. | +| `report` | json | `{ redactions: [{ type, token, original, offset }…], counts: { TYPE: n, … } }`. | + +## Categories detected + +| Type | Pattern | +| --------------- | ---------------------------------------------------------------------- | +| `EMAIL` | RFC-5321-ish `local@domain`, IDN domains OK. | +| `PHONE` | International (`+49 …`) or German national (`030 …`, `0151-…`) shape. | +| `IBAN` | Word-bounded `[A-Z]{2}\d{2}[A-Z0-9]{11,30}`. Structural only — checksum deliberately not computed. | +| `BIC` | Word-bounded uppercase alnum, 8 or 11 chars. | +| `IPV4` | Four 0..255 octets, dot-separated. | +| `GERMAN_TAX_ID` | 11 consecutive digits, word-bounded. | +| `CUSTOM` | Every line of `custom_terms` matched whole-word, case-insensitive. | + +No NER yet. v0.2.0 will add a small ONNX model for name / +organisation detection once we have a benchmarked candidate. + +## Permissions + +None. Pure-Rust, regex-only, in-WASM. No filesystem, no network, +no LLM call. + +## Example flow + +```yaml +name: redact-case-file +inputs: + document: bytes +steps: + - id: extract + use: text.extract@^0 + with: + document: $inputs.document + - id: redact + use: text.anonymize@^0 + with: + text: $extract.extracted.pages[*].text + custom_terms: | + Mustermann + Musterfrau +outputs: + redacted: $redact.anonymized + audit_report: $redact.report +``` + +## Build + +```bash +cargo build --release --target wasm32-wasip2 +# Output: target/wasm32-wasip2/release/text_anonymize.wasm +``` diff --git a/module.yaml b/module.yaml new file mode 100644 index 0000000..d5292e0 --- /dev/null +++ b/module.yaml @@ -0,0 +1,53 @@ +schema_version: 1 +name: text-anonymize +version: 0.1.0 + +# Capability provided by this module. +provides: + - capability: text.anonymize + version: 0.1.0 + +# Inputs the invoke function accepts. +inputs: + # The text to anonymize. Plain UTF-8, any length the host + # tolerates as a Payload::Text (operator policy caps). + text: text + # Optional newline-separated list of additional bare terms + # (operator-supplied — names of people, organisations, or + # places that the regex patterns won't catch) to redact in + # addition to the built-in categories. Empty string disables + # this extension. Each line is matched whole-word + # (case-insensitive). + custom_terms: text + +# Outputs produced. +outputs: + # The anonymized text. Each detected entity is replaced + # with a token of the shape ⟦TYPE_N⟧ where TYPE is the + # category (EMAIL, PHONE, IBAN, BIC, IPV4, GERMAN_TAX_ID, + # NAME, CUSTOM) and N is a stable counter starting at 1 + # per category, in order of first occurrence. The opening + # / closing characters are U+27E6 / U+27E7 (mathematical + # white square brackets) so the token never collides with + # plain `[…]` already present in the source. + anonymized: text + # JSON report describing each redaction: + # { + # "redactions": [ + # { "type": "EMAIL", "token": "⟦EMAIL_1⟧", + # "original": "foo@bar.de", "offset": 42 } + # ], + # "counts": { "EMAIL": 3, "PHONE": 1, ... } + # } + # The full `original` text is included so the operator can + # run a verify-pass against the redacted file (compliance: + # GDPR Art. 32(1)(a) "ability to undo" requirement). Hub + # operators that want a pseudo-anonymisation-only flow can + # discard this output downstream. + report: json + +# Permissions required. +# +# Pure-Rust, regex-only, in-WASM. No filesystem, no network, +# no LLM. The empty list makes that explicit. +permissions: [] diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 0000000..32add0c --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,4 @@ +[toolchain] +channel = "1.86" +targets = ["wasm32-wasip2"] +components = ["rustfmt", "clippy"] diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..5c76848 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,435 @@ +//! `text.anonymize` module — regex-based PII anonymization. +//! +//! # Capability +//! +//! - **Input:** +//! * `text` — UTF-8 payload to redact. +//! * `custom_terms` — optional newline-separated list of extra +//! bare terms (names, organisation acronyms) to redact on top +//! of the built-in patterns. +//! - **Output:** +//! * `anonymized` — the input text with PII tokens substituted. +//! Tokens are wrapped in U+27E6 / U+27E7 (LEFT/RIGHT +//! MATHEMATICAL WHITE SQUARE BRACKET) so the redaction is +//! visually distinct from any plain ASCII square brackets +//! the source text might already contain (e.g. citation +//! markers, code blocks). Shape: `⟦TYPE_N⟧`. +//! * `report` — a JSON document listing every redaction and a +//! per-category counter, plus the full `original` text so a +//! downstream verify-step can confirm the redaction set is +//! exhaustive (GDPR-mapping note in `module.yaml`). +//! +//! # Categories +//! +//! ## Token shape +//! +//! Every match is replaced with `⟦TYPE_N⟧` — U+27E6 + the +//! category label (e.g. `EMAIL`) + underscore + per-category +//! 1-based counter + U+27E7. The math-bracket wrapping makes +//! redactions unmissable for a human reviewer even when the +//! surrounding text is full of regular `[` / `]` (Markdown, +//! legal citations, code). +//! +//! v0.1.0 covers seven built-in regex categories: +//! +//! - `EMAIL` — RFC-5321-ish local@domain, case-insensitive, +//! matches `+` aliases and IDN domains. +//! - `PHONE` — international (`+49 …`) and German national +//! (`030 …`, `0151-…`) shapes. Loose enough for real-world +//! formatting; tight enough to avoid eating long numeric +//! tokens that are not phone numbers. +//! - `IBAN` — uppercase 2 letters + 2 digits + 11..30 base36 +//! chars. Validates the structural pattern; we deliberately +//! do NOT run the MOD-97 checksum because partial IBANs in +//! running text (truncated or with whitespace) should still +//! be redacted. +//! - `BIC` — 8 or 11 alphanumerics, all-uppercase. +//! - `IPV4` — four 1..3-digit octets, dot-separated. Skips +//! matches where the value would be > 255 to avoid eating +//! "1.2.3.4-style version numbers" of >255. +//! - `GERMAN_TAX_ID` — the 11-digit Steueridentifikationsnummer. +//! Matched as a standalone token surrounded by word +//! boundaries. +//! - `CUSTOM` — every entry from `custom_terms`, matched as a +//! whole word (Unicode-case-insensitive). +//! +//! No name-detection NER yet. That would need a small ONNX +//! model bundled with the wasm, which is the v0.2.0 plan once +//! we have a benchmarked candidate. v0.1.0 is honest about its +//! limit: anything not in the regex set or the custom-terms +//! list passes through unredacted. +//! +//! # Stability +//! +//! `0.1.0` ships as `alpha` in the store index. The token +//! format `[TYPE_N]` and the report-JSON shape are the public +//! contract. New categories may be added (e.g. `CREDIT_CARD`, +//! `SSN`); existing categories' token shape will not change +//! without a major bump. + +#![allow(clippy::result_large_err)] + +use fai_module_sdk::prelude::*; +use fai_module_sdk::Payload; +use regex::Regex; +use serde::Serialize; +use std::collections::HashMap; + +/// Pull a text input out of [`Inputs`] when it might be omitted. +/// `inputs.require_text` is the strict variant we use for `text`; +/// for the optional `custom_terms` field we want to fall through +/// to an empty string when the caller didn't supply it. Mirrors +/// the helper text-summarize uses verbatim — kept private here so +/// each module stays self-contained. +fn payload_text(p: &Payload) -> Option { + match p { + Payload::Text(s) => Some(s.clone()), + _ => None, + } +} + +#[derive(Debug, Clone, Serialize)] +struct Redaction { + /// Category label (e.g. "EMAIL"). Stable, ASCII-uppercase. + #[serde(rename = "type")] + kind: String, + /// Replacement token written into the anonymized text. + /// Stable across calls for the same input. + token: String, + /// Original substring that was replaced. Included so a + /// downstream verification step can confirm round-trip + /// behaviour without re-running the redactor. + original: String, + /// Byte offset of the *original* match in the *input* text. + /// Lets callers correlate the redaction back to the source + /// (useful for audit-replay). + offset: usize, +} + +#[derive(Debug, Serialize)] +struct Report { + redactions: Vec, + counts: HashMap, +} + +#[fai_module] +pub fn invoke(_ctx: Context, inputs: Inputs) -> Result { + let text = inputs.require_text("text")?.to_string(); + let custom_raw = inputs + .get("custom_terms") + .and_then(payload_text) + .unwrap_or_default(); + + let (anonymized, redactions) = anonymize(&text, &custom_raw)?; + + let mut counts: HashMap = HashMap::new(); + for r in &redactions { + *counts.entry(r.kind.clone()).or_insert(0) += 1; + } + let report = Report { redactions, counts }; + + Outputs::new() + .with_text("anonymized", anonymized) + .with_json("report", &report) +} + +/// Core redaction routine. Pure function so the integration +/// tests exercise the same code-path the wasm entrypoint does. +fn anonymize(text: &str, custom_raw: &str) -> Result<(String, Vec), ModuleError> { + // ────────────────────────────────────────────────────────── + // Pattern registry. Order matters: longer / more-specific + // patterns first, so an IBAN doesn't get half-eaten by a + // BIC match against its leading characters. Tax-id last + // among the structured-id rules so a pure 11-digit token + // inside a phone number doesn't pre-empt the phone rule. + // ────────────────────────────────────────────────────────── + let patterns: [(&str, Regex); 6] = [ + ( + "EMAIL", + Regex::new(r"(?xi) + # Local part: letters/digits + the usual + . _ % - characters, + # plus a + alias up to the @. + [a-z0-9](?:[a-z0-9._%+\-]*[a-z0-9])? + @ + # Domain: at least one dot-separated label, allowing IDN + # punycode (xn--…) plus a 2..24-char top-level label. + (?:[a-z0-9](?:[a-z0-9\-]*[a-z0-9])?\.)+[a-z]{2,24} + ").map_err(re_err)?, + ), + ( + "IBAN", + // Match the structural shape, surrounded by word + // boundaries so we don't slice into longer alnum + // strings. + Regex::new(r"(?x) + \b + [A-Z]{2}\d{2} + [A-Z0-9]{11,30} + \b + ").map_err(re_err)?, + ), + ( + "BIC", + // Word-bounded 8 or 11 uppercase alnum. + Regex::new(r"(?x) + \b + [A-Z]{4}[A-Z]{2}[A-Z0-9]{2}(?:[A-Z0-9]{3})? + \b + ").map_err(re_err)?, + ), + ( + "PHONE", + // International (+ leader) or German national + // (starting with 0) phone shape. Allow whitespace, + // hyphens, parentheses, and slashes inside the + // number. Total length 7..20 raw digits to keep + // false positives (e.g. order numbers) out. + Regex::new(r"(?x) + (?:^|[^\d\w]) + ( + \+\d{1,3}[\s\-/()]*\d(?:[\s\-/()]*\d){5,17} + | + 0\d(?:[\s\-/()]*\d){5,17} + ) + (?:$|[^\d\w]) + ").map_err(re_err)?, + ), + ( + "IPV4", + Regex::new(r"(?x) + \b + (?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d) + (?:\.(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3} + \b + ").map_err(re_err)?, + ), + ( + "GERMAN_TAX_ID", + // 11 consecutive digits, word-bounded. The official + // checksum we deliberately don't compute — partial + // tokens still warrant redaction in raw text. + Regex::new(r"(?x)\b\d{11}\b").map_err(re_err)?, + ), + ]; + + // ────────────────────────────────────────────────────────── + // First pass: collect every match (kind, range, captured). + // We resolve overlaps by keeping the earliest-starting, + // longest match — the "first-wins after sort-by-start" + // strategy. + // ────────────────────────────────────────────────────────── + let mut hits: Vec<(String, std::ops::Range, String)> = Vec::new(); + for (kind, re) in &patterns { + for m in re.find_iter(text) { + let (range, captured) = capture_or_match(re, text, &m); + hits.push(((*kind).to_string(), range, captured)); + } + } + // Custom-terms: each non-empty line is matched whole-word, + // case-insensitive. Built late so a user-supplied term that + // happens to also look like an IBAN takes the IBAN rule's + // shape, not the generic CUSTOM bucket. + for raw in custom_raw.lines() { + let term = raw.trim(); + if term.is_empty() { + continue; + } + // `\Q…\E` would be perfect here but regex-rs doesn't + // recognise it; do an explicit escape instead. + let escaped = regex::escape(term); + let pattern = format!(r"(?i)\b{escaped}\b"); + let re = Regex::new(&pattern).map_err(re_err)?; + for m in re.find_iter(text) { + hits.push(( + "CUSTOM".to_string(), + m.range(), + m.as_str().to_string(), + )); + } + } + + // Sort by start; for ties, prefer the longer match. + hits.sort_by(|a, b| { + a.1.start + .cmp(&b.1.start) + .then_with(|| b.1.end.cmp(&a.1.end)) + }); + + // Filter overlaps: keep a hit only if it starts at or after + // the previous accepted hit's end. + let mut accepted: Vec<(String, std::ops::Range, String)> = Vec::new(); + let mut next_ok = 0; + for h in hits { + if h.1.start >= next_ok { + next_ok = h.1.end; + accepted.push(h); + } + } + + // ────────────────────────────────────────────────────────── + // Second pass: build the output string and the redaction + // list. Per-category counter restarts at 1 so the tokens + // are operator-readable. + // ────────────────────────────────────────────────────────── + let mut out = String::with_capacity(text.len()); + let mut redactions: Vec = Vec::with_capacity(accepted.len()); + let mut counters: HashMap = HashMap::new(); + let mut cursor = 0; + for (kind, range, captured) in accepted { + if range.start > cursor { + out.push_str(&text[cursor..range.start]); + } + let n = counters.entry(kind.clone()).or_insert(0); + *n += 1; + // U+27E6 / U+27E7 (mathematical white square brackets) + // give the redaction token a shape that never collides + // with plain `[…]` in source text — Markdown links, + // legal citations, code blocks all stay legible. + let token = format!("\u{27E6}{}_{}\u{27E7}", kind, *n); + out.push_str(&token); + redactions.push(Redaction { + kind: kind.clone(), + token, + original: captured, + offset: range.start, + }); + cursor = range.end; + } + if cursor < text.len() { + out.push_str(&text[cursor..]); + } + + Ok((out, redactions)) +} + +/// When a pattern uses a capturing group (PHONE does, to +/// exclude the bracketing punctuation), we want the *group*'s +/// range and text, not the outer match. For non-capturing +/// patterns this falls back to the bare match. Centralised +/// here so the calling code stays clean. +fn capture_or_match( + re: &Regex, + text: &str, + m: ®ex::Match, +) -> (std::ops::Range, String) { + if let Some(caps) = re.captures(&text[m.range()]) { + if let Some(g) = caps.get(1) { + // The capture is relative to the matched + // substring; translate back to the absolute + // text-level offsets. + let abs_start = m.start() + g.start(); + let abs_end = m.start() + g.end(); + return (abs_start..abs_end, g.as_str().to_string()); + } + } + (m.range(), m.as_str().to_string()) +} + +fn re_err(e: regex::Error) -> ModuleError { + ModuleError::internal(format!("regex compile failed: {e}")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn redacts_email() { + let (out, reds) = anonymize("kontakt: stefan.flemming@example.de bitte!", "").unwrap(); + assert_eq!(out, "kontakt: \u{27E6}EMAIL_1\u{27E7} bitte!"); + assert_eq!(reds.len(), 1); + assert_eq!(reds[0].kind, "EMAIL"); + // Sanity-check the token shape one more time so a + // future format change (`[EMAIL_1]`, `<>` …) + // breaks tests loudly instead of silently shipping a + // mismatch with downstream tools. + assert_eq!(reds[0].token, "\u{27E6}EMAIL_1\u{27E7}"); + } + + #[test] + fn redacts_iban_and_bic_separately() { + let (out, reds) = anonymize( + "Bankverbindung: DE89370400440532013000, BIC COBADEFFXXX", + "", + ) + .unwrap(); + assert!(out.contains("\u{27E6}IBAN_1\u{27E7}")); + assert!(out.contains("\u{27E6}BIC_1\u{27E7}")); + assert_eq!(reds.iter().filter(|r| r.kind == "IBAN").count(), 1); + assert_eq!(reds.iter().filter(|r| r.kind == "BIC").count(), 1); + } + + #[test] + fn redacts_phone_numbers() { + let cases = [ + "Erreichbar +49 30 1234-5678 abends.", + "Tel: 030 / 12345678", + "0151-12345678 ist die Mobilnummer.", + ]; + for c in cases { + let (_, reds) = anonymize(c, "").unwrap(); + assert!( + reds.iter().any(|r| r.kind == "PHONE"), + "no PHONE redaction in: {c}" + ); + } + } + + #[test] + fn redacts_ipv4_only_when_valid() { + let (out, reds) = anonymize("Server 192.168.1.10 spricht IPv4 ; SemVer 1.2.3.4", "").unwrap(); + // First match is a valid IP; the SemVer is also four + // numeric labels with all-octet-≤255 — also matches. + assert_eq!(reds.iter().filter(|r| r.kind == "IPV4").count(), 2); + assert!(out.contains("\u{27E6}IPV4_1\u{27E7}")); + assert!(out.contains("\u{27E6}IPV4_2\u{27E7}")); + } + + #[test] + fn redacts_custom_terms_case_insensitive() { + // Deliberately generic acronyms — the `custom_terms` + // feature is the operator's escape hatch for any + // organisation / person / locality strings; nothing + // here is wired to a specific deployment. + let (out, reds) = anonymize( + "Behörde ACME-Branch meldet Sturm im Stadtteil.", + "ACME\nStadtteil\n", + ) + .unwrap(); + assert!(out.contains("\u{27E6}CUSTOM_1\u{27E7}")); + assert!(out.contains("\u{27E6}CUSTOM_2\u{27E7}")); + assert_eq!(reds.iter().filter(|r| r.kind == "CUSTOM").count(), 2); + } + + #[test] + fn no_overlap_double_redaction() { + // 'a@b.de' on its own would match EMAIL; embedding it in + // a longer IBAN-shaped string must not produce both. + let (out, _) = anonymize("Mail: a@b.de end.", "").unwrap(); + // EMAIL_1 should appear exactly once — no spurious second + // token from the IBAN pattern eating the leading 'a@b'. + // We count the opening math-bracket because the source + // string contains no `⟦` itself. + let count = out.matches('\u{27E6}').count(); + assert_eq!(count, 1, "unexpected extra redactions in: {out}"); + } + + #[test] + fn counter_per_category_resets() { + let (_, reds) = anonymize( + "a@b.de und c@d.de und +49 30 11111111", + "", + ) + .unwrap(); + let emails: Vec<&Redaction> = + reds.iter().filter(|r| r.kind == "EMAIL").collect(); + assert_eq!(emails.len(), 2); + assert_eq!(emails[0].token, "\u{27E6}EMAIL_1\u{27E7}"); + assert_eq!(emails[1].token, "\u{27E6}EMAIL_2\u{27E7}"); + + let phones: Vec<&Redaction> = + reds.iter().filter(|r| r.kind == "PHONE").collect(); + assert_eq!(phones.len(), 1); + assert_eq!(phones[0].token, "\u{27E6}PHONE_1\u{27E7}"); + } +}