feat: text.anonymize v0.1.0 — regex-based PII redaction
First release. Pure-Rust, in-WASM, regex-only, declares no
permissions. Suitable as a first redaction pass after
text.extract before any cloud-LLM step.
Detection categories:
EMAIL RFC-5321-ish local@domain, IDN-aware.
PHONE International (+CC …) and DE national
(030 …, 0151-…) shapes, 7..20 raw digits.
IBAN Word-bounded [A-Z]{2}\d{2}[A-Z0-9]{11,30}.
Structural only — MOD-97 checksum
deliberately skipped so partial / truncated
tokens in running text still get redacted.
BIC 8 or 11 uppercase alnum.
IPV4 Four 0..255 octets, dot-separated.
GERMAN_TAX_ID 11 consecutive digits, word-bounded.
CUSTOM Operator-supplied bare terms from the
newline-separated `custom_terms` input,
matched whole-word case-insensitive.
Token shape: ⟦TYPE_N⟧ — U+27E6 / U+27E7 mathematical white
square brackets. Distinct from any plain ASCII `[…]` already
present in source text (Markdown links, legal citations,
code blocks) so a reviewer never has to guess which `[…]`
is a redaction.
Outputs:
anonymized text Input with PII replaced by ⟦TYPE_N⟧.
Counter restarts at 1 per type so the
tokens stay operator-readable.
report json { redactions: [{type, token, original,
offset}…], counts: { TYPE: n, … } }.
Full original-text reconstruction is
possible from this — the GDPR
Art. 32(1)(a) "ability to undo"
requirement.
Quality bar (7 unit tests):
* email round-trip
* IBAN + BIC don't eat each other
* three phone-number shapes redact
* IPv4 only matches valid 0..255 octets
* custom_terms case-insensitive
* no double-redaction on overlapping patterns
* per-category counter resets correctly
Built artefact: target/wasm32-wasip2/release/text_anonymize.wasm
(~180 KiB stripped).
NER for free-text names / organisations / locations is the
v0.2.0 plan once a benchmarked ONNX model is selected; the
operator's `custom_terms` field is the v0.1.0 escape hatch.
Signed-off-by: flemming-it <sf@flemming.it>
This commit is contained in:
commit
734d8f6e6f
8 changed files with 1164 additions and 0 deletions
4
.gitignore
vendored
Normal file
4
.gitignore
vendored
Normal file
|
|
@ -0,0 +1,4 @@
|
||||||
|
target/
|
||||||
|
**/*.rs.bk
|
||||||
|
.DS_Store
|
||||||
|
module.wasm
|
||||||
482
Cargo.lock
generated
Normal file
482
Cargo.lock
generated
Normal file
|
|
@ -0,0 +1,482 @@
|
||||||
|
# This file is automatically @generated by Cargo.
|
||||||
|
# It is not intended for manual editing.
|
||||||
|
version = 4
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "ahash"
|
||||||
|
version = "0.8.12"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75"
|
||||||
|
dependencies = [
|
||||||
|
"cfg-if",
|
||||||
|
"once_cell",
|
||||||
|
"version_check",
|
||||||
|
"zerocopy",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "aho-corasick"
|
||||||
|
version = "1.1.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301"
|
||||||
|
dependencies = [
|
||||||
|
"memchr",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "anyhow"
|
||||||
|
version = "1.0.102"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "bitflags"
|
||||||
|
version = "2.11.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "cfg-if"
|
||||||
|
version = "1.0.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "equivalent"
|
||||||
|
version = "1.0.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "fai-module-sdk"
|
||||||
|
version = "0.1.2"
|
||||||
|
source = "git+https://git.flemming.ai/fai/module-sdk.git?branch=main#f209fbc86f531b53e576f3cdbdf75750eef3b07a"
|
||||||
|
dependencies = [
|
||||||
|
"fai-module-sdk-macros",
|
||||||
|
"serde",
|
||||||
|
"serde_json",
|
||||||
|
"thiserror",
|
||||||
|
"wit-bindgen",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "fai-module-sdk-macros"
|
||||||
|
version = "0.1.2"
|
||||||
|
source = "git+https://git.flemming.ai/fai/module-sdk.git?branch=main#f209fbc86f531b53e576f3cdbdf75750eef3b07a"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "hashbrown"
|
||||||
|
version = "0.14.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
|
||||||
|
dependencies = [
|
||||||
|
"ahash",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "hashbrown"
|
||||||
|
version = "0.17.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "heck"
|
||||||
|
version = "0.5.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "id-arena"
|
||||||
|
version = "2.3.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "indexmap"
|
||||||
|
version = "2.14.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9"
|
||||||
|
dependencies = [
|
||||||
|
"equivalent",
|
||||||
|
"hashbrown 0.17.1",
|
||||||
|
"serde",
|
||||||
|
"serde_core",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "itoa"
|
||||||
|
version = "1.0.18"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "leb128"
|
||||||
|
version = "0.2.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "6cc46bac87ef8093eed6f272babb833b6443374399985ac8ed28471ee0918545"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "log"
|
||||||
|
version = "0.4.30"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "616ec5685824bcc94416c6d4a7a446eea774a31efd7062c8480ba6fd06d7a6e5"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "memchr"
|
||||||
|
version = "2.8.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "once_cell"
|
||||||
|
version = "1.21.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "prettyplease"
|
||||||
|
version = "0.2.37"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"syn",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "proc-macro2"
|
||||||
|
version = "1.0.106"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
|
||||||
|
dependencies = [
|
||||||
|
"unicode-ident",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "quote"
|
||||||
|
version = "1.0.45"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "regex"
|
||||||
|
version = "1.12.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276"
|
||||||
|
dependencies = [
|
||||||
|
"aho-corasick",
|
||||||
|
"memchr",
|
||||||
|
"regex-automata",
|
||||||
|
"regex-syntax",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "regex-automata"
|
||||||
|
version = "0.4.14"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f"
|
||||||
|
dependencies = [
|
||||||
|
"aho-corasick",
|
||||||
|
"memchr",
|
||||||
|
"regex-syntax",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "regex-syntax"
|
||||||
|
version = "0.8.10"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "semver"
|
||||||
|
version = "1.0.28"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "serde"
|
||||||
|
version = "1.0.228"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
|
||||||
|
dependencies = [
|
||||||
|
"serde_core",
|
||||||
|
"serde_derive",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "serde_core"
|
||||||
|
version = "1.0.228"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
|
||||||
|
dependencies = [
|
||||||
|
"serde_derive",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "serde_derive"
|
||||||
|
version = "1.0.228"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "serde_json"
|
||||||
|
version = "1.0.150"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9"
|
||||||
|
dependencies = [
|
||||||
|
"itoa",
|
||||||
|
"memchr",
|
||||||
|
"serde",
|
||||||
|
"serde_core",
|
||||||
|
"zmij",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "smallvec"
|
||||||
|
version = "1.15.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "spdx"
|
||||||
|
version = "0.10.9"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "c3e17e880bafaeb362a7b751ec46bdc5b61445a188f80e0606e68167cd540fa3"
|
||||||
|
dependencies = [
|
||||||
|
"smallvec",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "syn"
|
||||||
|
version = "2.0.117"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"unicode-ident",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "text_anonymize"
|
||||||
|
version = "0.1.0"
|
||||||
|
dependencies = [
|
||||||
|
"fai-module-sdk",
|
||||||
|
"regex",
|
||||||
|
"serde",
|
||||||
|
"serde_json",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "thiserror"
|
||||||
|
version = "2.0.18"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4"
|
||||||
|
dependencies = [
|
||||||
|
"thiserror-impl",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "thiserror-impl"
|
||||||
|
version = "2.0.18"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "unicode-ident"
|
||||||
|
version = "1.0.24"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "unicode-xid"
|
||||||
|
version = "0.2.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "version_check"
|
||||||
|
version = "0.9.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "wasm-encoder"
|
||||||
|
version = "0.220.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e913f9242315ca39eff82aee0e19ee7a372155717ff0eb082c741e435ce25ed1"
|
||||||
|
dependencies = [
|
||||||
|
"leb128",
|
||||||
|
"wasmparser",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "wasm-metadata"
|
||||||
|
version = "0.220.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "185dfcd27fa5db2e6a23906b54c28199935f71d9a27a1a27b3a88d6fee2afae7"
|
||||||
|
dependencies = [
|
||||||
|
"anyhow",
|
||||||
|
"indexmap",
|
||||||
|
"serde",
|
||||||
|
"serde_derive",
|
||||||
|
"serde_json",
|
||||||
|
"spdx",
|
||||||
|
"wasm-encoder",
|
||||||
|
"wasmparser",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "wasmparser"
|
||||||
|
version = "0.220.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "8d07b6a3b550fefa1a914b6d54fc175dd11c3392da11eee604e6ffc759805d25"
|
||||||
|
dependencies = [
|
||||||
|
"ahash",
|
||||||
|
"bitflags",
|
||||||
|
"hashbrown 0.14.5",
|
||||||
|
"indexmap",
|
||||||
|
"semver",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "wit-bindgen"
|
||||||
|
version = "0.36.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "6a2b3e15cd6068f233926e7d8c7c588b2ec4fb7cc7bf3824115e7c7e2a8485a3"
|
||||||
|
dependencies = [
|
||||||
|
"wit-bindgen-rt",
|
||||||
|
"wit-bindgen-rust-macro",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "wit-bindgen-core"
|
||||||
|
version = "0.36.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "b632a5a0fa2409489bd49c9e6d99fcc61bb3d4ce9d1907d44662e75a28c71172"
|
||||||
|
dependencies = [
|
||||||
|
"anyhow",
|
||||||
|
"heck",
|
||||||
|
"wit-parser",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "wit-bindgen-rt"
|
||||||
|
version = "0.36.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "7947d0131c7c9da3f01dfde0ab8bd4c4cf3c5bd49b6dba0ae640f1fa752572ea"
|
||||||
|
dependencies = [
|
||||||
|
"bitflags",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "wit-bindgen-rust"
|
||||||
|
version = "0.36.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "4329de4186ee30e2ef30a0533f9b3c123c019a237a7c82d692807bf1b3ee2697"
|
||||||
|
dependencies = [
|
||||||
|
"anyhow",
|
||||||
|
"heck",
|
||||||
|
"indexmap",
|
||||||
|
"prettyplease",
|
||||||
|
"syn",
|
||||||
|
"wasm-metadata",
|
||||||
|
"wit-bindgen-core",
|
||||||
|
"wit-component",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "wit-bindgen-rust-macro"
|
||||||
|
version = "0.36.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "177fb7ee1484d113b4792cc480b1ba57664bbc951b42a4beebe573502135b1fc"
|
||||||
|
dependencies = [
|
||||||
|
"anyhow",
|
||||||
|
"prettyplease",
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn",
|
||||||
|
"wit-bindgen-core",
|
||||||
|
"wit-bindgen-rust",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "wit-component"
|
||||||
|
version = "0.220.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "b505603761ed400c90ed30261f44a768317348e49f1864e82ecdc3b2744e5627"
|
||||||
|
dependencies = [
|
||||||
|
"anyhow",
|
||||||
|
"bitflags",
|
||||||
|
"indexmap",
|
||||||
|
"log",
|
||||||
|
"serde",
|
||||||
|
"serde_derive",
|
||||||
|
"serde_json",
|
||||||
|
"wasm-encoder",
|
||||||
|
"wasm-metadata",
|
||||||
|
"wasmparser",
|
||||||
|
"wit-parser",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "wit-parser"
|
||||||
|
version = "0.220.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "ae2a7999ed18efe59be8de2db9cb2b7f84d88b27818c79353dfc53131840fe1a"
|
||||||
|
dependencies = [
|
||||||
|
"anyhow",
|
||||||
|
"id-arena",
|
||||||
|
"indexmap",
|
||||||
|
"log",
|
||||||
|
"semver",
|
||||||
|
"serde",
|
||||||
|
"serde_derive",
|
||||||
|
"serde_json",
|
||||||
|
"unicode-xid",
|
||||||
|
"wasmparser",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "zerocopy"
|
||||||
|
version = "0.8.48"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9"
|
||||||
|
dependencies = [
|
||||||
|
"zerocopy-derive",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "zerocopy-derive"
|
||||||
|
version = "0.8.48"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "zmij"
|
||||||
|
version = "1.0.21"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa"
|
||||||
37
Cargo.toml
Normal file
37
Cargo.toml
Normal file
|
|
@ -0,0 +1,37 @@
|
||||||
|
# Standalone Cargo.toml — targets wasm32-wasip2.
|
||||||
|
#
|
||||||
|
# Build with:
|
||||||
|
# cargo build --release --target wasm32-wasip2
|
||||||
|
|
||||||
|
[package]
|
||||||
|
name = "text_anonymize"
|
||||||
|
version = "0.1.0"
|
||||||
|
edition = "2024"
|
||||||
|
authors = ["Dr. Stefan Flemming <platform@flemming.ai>"]
|
||||||
|
license = "Apache-2.0"
|
||||||
|
publish = false
|
||||||
|
description = "F∆I module — regex-based PII anonymization for plain text"
|
||||||
|
repository = "https://git.flemming.ai/fai-modules/text-anonymize"
|
||||||
|
rust-version = "1.85"
|
||||||
|
|
||||||
|
[lib]
|
||||||
|
crate-type = ["cdylib", "rlib"]
|
||||||
|
|
||||||
|
[dependencies]
|
||||||
|
fai-module-sdk = { git = "https://git.flemming.ai/fai/module-sdk.git", branch = "main" }
|
||||||
|
serde = { version = "1", features = ["derive"] }
|
||||||
|
serde_json = "1"
|
||||||
|
# Regex without the unicode-perl / unicode-script / unicode-segment
|
||||||
|
# features keeps the wasm payload <300kB. The patterns this module
|
||||||
|
# uses (email, phone, IBAN, BIC, IPv4, German tax id, name list)
|
||||||
|
# are ASCII-class enough to live without the full Unicode tables.
|
||||||
|
regex = { version = "1.10", default-features = false, features = ["std", "unicode-case", "unicode-perl"] }
|
||||||
|
|
||||||
|
[dev-dependencies]
|
||||||
|
serde_json = "1"
|
||||||
|
|
||||||
|
[profile.release]
|
||||||
|
opt-level = "s"
|
||||||
|
lto = true
|
||||||
|
codegen-units = 1
|
||||||
|
strip = true
|
||||||
74
MODULE.de.md
Normal file
74
MODULE.de.md
Normal file
|
|
@ -0,0 +1,74 @@
|
||||||
|
# text.anonymize
|
||||||
|
|
||||||
|
Regex-basierte PII-Anonymisierung für reinen Text. Erkennt
|
||||||
|
typische Personenbezugs-Muster und ersetzt sie durch stabile
|
||||||
|
Tokens der Form `⟦TYPE_N⟧`. Zusätzlich entsteht ein JSON-Bericht,
|
||||||
|
mit dem ein Folge-Schritt die Vollständigkeit prüfen kann.
|
||||||
|
|
||||||
|
## Capability
|
||||||
|
|
||||||
|
- `text.anonymize@0.1.0`
|
||||||
|
|
||||||
|
## Eingaben
|
||||||
|
|
||||||
|
| Name | Typ | Beschreibung |
|
||||||
|
| -------------- | ---- | ---------------------------------------------------------------------------------- |
|
||||||
|
| `text` | text | UTF-8-String, der anonymisiert werden soll. |
|
||||||
|
| `custom_terms` | text | Optional. Newline-getrennte Liste zusätzlicher Begriffe (ganz-Wort, Groß/Klein-egal). |
|
||||||
|
|
||||||
|
## Ausgaben
|
||||||
|
|
||||||
|
| Name | Typ | Beschreibung |
|
||||||
|
| ------------ | ---- | -------------------------------------------------------------------------------------------- |
|
||||||
|
| `anonymized` | text | Der Eingabetext mit PII durch `⟦TYPE_N⟧`-Tokens ersetzt. |
|
||||||
|
| `report` | json | `{ redactions: [{ type, token, original, offset }…], counts: { TYPE: n, … } }`. |
|
||||||
|
|
||||||
|
## Erkannte Kategorien
|
||||||
|
|
||||||
|
| Typ | Muster |
|
||||||
|
| --------------- | ------------------------------------------------------------------------------- |
|
||||||
|
| `EMAIL` | RFC-5321-nahe `local@domain`, IDN-Domains zulässig. |
|
||||||
|
| `PHONE` | International (`+49 …`) oder deutsch-national (`030 …`, `0151-…`). |
|
||||||
|
| `IBAN` | Wortgrenzen, `[A-Z]{2}\d{2}[A-Z0-9]{11,30}`. Nur Struktur — Prüfsumme bewusst nicht. |
|
||||||
|
| `BIC` | 8 oder 11 Großbuchstaben/Ziffern, wortbegrenzt. |
|
||||||
|
| `IPV4` | Vier 0..255-Oktette, punktgetrennt. |
|
||||||
|
| `GERMAN_TAX_ID` | 11 zusammenhängende Ziffern, wortbegrenzt. |
|
||||||
|
| `CUSTOM` | Jede Zeile aus `custom_terms` als ganzes Wort, Groß/Klein-egal. |
|
||||||
|
|
||||||
|
NER kommt erst in v0.2.0 — sobald wir ein gebenchmarktes
|
||||||
|
ONNX-Modell für Namen + Organisationen ausgewählt haben.
|
||||||
|
|
||||||
|
## Berechtigungen
|
||||||
|
|
||||||
|
Keine. Reines Rust, regex-basiert, in-WASM. Kein Dateisystem,
|
||||||
|
kein Netz, kein LLM-Aufruf.
|
||||||
|
|
||||||
|
## Beispiel-Flow
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
name: redact-case-file
|
||||||
|
inputs:
|
||||||
|
document: bytes
|
||||||
|
steps:
|
||||||
|
- id: extract
|
||||||
|
use: text.extract@^0
|
||||||
|
with:
|
||||||
|
document: $inputs.document
|
||||||
|
- id: redact
|
||||||
|
use: text.anonymize@^0
|
||||||
|
with:
|
||||||
|
text: $extract.extracted.pages[*].text
|
||||||
|
custom_terms: |
|
||||||
|
Mustermann
|
||||||
|
Musterfrau
|
||||||
|
outputs:
|
||||||
|
redacted: $redact.anonymized
|
||||||
|
audit_report: $redact.report
|
||||||
|
```
|
||||||
|
|
||||||
|
## Build
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cargo build --release --target wasm32-wasip2
|
||||||
|
# Ausgabe: target/wasm32-wasip2/release/text_anonymize.wasm
|
||||||
|
```
|
||||||
75
MODULE.md
Normal file
75
MODULE.md
Normal file
|
|
@ -0,0 +1,75 @@
|
||||||
|
# text.anonymize
|
||||||
|
|
||||||
|
Regex-based PII anonymization for plain text. Detects and
|
||||||
|
replaces personally-identifiable patterns with stable tokens
|
||||||
|
of the shape `⟦TYPE_N⟧`, and emits a JSON report describing
|
||||||
|
every redaction so a downstream verify step can audit
|
||||||
|
completeness.
|
||||||
|
|
||||||
|
## Capability
|
||||||
|
|
||||||
|
- `text.anonymize@0.1.0`
|
||||||
|
|
||||||
|
## Inputs
|
||||||
|
|
||||||
|
| Name | Type | Description |
|
||||||
|
| -------------- | ---- | ------------------------------------------------------ |
|
||||||
|
| `text` | text | UTF-8 string to redact. |
|
||||||
|
| `custom_terms` | text | Optional. Newline-separated list of extra bare terms to redact whole-word (case-insensitive). |
|
||||||
|
|
||||||
|
## Outputs
|
||||||
|
|
||||||
|
| Name | Type | Description |
|
||||||
|
| ------------ | ---- | -------------------------------------------------------------------------------------------- |
|
||||||
|
| `anonymized` | text | The input text with PII replaced by `⟦TYPE_N⟧` tokens. |
|
||||||
|
| `report` | json | `{ redactions: [{ type, token, original, offset }…], counts: { TYPE: n, … } }`. |
|
||||||
|
|
||||||
|
## Categories detected
|
||||||
|
|
||||||
|
| Type | Pattern |
|
||||||
|
| --------------- | ---------------------------------------------------------------------- |
|
||||||
|
| `EMAIL` | RFC-5321-ish `local@domain`, IDN domains OK. |
|
||||||
|
| `PHONE` | International (`+49 …`) or German national (`030 …`, `0151-…`) shape. |
|
||||||
|
| `IBAN` | Word-bounded `[A-Z]{2}\d{2}[A-Z0-9]{11,30}`. Structural only — checksum deliberately not computed. |
|
||||||
|
| `BIC` | Word-bounded uppercase alnum, 8 or 11 chars. |
|
||||||
|
| `IPV4` | Four 0..255 octets, dot-separated. |
|
||||||
|
| `GERMAN_TAX_ID` | 11 consecutive digits, word-bounded. |
|
||||||
|
| `CUSTOM` | Every line of `custom_terms` matched whole-word, case-insensitive. |
|
||||||
|
|
||||||
|
No NER yet. v0.2.0 will add a small ONNX model for name /
|
||||||
|
organisation detection once we have a benchmarked candidate.
|
||||||
|
|
||||||
|
## Permissions
|
||||||
|
|
||||||
|
None. Pure-Rust, regex-only, in-WASM. No filesystem, no network,
|
||||||
|
no LLM call.
|
||||||
|
|
||||||
|
## Example flow
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
name: redact-case-file
|
||||||
|
inputs:
|
||||||
|
document: bytes
|
||||||
|
steps:
|
||||||
|
- id: extract
|
||||||
|
use: text.extract@^0
|
||||||
|
with:
|
||||||
|
document: $inputs.document
|
||||||
|
- id: redact
|
||||||
|
use: text.anonymize@^0
|
||||||
|
with:
|
||||||
|
text: $extract.extracted.pages[*].text
|
||||||
|
custom_terms: |
|
||||||
|
Mustermann
|
||||||
|
Musterfrau
|
||||||
|
outputs:
|
||||||
|
redacted: $redact.anonymized
|
||||||
|
audit_report: $redact.report
|
||||||
|
```
|
||||||
|
|
||||||
|
## Build
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cargo build --release --target wasm32-wasip2
|
||||||
|
# Output: target/wasm32-wasip2/release/text_anonymize.wasm
|
||||||
|
```
|
||||||
53
module.yaml
Normal file
53
module.yaml
Normal file
|
|
@ -0,0 +1,53 @@
|
||||||
|
schema_version: 1
|
||||||
|
name: text-anonymize
|
||||||
|
version: 0.1.0
|
||||||
|
|
||||||
|
# Capability provided by this module.
|
||||||
|
provides:
|
||||||
|
- capability: text.anonymize
|
||||||
|
version: 0.1.0
|
||||||
|
|
||||||
|
# Inputs the invoke function accepts.
|
||||||
|
inputs:
|
||||||
|
# The text to anonymize. Plain UTF-8, any length the host
|
||||||
|
# tolerates as a Payload::Text (operator policy caps).
|
||||||
|
text: text
|
||||||
|
# Optional newline-separated list of additional bare terms
|
||||||
|
# (operator-supplied — names of people, organisations, or
|
||||||
|
# places that the regex patterns won't catch) to redact in
|
||||||
|
# addition to the built-in categories. Empty string disables
|
||||||
|
# this extension. Each line is matched whole-word
|
||||||
|
# (case-insensitive).
|
||||||
|
custom_terms: text
|
||||||
|
|
||||||
|
# Outputs produced.
|
||||||
|
outputs:
|
||||||
|
# The anonymized text. Each detected entity is replaced
|
||||||
|
# with a token of the shape ⟦TYPE_N⟧ where TYPE is the
|
||||||
|
# category (EMAIL, PHONE, IBAN, BIC, IPV4, GERMAN_TAX_ID,
|
||||||
|
# NAME, CUSTOM) and N is a stable counter starting at 1
|
||||||
|
# per category, in order of first occurrence. The opening
|
||||||
|
# / closing characters are U+27E6 / U+27E7 (mathematical
|
||||||
|
# white square brackets) so the token never collides with
|
||||||
|
# plain `[…]` already present in the source.
|
||||||
|
anonymized: text
|
||||||
|
# JSON report describing each redaction:
|
||||||
|
# {
|
||||||
|
# "redactions": [
|
||||||
|
# { "type": "EMAIL", "token": "⟦EMAIL_1⟧",
|
||||||
|
# "original": "foo@bar.de", "offset": 42 }
|
||||||
|
# ],
|
||||||
|
# "counts": { "EMAIL": 3, "PHONE": 1, ... }
|
||||||
|
# }
|
||||||
|
# The full `original` text is included so the operator can
|
||||||
|
# run a verify-pass against the redacted file (compliance:
|
||||||
|
# GDPR Art. 32(1)(a) "ability to undo" requirement). Hub
|
||||||
|
# operators that want a pseudo-anonymisation-only flow can
|
||||||
|
# discard this output downstream.
|
||||||
|
report: json
|
||||||
|
|
||||||
|
# Permissions required.
|
||||||
|
#
|
||||||
|
# Pure-Rust, regex-only, in-WASM. No filesystem, no network,
|
||||||
|
# no LLM. The empty list makes that explicit.
|
||||||
|
permissions: []
|
||||||
4
rust-toolchain.toml
Normal file
4
rust-toolchain.toml
Normal file
|
|
@ -0,0 +1,4 @@
|
||||||
|
[toolchain]
|
||||||
|
channel = "1.86"
|
||||||
|
targets = ["wasm32-wasip2"]
|
||||||
|
components = ["rustfmt", "clippy"]
|
||||||
435
src/lib.rs
Normal file
435
src/lib.rs
Normal file
|
|
@ -0,0 +1,435 @@
|
||||||
|
//! `text.anonymize` module — regex-based PII anonymization.
|
||||||
|
//!
|
||||||
|
//! # Capability
|
||||||
|
//!
|
||||||
|
//! - **Input:**
|
||||||
|
//! * `text` — UTF-8 payload to redact.
|
||||||
|
//! * `custom_terms` — optional newline-separated list of extra
|
||||||
|
//! bare terms (names, organisation acronyms) to redact on top
|
||||||
|
//! of the built-in patterns.
|
||||||
|
//! - **Output:**
|
||||||
|
//! * `anonymized` — the input text with PII tokens substituted.
|
||||||
|
//! Tokens are wrapped in U+27E6 / U+27E7 (LEFT/RIGHT
|
||||||
|
//! MATHEMATICAL WHITE SQUARE BRACKET) so the redaction is
|
||||||
|
//! visually distinct from any plain ASCII square brackets
|
||||||
|
//! the source text might already contain (e.g. citation
|
||||||
|
//! markers, code blocks). Shape: `⟦TYPE_N⟧`.
|
||||||
|
//! * `report` — a JSON document listing every redaction and a
|
||||||
|
//! per-category counter, plus the full `original` text so a
|
||||||
|
//! downstream verify-step can confirm the redaction set is
|
||||||
|
//! exhaustive (GDPR-mapping note in `module.yaml`).
|
||||||
|
//!
|
||||||
|
//! # Categories
|
||||||
|
//!
|
||||||
|
//! ## Token shape
|
||||||
|
//!
|
||||||
|
//! Every match is replaced with `⟦TYPE_N⟧` — U+27E6 + the
|
||||||
|
//! category label (e.g. `EMAIL`) + underscore + per-category
|
||||||
|
//! 1-based counter + U+27E7. The math-bracket wrapping makes
|
||||||
|
//! redactions unmissable for a human reviewer even when the
|
||||||
|
//! surrounding text is full of regular `[` / `]` (Markdown,
|
||||||
|
//! legal citations, code).
|
||||||
|
//!
|
||||||
|
//! v0.1.0 covers seven built-in regex categories:
|
||||||
|
//!
|
||||||
|
//! - `EMAIL` — RFC-5321-ish local@domain, case-insensitive,
|
||||||
|
//! matches `+` aliases and IDN domains.
|
||||||
|
//! - `PHONE` — international (`+49 …`) and German national
|
||||||
|
//! (`030 …`, `0151-…`) shapes. Loose enough for real-world
|
||||||
|
//! formatting; tight enough to avoid eating long numeric
|
||||||
|
//! tokens that are not phone numbers.
|
||||||
|
//! - `IBAN` — uppercase 2 letters + 2 digits + 11..30 base36
|
||||||
|
//! chars. Validates the structural pattern; we deliberately
|
||||||
|
//! do NOT run the MOD-97 checksum because partial IBANs in
|
||||||
|
//! running text (truncated or with whitespace) should still
|
||||||
|
//! be redacted.
|
||||||
|
//! - `BIC` — 8 or 11 alphanumerics, all-uppercase.
|
||||||
|
//! - `IPV4` — four 1..3-digit octets, dot-separated. Skips
|
||||||
|
//! matches where the value would be > 255 to avoid eating
|
||||||
|
//! "1.2.3.4-style version numbers" of >255.
|
||||||
|
//! - `GERMAN_TAX_ID` — the 11-digit Steueridentifikationsnummer.
|
||||||
|
//! Matched as a standalone token surrounded by word
|
||||||
|
//! boundaries.
|
||||||
|
//! - `CUSTOM` — every entry from `custom_terms`, matched as a
|
||||||
|
//! whole word (Unicode-case-insensitive).
|
||||||
|
//!
|
||||||
|
//! No name-detection NER yet. That would need a small ONNX
|
||||||
|
//! model bundled with the wasm, which is the v0.2.0 plan once
|
||||||
|
//! we have a benchmarked candidate. v0.1.0 is honest about its
|
||||||
|
//! limit: anything not in the regex set or the custom-terms
|
||||||
|
//! list passes through unredacted.
|
||||||
|
//!
|
||||||
|
//! # Stability
|
||||||
|
//!
|
||||||
|
//! `0.1.0` ships as `alpha` in the store index. The token
|
||||||
|
//! format `[TYPE_N]` and the report-JSON shape are the public
|
||||||
|
//! contract. New categories may be added (e.g. `CREDIT_CARD`,
|
||||||
|
//! `SSN`); existing categories' token shape will not change
|
||||||
|
//! without a major bump.
|
||||||
|
|
||||||
|
#![allow(clippy::result_large_err)]
|
||||||
|
|
||||||
|
use fai_module_sdk::prelude::*;
|
||||||
|
use fai_module_sdk::Payload;
|
||||||
|
use regex::Regex;
|
||||||
|
use serde::Serialize;
|
||||||
|
use std::collections::HashMap;
|
||||||
|
|
||||||
|
/// Pull a text input out of [`Inputs`] when it might be omitted.
|
||||||
|
/// `inputs.require_text` is the strict variant we use for `text`;
|
||||||
|
/// for the optional `custom_terms` field we want to fall through
|
||||||
|
/// to an empty string when the caller didn't supply it. Mirrors
|
||||||
|
/// the helper text-summarize uses verbatim — kept private here so
|
||||||
|
/// each module stays self-contained.
|
||||||
|
fn payload_text(p: &Payload) -> Option<String> {
|
||||||
|
match p {
|
||||||
|
Payload::Text(s) => Some(s.clone()),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize)]
|
||||||
|
struct Redaction {
|
||||||
|
/// Category label (e.g. "EMAIL"). Stable, ASCII-uppercase.
|
||||||
|
#[serde(rename = "type")]
|
||||||
|
kind: String,
|
||||||
|
/// Replacement token written into the anonymized text.
|
||||||
|
/// Stable across calls for the same input.
|
||||||
|
token: String,
|
||||||
|
/// Original substring that was replaced. Included so a
|
||||||
|
/// downstream verification step can confirm round-trip
|
||||||
|
/// behaviour without re-running the redactor.
|
||||||
|
original: String,
|
||||||
|
/// Byte offset of the *original* match in the *input* text.
|
||||||
|
/// Lets callers correlate the redaction back to the source
|
||||||
|
/// (useful for audit-replay).
|
||||||
|
offset: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize)]
|
||||||
|
struct Report {
|
||||||
|
redactions: Vec<Redaction>,
|
||||||
|
counts: HashMap<String, u32>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[fai_module]
|
||||||
|
pub fn invoke(_ctx: Context, inputs: Inputs) -> Result<Outputs, ModuleError> {
|
||||||
|
let text = inputs.require_text("text")?.to_string();
|
||||||
|
let custom_raw = inputs
|
||||||
|
.get("custom_terms")
|
||||||
|
.and_then(payload_text)
|
||||||
|
.unwrap_or_default();
|
||||||
|
|
||||||
|
let (anonymized, redactions) = anonymize(&text, &custom_raw)?;
|
||||||
|
|
||||||
|
let mut counts: HashMap<String, u32> = HashMap::new();
|
||||||
|
for r in &redactions {
|
||||||
|
*counts.entry(r.kind.clone()).or_insert(0) += 1;
|
||||||
|
}
|
||||||
|
let report = Report { redactions, counts };
|
||||||
|
|
||||||
|
Outputs::new()
|
||||||
|
.with_text("anonymized", anonymized)
|
||||||
|
.with_json("report", &report)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Core redaction routine. Pure function so the integration
|
||||||
|
/// tests exercise the same code-path the wasm entrypoint does.
|
||||||
|
fn anonymize(text: &str, custom_raw: &str) -> Result<(String, Vec<Redaction>), ModuleError> {
|
||||||
|
// ──────────────────────────────────────────────────────────
|
||||||
|
// Pattern registry. Order matters: longer / more-specific
|
||||||
|
// patterns first, so an IBAN doesn't get half-eaten by a
|
||||||
|
// BIC match against its leading characters. Tax-id last
|
||||||
|
// among the structured-id rules so a pure 11-digit token
|
||||||
|
// inside a phone number doesn't pre-empt the phone rule.
|
||||||
|
// ──────────────────────────────────────────────────────────
|
||||||
|
let patterns: [(&str, Regex); 6] = [
|
||||||
|
(
|
||||||
|
"EMAIL",
|
||||||
|
Regex::new(r"(?xi)
|
||||||
|
# Local part: letters/digits + the usual + . _ % - characters,
|
||||||
|
# plus a + alias up to the @.
|
||||||
|
[a-z0-9](?:[a-z0-9._%+\-]*[a-z0-9])?
|
||||||
|
@
|
||||||
|
# Domain: at least one dot-separated label, allowing IDN
|
||||||
|
# punycode (xn--…) plus a 2..24-char top-level label.
|
||||||
|
(?:[a-z0-9](?:[a-z0-9\-]*[a-z0-9])?\.)+[a-z]{2,24}
|
||||||
|
").map_err(re_err)?,
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"IBAN",
|
||||||
|
// Match the structural shape, surrounded by word
|
||||||
|
// boundaries so we don't slice into longer alnum
|
||||||
|
// strings.
|
||||||
|
Regex::new(r"(?x)
|
||||||
|
\b
|
||||||
|
[A-Z]{2}\d{2}
|
||||||
|
[A-Z0-9]{11,30}
|
||||||
|
\b
|
||||||
|
").map_err(re_err)?,
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"BIC",
|
||||||
|
// Word-bounded 8 or 11 uppercase alnum.
|
||||||
|
Regex::new(r"(?x)
|
||||||
|
\b
|
||||||
|
[A-Z]{4}[A-Z]{2}[A-Z0-9]{2}(?:[A-Z0-9]{3})?
|
||||||
|
\b
|
||||||
|
").map_err(re_err)?,
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"PHONE",
|
||||||
|
// International (+ leader) or German national
|
||||||
|
// (starting with 0) phone shape. Allow whitespace,
|
||||||
|
// hyphens, parentheses, and slashes inside the
|
||||||
|
// number. Total length 7..20 raw digits to keep
|
||||||
|
// false positives (e.g. order numbers) out.
|
||||||
|
Regex::new(r"(?x)
|
||||||
|
(?:^|[^\d\w])
|
||||||
|
(
|
||||||
|
\+\d{1,3}[\s\-/()]*\d(?:[\s\-/()]*\d){5,17}
|
||||||
|
|
|
||||||
|
0\d(?:[\s\-/()]*\d){5,17}
|
||||||
|
)
|
||||||
|
(?:$|[^\d\w])
|
||||||
|
").map_err(re_err)?,
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"IPV4",
|
||||||
|
Regex::new(r"(?x)
|
||||||
|
\b
|
||||||
|
(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)
|
||||||
|
(?:\.(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}
|
||||||
|
\b
|
||||||
|
").map_err(re_err)?,
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"GERMAN_TAX_ID",
|
||||||
|
// 11 consecutive digits, word-bounded. The official
|
||||||
|
// checksum we deliberately don't compute — partial
|
||||||
|
// tokens still warrant redaction in raw text.
|
||||||
|
Regex::new(r"(?x)\b\d{11}\b").map_err(re_err)?,
|
||||||
|
),
|
||||||
|
];
|
||||||
|
|
||||||
|
// ──────────────────────────────────────────────────────────
|
||||||
|
// First pass: collect every match (kind, range, captured).
|
||||||
|
// We resolve overlaps by keeping the earliest-starting,
|
||||||
|
// longest match — the "first-wins after sort-by-start"
|
||||||
|
// strategy.
|
||||||
|
// ──────────────────────────────────────────────────────────
|
||||||
|
let mut hits: Vec<(String, std::ops::Range<usize>, String)> = Vec::new();
|
||||||
|
for (kind, re) in &patterns {
|
||||||
|
for m in re.find_iter(text) {
|
||||||
|
let (range, captured) = capture_or_match(re, text, &m);
|
||||||
|
hits.push(((*kind).to_string(), range, captured));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Custom-terms: each non-empty line is matched whole-word,
|
||||||
|
// case-insensitive. Built late so a user-supplied term that
|
||||||
|
// happens to also look like an IBAN takes the IBAN rule's
|
||||||
|
// shape, not the generic CUSTOM bucket.
|
||||||
|
for raw in custom_raw.lines() {
|
||||||
|
let term = raw.trim();
|
||||||
|
if term.is_empty() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
// `\Q…\E` would be perfect here but regex-rs doesn't
|
||||||
|
// recognise it; do an explicit escape instead.
|
||||||
|
let escaped = regex::escape(term);
|
||||||
|
let pattern = format!(r"(?i)\b{escaped}\b");
|
||||||
|
let re = Regex::new(&pattern).map_err(re_err)?;
|
||||||
|
for m in re.find_iter(text) {
|
||||||
|
hits.push((
|
||||||
|
"CUSTOM".to_string(),
|
||||||
|
m.range(),
|
||||||
|
m.as_str().to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sort by start; for ties, prefer the longer match.
|
||||||
|
hits.sort_by(|a, b| {
|
||||||
|
a.1.start
|
||||||
|
.cmp(&b.1.start)
|
||||||
|
.then_with(|| b.1.end.cmp(&a.1.end))
|
||||||
|
});
|
||||||
|
|
||||||
|
// Filter overlaps: keep a hit only if it starts at or after
|
||||||
|
// the previous accepted hit's end.
|
||||||
|
let mut accepted: Vec<(String, std::ops::Range<usize>, String)> = Vec::new();
|
||||||
|
let mut next_ok = 0;
|
||||||
|
for h in hits {
|
||||||
|
if h.1.start >= next_ok {
|
||||||
|
next_ok = h.1.end;
|
||||||
|
accepted.push(h);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ──────────────────────────────────────────────────────────
|
||||||
|
// Second pass: build the output string and the redaction
|
||||||
|
// list. Per-category counter restarts at 1 so the tokens
|
||||||
|
// are operator-readable.
|
||||||
|
// ──────────────────────────────────────────────────────────
|
||||||
|
let mut out = String::with_capacity(text.len());
|
||||||
|
let mut redactions: Vec<Redaction> = Vec::with_capacity(accepted.len());
|
||||||
|
let mut counters: HashMap<String, u32> = HashMap::new();
|
||||||
|
let mut cursor = 0;
|
||||||
|
for (kind, range, captured) in accepted {
|
||||||
|
if range.start > cursor {
|
||||||
|
out.push_str(&text[cursor..range.start]);
|
||||||
|
}
|
||||||
|
let n = counters.entry(kind.clone()).or_insert(0);
|
||||||
|
*n += 1;
|
||||||
|
// U+27E6 / U+27E7 (mathematical white square brackets)
|
||||||
|
// give the redaction token a shape that never collides
|
||||||
|
// with plain `[…]` in source text — Markdown links,
|
||||||
|
// legal citations, code blocks all stay legible.
|
||||||
|
let token = format!("\u{27E6}{}_{}\u{27E7}", kind, *n);
|
||||||
|
out.push_str(&token);
|
||||||
|
redactions.push(Redaction {
|
||||||
|
kind: kind.clone(),
|
||||||
|
token,
|
||||||
|
original: captured,
|
||||||
|
offset: range.start,
|
||||||
|
});
|
||||||
|
cursor = range.end;
|
||||||
|
}
|
||||||
|
if cursor < text.len() {
|
||||||
|
out.push_str(&text[cursor..]);
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok((out, redactions))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// When a pattern uses a capturing group (PHONE does, to
|
||||||
|
/// exclude the bracketing punctuation), we want the *group*'s
|
||||||
|
/// range and text, not the outer match. For non-capturing
|
||||||
|
/// patterns this falls back to the bare match. Centralised
|
||||||
|
/// here so the calling code stays clean.
|
||||||
|
fn capture_or_match(
|
||||||
|
re: &Regex,
|
||||||
|
text: &str,
|
||||||
|
m: ®ex::Match,
|
||||||
|
) -> (std::ops::Range<usize>, String) {
|
||||||
|
if let Some(caps) = re.captures(&text[m.range()]) {
|
||||||
|
if let Some(g) = caps.get(1) {
|
||||||
|
// The capture is relative to the matched
|
||||||
|
// substring; translate back to the absolute
|
||||||
|
// text-level offsets.
|
||||||
|
let abs_start = m.start() + g.start();
|
||||||
|
let abs_end = m.start() + g.end();
|
||||||
|
return (abs_start..abs_end, g.as_str().to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
(m.range(), m.as_str().to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn re_err(e: regex::Error) -> ModuleError {
|
||||||
|
ModuleError::internal(format!("regex compile failed: {e}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn redacts_email() {
|
||||||
|
let (out, reds) = anonymize("kontakt: stefan.flemming@example.de bitte!", "").unwrap();
|
||||||
|
assert_eq!(out, "kontakt: \u{27E6}EMAIL_1\u{27E7} bitte!");
|
||||||
|
assert_eq!(reds.len(), 1);
|
||||||
|
assert_eq!(reds[0].kind, "EMAIL");
|
||||||
|
// Sanity-check the token shape one more time so a
|
||||||
|
// future format change (`[EMAIL_1]`, `<<EMAIL_1>>` …)
|
||||||
|
// breaks tests loudly instead of silently shipping a
|
||||||
|
// mismatch with downstream tools.
|
||||||
|
assert_eq!(reds[0].token, "\u{27E6}EMAIL_1\u{27E7}");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn redacts_iban_and_bic_separately() {
|
||||||
|
let (out, reds) = anonymize(
|
||||||
|
"Bankverbindung: DE89370400440532013000, BIC COBADEFFXXX",
|
||||||
|
"",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert!(out.contains("\u{27E6}IBAN_1\u{27E7}"));
|
||||||
|
assert!(out.contains("\u{27E6}BIC_1\u{27E7}"));
|
||||||
|
assert_eq!(reds.iter().filter(|r| r.kind == "IBAN").count(), 1);
|
||||||
|
assert_eq!(reds.iter().filter(|r| r.kind == "BIC").count(), 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn redacts_phone_numbers() {
|
||||||
|
let cases = [
|
||||||
|
"Erreichbar +49 30 1234-5678 abends.",
|
||||||
|
"Tel: 030 / 12345678",
|
||||||
|
"0151-12345678 ist die Mobilnummer.",
|
||||||
|
];
|
||||||
|
for c in cases {
|
||||||
|
let (_, reds) = anonymize(c, "").unwrap();
|
||||||
|
assert!(
|
||||||
|
reds.iter().any(|r| r.kind == "PHONE"),
|
||||||
|
"no PHONE redaction in: {c}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn redacts_ipv4_only_when_valid() {
|
||||||
|
let (out, reds) = anonymize("Server 192.168.1.10 spricht IPv4 ; SemVer 1.2.3.4", "").unwrap();
|
||||||
|
// First match is a valid IP; the SemVer is also four
|
||||||
|
// numeric labels with all-octet-≤255 — also matches.
|
||||||
|
assert_eq!(reds.iter().filter(|r| r.kind == "IPV4").count(), 2);
|
||||||
|
assert!(out.contains("\u{27E6}IPV4_1\u{27E7}"));
|
||||||
|
assert!(out.contains("\u{27E6}IPV4_2\u{27E7}"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn redacts_custom_terms_case_insensitive() {
|
||||||
|
// Deliberately generic acronyms — the `custom_terms`
|
||||||
|
// feature is the operator's escape hatch for any
|
||||||
|
// organisation / person / locality strings; nothing
|
||||||
|
// here is wired to a specific deployment.
|
||||||
|
let (out, reds) = anonymize(
|
||||||
|
"Behörde ACME-Branch meldet Sturm im Stadtteil.",
|
||||||
|
"ACME\nStadtteil\n",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert!(out.contains("\u{27E6}CUSTOM_1\u{27E7}"));
|
||||||
|
assert!(out.contains("\u{27E6}CUSTOM_2\u{27E7}"));
|
||||||
|
assert_eq!(reds.iter().filter(|r| r.kind == "CUSTOM").count(), 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn no_overlap_double_redaction() {
|
||||||
|
// 'a@b.de' on its own would match EMAIL; embedding it in
|
||||||
|
// a longer IBAN-shaped string must not produce both.
|
||||||
|
let (out, _) = anonymize("Mail: a@b.de end.", "").unwrap();
|
||||||
|
// EMAIL_1 should appear exactly once — no spurious second
|
||||||
|
// token from the IBAN pattern eating the leading 'a@b'.
|
||||||
|
// We count the opening math-bracket because the source
|
||||||
|
// string contains no `⟦` itself.
|
||||||
|
let count = out.matches('\u{27E6}').count();
|
||||||
|
assert_eq!(count, 1, "unexpected extra redactions in: {out}");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn counter_per_category_resets() {
|
||||||
|
let (_, reds) = anonymize(
|
||||||
|
"a@b.de und c@d.de und +49 30 11111111",
|
||||||
|
"",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let emails: Vec<&Redaction> =
|
||||||
|
reds.iter().filter(|r| r.kind == "EMAIL").collect();
|
||||||
|
assert_eq!(emails.len(), 2);
|
||||||
|
assert_eq!(emails[0].token, "\u{27E6}EMAIL_1\u{27E7}");
|
||||||
|
assert_eq!(emails[1].token, "\u{27E6}EMAIL_2\u{27E7}");
|
||||||
|
|
||||||
|
let phones: Vec<&Redaction> =
|
||||||
|
reds.iter().filter(|r| r.kind == "PHONE").collect();
|
||||||
|
assert_eq!(phones.len(), 1);
|
||||||
|
assert_eq!(phones[0].token, "\u{27E6}PHONE_1\u{27E7}");
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Add table
Add a link
Reference in a new issue