feat: initial text-akoma-normalize v0.1.0 (text.akoma_normalize@0.1.0)
First F∆I capability module that normalises heterogeneous legal
document sources to a structured representation aligned with
Akoma Ntoso 1.0 (OASIS LegalDocML).
v0.1.0 ships one adapter:
- gesetze-im-internet (BMJ) XML — the BMJ XML schema with
jurabk / enbez / titel / textdaten elements. Lossy but
deterministic: SHA-256 of source + SHA-256 of canonicalised
payload land in the output so the Hub audit log can verify
that downstream evaluations were computed against the same
bytes the auditor sees.
Reserved for next versions, without an interface break:
- 0.2: landesrecht-berlin HTML adapter
- 0.3: EUR-Lex FORMEX-4 adapter
- 0.4+: emit akoma_xml (full Akoma Ntoso roundtrip)
ELI synthesis in 0.1 is a deterministic eli/bund/<jurabk>/<para>
fallback because Germany lacks an upstream Bund-level ELI
registrar. Replace once BMJ publishes ELI URIs upstream.
Pure in-WASM (quick-xml + sha2), zero filesystem, zero network —
the empty permissions list in module.yaml makes that explicit
and is enforced by the Hub.
Signed-off-by: flemming-it <sf@flemming.it>
This commit is contained in:
commit
b73b3e4b73
6 changed files with 1025 additions and 0 deletions
357
src/lib.rs
Normal file
357
src/lib.rs
Normal file
|
|
@ -0,0 +1,357 @@
|
|||
//! `text.akoma_normalize` module — normalise heterogeneous legal
|
||||
//! document sources to a structured representation aligned with
|
||||
//! Akoma Ntoso 1.0 (OASIS LegalDocML).
|
||||
//!
|
||||
//! # Capability
|
||||
//!
|
||||
//! - **Input:**
|
||||
//! * `content` — `bytes` payload of one legal-document file.
|
||||
//! * `mime` — optional `text` MIME hint. If omitted, the module
|
||||
//! sniffs leading bytes.
|
||||
//! - **Output:**
|
||||
//! * `norm` — `json` payload conforming to the `NormalizedNorm`
|
||||
//! schema below.
|
||||
//!
|
||||
//! # Adapter coverage (v0.1.0)
|
||||
//!
|
||||
//! Ships ONE adapter: gesetze-im-internet (BMJ) XML schema.
|
||||
//! HTML (landesrecht-berlin) and FORMEX (EUR-Lex) adapters follow
|
||||
//! in 0.2 / 0.3 without an interface break — the `adapter` field
|
||||
//! tags which adapter ran, so downstream consumers can route.
|
||||
//!
|
||||
//! # ELI synthesis (v0.1.0 caveat)
|
||||
//!
|
||||
//! Germany lacks an official Bundesebene ELI registrar. v0.1.0
|
||||
//! synthesises a deterministic ELI from the source metadata
|
||||
//! (`jurabk` + paragraph label), prefixed `eli/bund/` to mark it
|
||||
//! synthetic. Replace once BMJ publishes ELI URIs upstream.
|
||||
//!
|
||||
//! # Stability
|
||||
//!
|
||||
//! `0.1.0` ships as `alpha` in the store index. The output schema
|
||||
//! is intentionally forward-compatible: future adapters add
|
||||
//! fields, never rename or repurpose.
|
||||
|
||||
#![allow(clippy::result_large_err)]
|
||||
|
||||
use chain_module_sdk::prelude::*;
|
||||
use quick_xml::events::Event;
|
||||
use quick_xml::reader::Reader;
|
||||
use serde::Serialize;
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
const XML_MIME: &str = "application/xml";
|
||||
const XML_MIME_ALT: &str = "text/xml";
|
||||
|
||||
const ADAPTER_GII: &str = "gesetze-im-internet-xml";
|
||||
const ADAPTER_GII_VERSION: &str = "0.1.0";
|
||||
const SCHEMA_VERSION: &str = "0.1";
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct NormalizedNorm {
|
||||
schema_version: &'static str,
|
||||
adapter: &'static str,
|
||||
adapter_version: &'static str,
|
||||
eli: String,
|
||||
jurabk: Option<String>,
|
||||
paragraph: Option<String>,
|
||||
title: Option<String>,
|
||||
paragraphs: Vec<Paragraph>,
|
||||
source_sha256: String,
|
||||
normalized_sha256: String,
|
||||
/// Akoma Ntoso XML emission is reserved for 0.2+. v0.1 omits.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
akoma_xml: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct Paragraph {
|
||||
/// Absatz number as it appears in the source (e.g. "(1)", "(2)").
|
||||
/// Empty when the source provides no numbering.
|
||||
number: String,
|
||||
text: String,
|
||||
}
|
||||
|
||||
#[fai_module]
|
||||
pub fn invoke(_ctx: Context, inputs: Inputs) -> Result<Outputs, ModuleError> {
|
||||
let bytes = inputs.require_bytes("content")?;
|
||||
let mime_hint_lower: Option<String> =
|
||||
inputs.get("mime").and_then(|p| match p {
|
||||
Payload::Text(s) => Some(s.to_ascii_lowercase()),
|
||||
_ => None,
|
||||
});
|
||||
|
||||
let mime = mime_hint_lower
|
||||
.or_else(|| sniff_mime(&bytes.data))
|
||||
.unwrap_or_else(|| bytes.mime_type.clone());
|
||||
|
||||
let norm = match mime.as_str() {
|
||||
XML_MIME | XML_MIME_ALT => normalize_gii_xml(&bytes.data)?,
|
||||
other => {
|
||||
return Err(ModuleError::invalid_input(format!(
|
||||
"unsupported MIME in v0.1.0: {other} \
|
||||
(supported: {XML_MIME}, {XML_MIME_ALT}; \
|
||||
HTML/FORMEX adapters land in 0.2+)",
|
||||
)));
|
||||
}
|
||||
};
|
||||
Outputs::new().with_json("norm", &norm)
|
||||
}
|
||||
|
||||
/// Naive sniff: if the buffer starts with `<?xml` or `<dokumente`,
|
||||
/// we call it XML. Real adapter dispatch happens in `invoke` once
|
||||
/// MIME is known.
|
||||
fn sniff_mime(data: &[u8]) -> Option<String> {
|
||||
let head = data.get(..256.min(data.len()))?;
|
||||
let head_str = core::str::from_utf8(head).ok()?.trim_start();
|
||||
if head_str.starts_with("<?xml") || head_str.starts_with("<dokumente") {
|
||||
Some(XML_MIME.to_string())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// gesetze-im-internet (BMJ) XML schema adapter.
|
||||
///
|
||||
/// Extracts the well-known metadata keys (`jurabk`, `enbez`,
|
||||
/// `titel`) and joins the body text. Forward-compatible: unknown
|
||||
/// elements are silently skipped, so a schema extension upstream
|
||||
/// does not break this adapter.
|
||||
fn normalize_gii_xml(data: &[u8]) -> Result<NormalizedNorm, ModuleError> {
|
||||
let source_sha = hex_sha256(data);
|
||||
|
||||
let mut reader = Reader::from_reader(data);
|
||||
reader.config_mut().trim_text(true);
|
||||
|
||||
let mut jurabk: Option<String> = None;
|
||||
let mut paragraph: Option<String> = None;
|
||||
let mut title: Option<String> = None;
|
||||
let mut text_buffer = String::new();
|
||||
let mut in_text = false;
|
||||
let mut current_tag: Option<Vec<u8>> = None;
|
||||
let mut buf = Vec::new();
|
||||
|
||||
loop {
|
||||
match reader.read_event_into(&mut buf) {
|
||||
Ok(Event::Start(e)) => {
|
||||
let name = e.name().as_ref().to_vec();
|
||||
if name == b"text" || name == b"textdaten" {
|
||||
in_text = true;
|
||||
}
|
||||
current_tag = Some(name);
|
||||
}
|
||||
Ok(Event::End(e)) => {
|
||||
let name_owned = e.name();
|
||||
let name = name_owned.as_ref();
|
||||
if name == b"text" || name == b"textdaten" {
|
||||
in_text = false;
|
||||
}
|
||||
current_tag = None;
|
||||
}
|
||||
Ok(Event::Text(t)) => {
|
||||
let text = t.unescape().map_err(xml_err)?.into_owned();
|
||||
if in_text {
|
||||
if !text_buffer.is_empty() {
|
||||
text_buffer.push('\n');
|
||||
}
|
||||
text_buffer.push_str(&text);
|
||||
} else if let Some(tag) = current_tag.as_deref() {
|
||||
match tag {
|
||||
b"jurabk" if jurabk.is_none() => jurabk = Some(text),
|
||||
b"enbez" if paragraph.is_none() => paragraph = Some(text),
|
||||
b"titel" if title.is_none() => title = Some(text),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(Event::Eof) => break,
|
||||
Err(e) => return Err(xml_err(e)),
|
||||
_ => {}
|
||||
}
|
||||
buf.clear();
|
||||
}
|
||||
|
||||
let paragraphs = split_paragraphs(&text_buffer);
|
||||
let eli = synthesise_eli(jurabk.as_deref(), paragraph.as_deref());
|
||||
let normalized_payload = format!(
|
||||
"{}|{}|{}|{}",
|
||||
eli,
|
||||
jurabk.as_deref().unwrap_or(""),
|
||||
paragraph.as_deref().unwrap_or(""),
|
||||
text_buffer,
|
||||
);
|
||||
let normalized_sha = hex_sha256(normalized_payload.as_bytes());
|
||||
|
||||
Ok(NormalizedNorm {
|
||||
schema_version: SCHEMA_VERSION,
|
||||
adapter: ADAPTER_GII,
|
||||
adapter_version: ADAPTER_GII_VERSION,
|
||||
eli,
|
||||
jurabk,
|
||||
paragraph,
|
||||
title,
|
||||
paragraphs,
|
||||
source_sha256: source_sha,
|
||||
normalized_sha256: normalized_sha,
|
||||
akoma_xml: None,
|
||||
})
|
||||
}
|
||||
|
||||
/// Split the joined text into Absatz units when leading
|
||||
/// `(N)` markers are present. Sources without numbered Absatzes
|
||||
/// land as a single Paragraph with empty number.
|
||||
fn split_paragraphs(text: &str) -> Vec<Paragraph> {
|
||||
let text = text.trim();
|
||||
if text.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
let mut out: Vec<Paragraph> = Vec::new();
|
||||
let mut current = Paragraph { number: String::new(), text: String::new() };
|
||||
for line in text.lines() {
|
||||
let line = line.trim();
|
||||
if line.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if let Some(num) = leading_paragraph_marker(line) {
|
||||
if !current.text.is_empty() {
|
||||
out.push(core::mem::replace(
|
||||
&mut current,
|
||||
Paragraph { number: String::new(), text: String::new() },
|
||||
));
|
||||
}
|
||||
current.number = num.to_string();
|
||||
let rest = line.get(num.len()..).unwrap_or("").trim_start();
|
||||
current.text.push_str(rest);
|
||||
} else {
|
||||
if !current.text.is_empty() {
|
||||
current.text.push('\n');
|
||||
}
|
||||
current.text.push_str(line);
|
||||
}
|
||||
}
|
||||
if !current.text.is_empty() || !current.number.is_empty() {
|
||||
out.push(current);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Recognises `(1)`, `(2a)`, `(10)` at the start of a line.
|
||||
fn leading_paragraph_marker(line: &str) -> Option<&str> {
|
||||
let bytes = line.as_bytes();
|
||||
if bytes.first() != Some(&b'(') {
|
||||
return None;
|
||||
}
|
||||
let close = bytes.iter().position(|&b| b == b')')?;
|
||||
if close < 2 || close > 5 {
|
||||
return None;
|
||||
}
|
||||
let inner = line.get(1..close)?;
|
||||
if inner.chars().all(|c| c.is_ascii_alphanumeric()) {
|
||||
line.get(..=close)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Synthetic ELI: `eli/bund/<jurabk-lower>/<paragraph-token>`.
|
||||
/// Lossy and deterministic. Replace when BMJ publishes upstream
|
||||
/// ELI URIs.
|
||||
fn synthesise_eli(jurabk: Option<&str>, paragraph: Option<&str>) -> String {
|
||||
let abk = jurabk.unwrap_or("unknown").trim().to_ascii_lowercase();
|
||||
let para = paragraph
|
||||
.unwrap_or("")
|
||||
.chars()
|
||||
.filter(|c| c.is_ascii_alphanumeric())
|
||||
.collect::<String>()
|
||||
.to_ascii_lowercase();
|
||||
if para.is_empty() {
|
||||
format!("eli/bund/{abk}")
|
||||
} else {
|
||||
format!("eli/bund/{abk}/{para}")
|
||||
}
|
||||
}
|
||||
|
||||
fn hex_sha256(data: &[u8]) -> String {
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(data);
|
||||
let digest = hasher.finalize();
|
||||
let mut s = String::with_capacity(64);
|
||||
for byte in digest {
|
||||
let hi = byte >> 4;
|
||||
let lo = byte & 0x0f;
|
||||
s.push(nibble_to_hex(hi));
|
||||
s.push(nibble_to_hex(lo));
|
||||
}
|
||||
s
|
||||
}
|
||||
|
||||
fn nibble_to_hex(n: u8) -> char {
|
||||
match n {
|
||||
0..=9 => (b'0' + n) as char,
|
||||
10..=15 => (b'a' + n - 10) as char,
|
||||
_ => '?',
|
||||
}
|
||||
}
|
||||
|
||||
fn xml_err<E: core::fmt::Display>(e: E) -> ModuleError {
|
||||
ModuleError::invalid_input(format!("XML parse failed: {e}"))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const SAMPLE_GEWO_14: &str = r#"<?xml version="1.0" encoding="UTF-8"?>
|
||||
<dokumente>
|
||||
<norm doknr="BJNR9410000">
|
||||
<metadaten>
|
||||
<jurabk>GewO</jurabk>
|
||||
<enbez>§ 14</enbez>
|
||||
<titel>Anzeigepflicht</titel>
|
||||
</metadaten>
|
||||
<textdaten>
|
||||
<text>
|
||||
(1) Wer den selbstständigen Betrieb eines stehenden Gewerbes anfängt, muss dies der zuständigen Behörde gleichzeitig anzeigen.
|
||||
(2) Die Anzeige hat unverzüglich zu erfolgen.
|
||||
</text>
|
||||
</textdaten>
|
||||
</norm>
|
||||
</dokumente>"#;
|
||||
|
||||
#[test]
|
||||
fn parses_gewo_14_metadata() {
|
||||
let norm = normalize_gii_xml(SAMPLE_GEWO_14.as_bytes()).expect("parse ok");
|
||||
assert_eq!(norm.jurabk.as_deref(), Some("GewO"));
|
||||
assert_eq!(norm.paragraph.as_deref(), Some("§ 14"));
|
||||
assert_eq!(norm.title.as_deref(), Some("Anzeigepflicht"));
|
||||
assert_eq!(norm.eli, "eli/bund/gewo/14");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn splits_into_absatz_paragraphs() {
|
||||
let norm = normalize_gii_xml(SAMPLE_GEWO_14.as_bytes()).expect("parse ok");
|
||||
assert_eq!(norm.paragraphs.len(), 2);
|
||||
assert_eq!(norm.paragraphs[0].number, "(1)");
|
||||
assert!(norm.paragraphs[0].text.starts_with("Wer den selbstständigen"));
|
||||
assert_eq!(norm.paragraphs[1].number, "(2)");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deterministic_hash() {
|
||||
let a = normalize_gii_xml(SAMPLE_GEWO_14.as_bytes()).expect("parse ok");
|
||||
let b = normalize_gii_xml(SAMPLE_GEWO_14.as_bytes()).expect("parse ok");
|
||||
assert_eq!(a.source_sha256, b.source_sha256);
|
||||
assert_eq!(a.normalized_sha256, b.normalized_sha256);
|
||||
assert_eq!(a.source_sha256.len(), 64);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sniffs_xml_from_dokumente() {
|
||||
assert_eq!(
|
||||
sniff_mime(b"<?xml version=\"1.0\"?><dokumente>"),
|
||||
Some("application/xml".into())
|
||||
);
|
||||
assert_eq!(sniff_mime(b"<dokumente><norm/>"), Some("application/xml".into()));
|
||||
assert_eq!(sniff_mime(b"%PDF-1.4 ..."), None);
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue