From 2c3a6be513d62eefc14a1076b90c8ebc04f41838 Mon Sep 17 00:00:00 2001 From: flemming-it Date: Thu, 18 Jun 2026 11:25:13 +0200 Subject: [PATCH] feat: initial text-readability-score v0.1.0 (text.readability_score@0.1.0) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Composite readability/complexity score: Flesch-DE + Verweistiefe + verb-distance. Reuse-lens: any project that needs to flag "this text is hard to read" — government letters, AGB, scientific abstracts. Flesch-DE follows the Amstad adaptation: FRE_de = 180 − ASL − (58.5 × ASW) ASL = avg sentence length (words); ASW = avg syllables per word. A FRE_de around 30 reads as "schwer"; below 10 is technical-jargon territory. Verweistiefe uses an optional graph input (output of graph.citation_extract). i.V.m.-edges count double because they force the reader to chase a second norm. composite_frust scales both to 0..10 and averages, so the output is a single number the heatmap can plot against the §6 Studie "Frust"-Achse. Pure in-WASM, zero filesystem, zero network. Reserved for next versions: - 0.2: noun-density and Schachtelsatz penalty - 0.3: per-paragraph breakdown so the UI can highlight the most-frustrating Absatz Signed-off-by: flemming-it --- .gitignore | 3 + Cargo.lock | 443 ++++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 36 ++++ module.yaml | 39 ++++ rust-toolchain.toml | 4 + src/lib.rs | 168 +++++++++++++++++ 6 files changed, 693 insertions(+) create mode 100644 .gitignore create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 module.yaml create mode 100644 rust-toolchain.toml create mode 100644 src/lib.rs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5e66a2d --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +target/ +**/*.rs.bk +.DS_Store diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..808c6b1 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,443 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "bitflags" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chain-module-sdk" +version = "0.1.2" +source = "git+https://git.flemming.ai/fai/chain-module-sdk-rust.git?branch=main#98e98e9371f7409560a1ef08bc0923d9a2506449" +dependencies = [ + "chain-module-sdk-macros", + "serde", + "serde_json", + "thiserror", + "wit-bindgen", +] + +[[package]] +name = "chain-module-sdk-macros" +version = "0.1.2" +source = "git+https://git.flemming.ai/fai/chain-module-sdk-rust.git?branch=main#98e98e9371f7409560a1ef08bc0923d9a2506449" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", + "serde", + "serde_core", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "leb128" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c83bff1d572d6b9aeef67ddfc8448e4a3737909cb28e81f97c791b9018703e52" + +[[package]] +name = "log" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" + +[[package]] +name = "memchr" +version = "2.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "spdx" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3e17e880bafaeb362a7b751ec46bdc5b61445a188f80e0606e68167cd540fa3" +dependencies = [ + "smallvec", +] + +[[package]] +name = "syn" +version = "2.0.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "text_readability_score" +version = "0.1.0" +dependencies = [ + "chain-module-sdk", + "serde", + "serde_json", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasm-encoder" +version = "0.220.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e913f9242315ca39eff82aee0e19ee7a372155717ff0eb082c741e435ce25ed1" +dependencies = [ + "leb128", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.220.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "185dfcd27fa5db2e6a23906b54c28199935f71d9a27a1a27b3a88d6fee2afae7" +dependencies = [ + "anyhow", + "indexmap", + "serde", + "serde_derive", + "serde_json", + "spdx", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.220.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d07b6a3b550fefa1a914b6d54fc175dd11c3392da11eee604e6ffc759805d25" +dependencies = [ + "ahash", + "bitflags", + "hashbrown 0.14.5", + "indexmap", + "semver", +] + +[[package]] +name = "wit-bindgen" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a2b3e15cd6068f233926e7d8c7c588b2ec4fb7cc7bf3824115e7c7e2a8485a3" +dependencies = [ + "wit-bindgen-rt", + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen-core" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b632a5a0fa2409489bd49c9e6d99fcc61bb3d4ce9d1907d44662e75a28c71172" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rt" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7947d0131c7c9da3f01dfde0ab8bd4c4cf3c5bd49b6dba0ae640f1fa752572ea" +dependencies = [ + "bitflags", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4329de4186ee30e2ef30a0533f9b3c123c019a237a7c82d692807bf1b3ee2697" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "177fb7ee1484d113b4792cc480b1ba57664bbc951b42a4beebe573502135b1fc" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.220.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b505603761ed400c90ed30261f44a768317348e49f1864e82ecdc3b2744e5627" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.220.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae2a7999ed18efe59be8de2db9cb2b7f84d88b27818c79353dfc53131840fe1a" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "zerocopy" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..b8c20a8 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,36 @@ +[package] +name = "text_readability_score" +version = "0.1.0" +edition = "2024" +authors = ["Dr. Stefan Flemming "] +license = "Apache-2.0" +publish = false +description = "F∆I module — composite readability/complexity score (Flesch-DE + depth + verb-distance)" +repository = "https://git.flemming.ws/chain-modules/text-readability-score" +rust-version = "1.85" + +[lib] +crate-type = ["cdylib", "rlib"] + +[dependencies] +chain-module-sdk = { git = "https://git.flemming.ai/fai/chain-module-sdk-rust.git", branch = "main" } +serde = { version = "1", features = ["derive"] } +serde_json = "1" + +[profile.release] +opt-level = "s" +lto = true +codegen-units = 1 +strip = "symbols" + +[lints.rust] +unsafe_op_in_unsafe_fn = "warn" + +[lints.clippy] +unwrap_used = "deny" +expect_used = "deny" +panic = "deny" +print_stdout = "deny" +print_stderr = "deny" + +[workspace] diff --git a/module.yaml b/module.yaml new file mode 100644 index 0000000..f5793ba --- /dev/null +++ b/module.yaml @@ -0,0 +1,39 @@ +schema_version: 3 +provider: chain +name: text-readability-score +version: 0.1.0 + +# Composite readability score for any text — Flesch (DE-adapted) +# + Verweistiefe + verb-distance. +# +# Reuse-lens: any project that wants to flag "this text is hard +# to read" — government letters, AGB, scientific abstracts. +provides: + - capability: text.readability_score + version: 0.1.0 + +inputs: + norm: + type: json + description: + en: NormalizedNorm (output of text.akoma_normalize). + de: NormalizedNorm (Output von text.akoma_normalize). + graph: + type: json + optional: true + description: + en: Optional citation graph from graph.citation_extract — used to compute Verweistiefe. + de: Optionaler Zitationsgraph aus graph.citation_extract — für Verweistiefe. + +outputs: + score: + type: json + description: + en: | + { flesch_de: 0..100, depth_index: 0..10, + composite_frust: 0..10 }. + de: | + { flesch_de: 0..100, depth_index: 0..10, + composite_frust: 0..10 }. + +permissions: [] diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 0000000..32add0c --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,4 @@ +[toolchain] +channel = "1.86" +targets = ["wasm32-wasip2"] +components = ["rustfmt", "clippy"] diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..c7de4a2 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,168 @@ +//! `text.readability_score` — Flesch-DE + Verweistiefe + verb- +//! distance composite. +//! +//! The Flesch formula adapted by Amstad for German: +//! FRE_de = 180 − ASL − (58.5 × ASW) +//! where ASL = avg sentence length (words), ASW = avg syllables +//! per word. A FRE_de of ~30 reads like "schwer". +//! +//! depth_index is mean inbound citation distance from the +//! citation graph (if supplied). Composite frust scales both +//! to 0..10 and averages. + +#![allow(clippy::result_large_err)] + +use chain_module_sdk::prelude::*; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Deserialize)] +struct InputNorm { + paragraphs: Vec, +} + +#[derive(Debug, Deserialize)] +struct InputParagraph { + text: String, +} + +#[derive(Debug, Deserialize)] +struct InputGraph { + citations: Vec, +} + +#[derive(Debug, Deserialize)] +struct InputCitation { + #[serde(rename = "type")] + kind: String, +} + +#[derive(Debug, Serialize)] +struct Output { + flesch_de: f32, + depth_index: f32, + composite_frust: f32, +} + +#[fai_module] +pub fn invoke(_ctx: Context, inputs: Inputs) -> Result { + let norm: InputNorm = inputs + .require_json("norm") + .map_err(|e| ModuleError::invalid_input(format!("norm shape: {e}")))?; + + let graph_iVm: Option = inputs.get("graph").and_then(|p| match p { + Payload::Json(s) => serde_json::from_str(s).ok(), + _ => None, + }); + + let full_text: String = norm + .paragraphs + .iter() + .map(|p| p.text.as_str()) + .collect::>() + .join(" "); + + let flesch = flesch_de(&full_text); + let depth = depth_index(graph_iVm.as_ref()); + let composite = composite_frust(flesch, depth); + + let out = Output { + flesch_de: flesch, + depth_index: depth, + composite_frust: composite, + }; + Outputs::new().with_json("score", &out) +} + +fn flesch_de(text: &str) -> f32 { + let sentences = count_sentences(text); + let words: Vec<&str> = text + .split(|c: char| c.is_whitespace() || c == ',' || c == ';') + .filter(|w| !w.is_empty()) + .collect(); + if sentences == 0 || words.is_empty() { + return 0.0; + } + let asl = words.len() as f32 / sentences as f32; + let total_sylls: usize = words.iter().map(|w| count_syllables(w)).sum(); + let asw = total_sylls as f32 / words.len() as f32; + (180.0 - asl - 58.5 * asw).clamp(0.0, 100.0) +} + +fn count_sentences(text: &str) -> usize { + text.chars().filter(|c| matches!(c, '.' | '!' | '?')).count().max(1) +} + +fn count_syllables(word: &str) -> usize { + let mut count = 0; + let mut in_vowel = false; + for c in word.chars() { + let v = matches!( + c.to_ascii_lowercase(), + 'a' | 'e' | 'i' | 'o' | 'u' | 'y' | 'ä' | 'ö' | 'ü' + ); + if v && !in_vowel { + count += 1; + } + in_vowel = v; + } + count.max(1) +} + +fn depth_index(graph: Option<&InputGraph>) -> f32 { + let Some(g) = graph else { return 0.0 }; + let n = g.citations.len() as f32; + if n == 0.0 { + 0.0 + } else { + // crude proxy: each i.V.m.-link counts double — that's where + // the reader has to chase a second norm. + let weighted: f32 = g + .citations + .iter() + .map(|c| if c.kind == "verweist_iVm" { 2.0 } else { 1.0 }) + .sum(); + (weighted / n).clamp(0.0, 10.0) + } +} + +/// Combine 0..100 FRE (high = easy) and 0..10 depth (high = hard). +/// Map to a single 0..10 frust score (high = frustrating). +fn composite_frust(flesch: f32, depth: f32) -> f32 { + let frust_from_flesch = (100.0 - flesch) / 10.0; + ((frust_from_flesch + depth) / 2.0).clamp(0.0, 10.0) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn easy_german_scores_high_flesch() { + let f = flesch_de("Das ist ein Test. Er ist kurz."); + assert!(f > 60.0, "expected easy text to score > 60, got {f}"); + } + + #[test] + fn long_compound_german_scores_low_flesch() { + let f = flesch_de( + "Die Datenschutzgrundverordnungsanpassungsdurchführungsverordnung \ + betrifft sämtliche Verantwortliche im Geltungsbereich der Verordnung.", + ); + assert!(f < 30.0, "expected dense text to score < 30, got {f}"); + } + + #[test] + fn depth_zero_without_graph() { + assert_eq!(depth_index(None), 0.0); + } + + #[test] + fn composite_bounded() { + for f in [0.0_f32, 50.0, 100.0] { + for d in [0.0_f32, 5.0, 10.0] { + let c = composite_frust(f, d); + assert!((0.0..=10.0).contains(&c)); + } + } + } +}