Compare commits
12 commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 10aa1e1b51 | |||
| 62e0d0b7b7 | |||
| 36b175d334 | |||
| 93bce2de12 | |||
| 4b649f75bf | |||
| 3b9aebd18e | |||
| 331a97d3c6 | |||
| 6156cd5bd8 | |||
| 9732a88c3e | |||
| 20ff097e3b | |||
| fdb5deaa8b | |||
| 20b30d0b12 |
12 changed files with 482 additions and 44 deletions
|
|
@ -30,7 +30,7 @@ jobs:
|
||||||
git fetch --depth=1 origin "$GITHUB_SHA"
|
git fetch --depth=1 origin "$GITHUB_SHA"
|
||||||
git checkout -q FETCH_HEAD
|
git checkout -q FETCH_HEAD
|
||||||
|
|
||||||
# The fai-module-sdk dependency is a git dep against a repo
|
# The chain-module-sdk dependency is a git dep against a repo
|
||||||
# that lives in another Forgejo org. The repo-scoped
|
# that lives in another Forgejo org. The repo-scoped
|
||||||
# GITHUB_TOKEN cannot read it, and the Forgejo instance has
|
# GITHUB_TOKEN cannot read it, and the Forgejo instance has
|
||||||
# REQUIRE_SIGNIN_VIEW=true so anonymous read is also denied.
|
# REQUIRE_SIGNIN_VIEW=true so anonymous read is also denied.
|
||||||
|
|
|
||||||
223
.forgejo/workflows/sign.yml
Normal file
223
.forgejo/workflows/sign.yml
Normal file
|
|
@ -0,0 +1,223 @@
|
||||||
|
# F∆I module bundle signing workflow (cosign key-pinning model).
|
||||||
|
#
|
||||||
|
# Drop-in template for any module repo under chain-modules/.
|
||||||
|
# Triggers on tag push (vX.Y.Z) — builds the module, packs the
|
||||||
|
# .fai bundle, signs the bundle bytes with the Flemming.AI
|
||||||
|
# signing key (ECDSA P-256), and attaches bundle + .sig sidecar
|
||||||
|
# to the matching Forgejo Release.
|
||||||
|
#
|
||||||
|
# The hub-side verifier ships in 0.12.0+: when an operator sets
|
||||||
|
# `require_signatures: true`, install downloads the .sig sidecar
|
||||||
|
# from `<wasm_url>.sig` and verifies it against the pinned store
|
||||||
|
# public key (built-in for `bundled` / `official`, per-store
|
||||||
|
# config for everything else).
|
||||||
|
#
|
||||||
|
# Adopt by copying this file to `.forgejo/workflows/sign.yml` in
|
||||||
|
# the module repo and replacing the MODULE_NAME placeholder.
|
||||||
|
# Add the signing key to Forgejo secrets as FAI_SIGNING_KEY (PEM,
|
||||||
|
# unencrypted, ECDSA P-256). Rotation is a 5-minute key-roll +
|
||||||
|
# binary re-release; see infra/cosign/README.md.
|
||||||
|
|
||||||
|
name: sign-bundle
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
tags:
|
||||||
|
- 'v*.*.*'
|
||||||
|
|
||||||
|
env:
|
||||||
|
CARGO_TERM_COLOR: always
|
||||||
|
RUST_TOOLCHAIN: "1.86"
|
||||||
|
CARGO_NET_GIT_FETCH_WITH_CLI: "true"
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
sign:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
permissions:
|
||||||
|
contents: write # attach signature to release
|
||||||
|
|
||||||
|
env:
|
||||||
|
# The canonical module name as written in module.yaml.
|
||||||
|
# Used in the bundle filename + wasm path.
|
||||||
|
MODULE_NAME: text-extract
|
||||||
|
|
||||||
|
steps:
|
||||||
|
# See fai/platform CI for background on the manual external
|
||||||
|
# checkout: the DinD runner cannot resolve forgejo:3000, so
|
||||||
|
# we clone via the external URL with $GITHUB_TOKEN.
|
||||||
|
- name: Checkout via external URL
|
||||||
|
run: |
|
||||||
|
set -eu
|
||||||
|
mkdir -p "$GITHUB_WORKSPACE"
|
||||||
|
cd "$GITHUB_WORKSPACE"
|
||||||
|
git init -q
|
||||||
|
git remote add origin \
|
||||||
|
"https://x-access-token:${GITHUB_TOKEN}@git.flemming.ai/${GITHUB_REPOSITORY}.git"
|
||||||
|
git fetch --depth=1 origin "$GITHUB_SHA"
|
||||||
|
git checkout -q FETCH_HEAD
|
||||||
|
|
||||||
|
# The chain-module-sdk dependency lives in another Forgejo org
|
||||||
|
# behind REQUIRE_SIGNIN_VIEW. MODULE_SDK_PAT (org-level
|
||||||
|
# secret, read-only on fai/module-sdk) authenticates cargo's
|
||||||
|
# git fetch via insteadOf.
|
||||||
|
- name: Configure git URL rewrite for SDK fetch
|
||||||
|
env:
|
||||||
|
SDK_PAT: ${{ secrets.MODULE_SDK_PAT }}
|
||||||
|
run: |
|
||||||
|
git config --global \
|
||||||
|
"url.https://x-access-token:${SDK_PAT}@git.flemming.ai/.insteadOf" \
|
||||||
|
"https://git.flemming.ai/"
|
||||||
|
|
||||||
|
- name: Install system dependencies
|
||||||
|
run: |
|
||||||
|
apt-get update -qq
|
||||||
|
apt-get install -y --no-install-recommends \
|
||||||
|
curl \
|
||||||
|
ca-certificates \
|
||||||
|
build-essential \
|
||||||
|
pkg-config \
|
||||||
|
libssl-dev \
|
||||||
|
git \
|
||||||
|
jq \
|
||||||
|
openssl
|
||||||
|
|
||||||
|
- name: Install Rust toolchain
|
||||||
|
run: |
|
||||||
|
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \
|
||||||
|
| sh -s -- -y \
|
||||||
|
--profile minimal \
|
||||||
|
--default-toolchain "$RUST_TOOLCHAIN" \
|
||||||
|
--target wasm32-wasip2
|
||||||
|
echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
|
||||||
|
|
||||||
|
- name: Build module (wasm32-wasip2)
|
||||||
|
run: cargo build --release --target wasm32-wasip2
|
||||||
|
|
||||||
|
- name: Install fai CLI
|
||||||
|
# Pinned to the production channel; signing requires the
|
||||||
|
# `fai pack` command. Override FAI_VERSION when bundle
|
||||||
|
# format compatibility matters.
|
||||||
|
run: |
|
||||||
|
curl -fsSL https://get.chain.flemming.ai | sh -s -- --no-bootstrap
|
||||||
|
# The installer drops fai at $HOME/.local/bin
|
||||||
|
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
|
||||||
|
export PATH="$HOME/.local/bin:$PATH"
|
||||||
|
fai --version
|
||||||
|
|
||||||
|
- name: Stage bundle inputs
|
||||||
|
# The wasm artefact is named after the Cargo package
|
||||||
|
# (underscored), which may differ from the module name
|
||||||
|
# in module.yaml (dashed). Resolve it from Cargo.toml so
|
||||||
|
# the template stays agnostic to the project's naming
|
||||||
|
# convention.
|
||||||
|
run: |
|
||||||
|
set -eu
|
||||||
|
CARGO_NAME=$(grep -E '^name *= *"' Cargo.toml \
|
||||||
|
| head -1 | sed 's/.*"\(.*\)".*/\1/')
|
||||||
|
mkdir -p staging
|
||||||
|
cp module.yaml staging/
|
||||||
|
cp "target/wasm32-wasip2/release/${CARGO_NAME}.wasm" staging/module.wasm
|
||||||
|
if [ -f sbom.cdx.json ]; then cp sbom.cdx.json staging/; fi
|
||||||
|
if [ -f MODULE.md ]; then cp MODULE.md staging/; fi
|
||||||
|
if [ -f MODULE.de.md ]; then cp MODULE.de.md staging/; fi
|
||||||
|
|
||||||
|
- name: Pack bundle
|
||||||
|
id: pack
|
||||||
|
run: |
|
||||||
|
export PATH="$HOME/.local/bin:$PATH"
|
||||||
|
BUNDLE="${{ env.MODULE_NAME }}-${{ github.ref_name }}.fai"
|
||||||
|
fai pack staging --output "$BUNDLE"
|
||||||
|
echo "bundle=$BUNDLE" >> "$GITHUB_OUTPUT"
|
||||||
|
ls -la "$BUNDLE"
|
||||||
|
|
||||||
|
- name: Sign bundle (Flemming.AI signing key)
|
||||||
|
env:
|
||||||
|
FAI_SIGNING_KEY_PEM: ${{ secrets.FAI_SIGNING_KEY }}
|
||||||
|
run: |
|
||||||
|
if [ -z "${FAI_SIGNING_KEY_PEM:-}" ]; then
|
||||||
|
cat >&2 <<'EOF'
|
||||||
|
|
||||||
|
FAI_SIGNING_KEY secret is empty or undefined.
|
||||||
|
|
||||||
|
This workflow signs module bundles with the Flemming.AI
|
||||||
|
module-signing key (ECDSA P-256, PKCS#8 PEM, unencrypted).
|
||||||
|
The matching public key is pinned in the fai-hub binary at
|
||||||
|
infra/cosign/official.pub — hubs that require_signatures
|
||||||
|
will reject anything not signed with this key.
|
||||||
|
|
||||||
|
To configure:
|
||||||
|
1. Forgejo -> chain-modules org -> Settings -> Secrets
|
||||||
|
2. Add secret FAI_SIGNING_KEY (org-level recommended)
|
||||||
|
3. Paste the unencrypted PEM (BEGIN PRIVATE KEY ...)
|
||||||
|
The encrypted source-of-truth lives on Stefan's Mac at
|
||||||
|
~/.fai-secrets/flemming-ai-signing.key (passphrase in
|
||||||
|
mSecure under "Ch∆In — Module Signing Key (Private)").
|
||||||
|
|
||||||
|
See fai/platform infra/cosign/OPERATOR-HANDOFF.md for the
|
||||||
|
full setup runbook and rotation process.
|
||||||
|
EOF
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
umask 077
|
||||||
|
echo "$FAI_SIGNING_KEY_PEM" > /tmp/signing-key.pem
|
||||||
|
openssl dgst -sha256 \
|
||||||
|
-sign /tmp/signing-key.pem \
|
||||||
|
-out "${{ steps.pack.outputs.bundle }}.sig.raw" \
|
||||||
|
"${{ steps.pack.outputs.bundle }}"
|
||||||
|
base64 < "${{ steps.pack.outputs.bundle }}.sig.raw" \
|
||||||
|
> "${{ steps.pack.outputs.bundle }}.sig"
|
||||||
|
rm "${{ steps.pack.outputs.bundle }}.sig.raw"
|
||||||
|
shred -u /tmp/signing-key.pem || rm -f /tmp/signing-key.pem
|
||||||
|
ls -la "${{ steps.pack.outputs.bundle }}.sig"
|
||||||
|
|
||||||
|
- name: Round-trip verify
|
||||||
|
# Sanity-check: the freshly-signed bundle must verify
|
||||||
|
# against the well-known public key before we publish it.
|
||||||
|
run: |
|
||||||
|
curl -fsSL https://git.flemming.ai/fai/platform/raw/branch/main/infra/cosign/official.pub \
|
||||||
|
-o /tmp/official.pub
|
||||||
|
openssl dgst -sha256 \
|
||||||
|
-verify /tmp/official.pub \
|
||||||
|
-signature <(base64 -d < "${{ steps.pack.outputs.bundle }}.sig") \
|
||||||
|
"${{ steps.pack.outputs.bundle }}"
|
||||||
|
|
||||||
|
- name: Attach bundle + signature to Forgejo Release
|
||||||
|
run: |
|
||||||
|
set -eu
|
||||||
|
BUNDLE="${{ steps.pack.outputs.bundle }}"
|
||||||
|
SIG="$BUNDLE.sig"
|
||||||
|
REL_JSON=$(curl -sS \
|
||||||
|
-H "Authorization: token ${GITHUB_TOKEN}" \
|
||||||
|
"https://git.flemming.ai/api/v1/repos/${GITHUB_REPOSITORY}/releases/tags/${GITHUB_REF_NAME}")
|
||||||
|
REL_ID=$(echo "$REL_JSON" | jq -r '.id // empty')
|
||||||
|
if [ -z "$REL_ID" ] || [ "$REL_ID" = "null" ]; then
|
||||||
|
REL_ID=$(curl -sS -X POST \
|
||||||
|
-H "Authorization: token ${GITHUB_TOKEN}" \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
"https://git.flemming.ai/api/v1/repos/${GITHUB_REPOSITORY}/releases" \
|
||||||
|
-d "{\"tag_name\":\"${GITHUB_REF_NAME}\",\"name\":\"${GITHUB_REF_NAME}\",\"draft\":false,\"prerelease\":false,\"body\":\"Auto-generated by sign-bundle workflow\"}" \
|
||||||
|
| jq -r '.id')
|
||||||
|
fi
|
||||||
|
upload_one() {
|
||||||
|
local asset_path="$1" asset
|
||||||
|
asset=$(basename "$asset_path")
|
||||||
|
EXISTING=$(curl -sS \
|
||||||
|
-H "Authorization: token ${GITHUB_TOKEN}" \
|
||||||
|
"https://git.flemming.ai/api/v1/repos/${GITHUB_REPOSITORY}/releases/${REL_ID}/assets" \
|
||||||
|
| jq -r ".[] | select(.name==\"${asset}\") | .id")
|
||||||
|
if [ -n "$EXISTING" ]; then
|
||||||
|
curl -sS -X DELETE \
|
||||||
|
-H "Authorization: token ${GITHUB_TOKEN}" \
|
||||||
|
"https://git.flemming.ai/api/v1/repos/${GITHUB_REPOSITORY}/releases/${REL_ID}/assets/${EXISTING}" \
|
||||||
|
|| true
|
||||||
|
fi
|
||||||
|
curl -sS -X POST \
|
||||||
|
-H "Authorization: token ${GITHUB_TOKEN}" \
|
||||||
|
-F "attachment=@${asset_path}" \
|
||||||
|
"https://git.flemming.ai/api/v1/repos/${GITHUB_REPOSITORY}/releases/${REL_ID}/assets?name=${asset}" \
|
||||||
|
> /dev/null
|
||||||
|
echo "uploaded $asset"
|
||||||
|
}
|
||||||
|
upload_one "$BUNDLE"
|
||||||
|
upload_one "$SIG"
|
||||||
|
echo "release ${GITHUB_REF_NAME} ready: https://git.flemming.ai/${GITHUB_REPOSITORY}/releases/tag/${GITHUB_REF_NAME}"
|
||||||
46
Cargo.lock
generated
46
Cargo.lock
generated
|
|
@ -77,6 +77,28 @@ version = "1.0.4"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
|
checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "chain-module-sdk"
|
||||||
|
version = "0.1.2"
|
||||||
|
source = "git+https://git.flemming.ai/fai/chain-module-sdk-rust.git?branch=main#98e98e9371f7409560a1ef08bc0923d9a2506449"
|
||||||
|
dependencies = [
|
||||||
|
"chain-module-sdk-macros",
|
||||||
|
"serde",
|
||||||
|
"serde_json",
|
||||||
|
"thiserror",
|
||||||
|
"wit-bindgen",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "chain-module-sdk-macros"
|
||||||
|
version = "0.1.2"
|
||||||
|
source = "git+https://git.flemming.ai/fai/chain-module-sdk-rust.git?branch=main#98e98e9371f7409560a1ef08bc0923d9a2506449"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "crc32fast"
|
name = "crc32fast"
|
||||||
version = "1.5.0"
|
version = "1.5.0"
|
||||||
|
|
@ -167,28 +189,6 @@ dependencies = [
|
||||||
"num-traits",
|
"num-traits",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "fai-module-sdk"
|
|
||||||
version = "0.1.2"
|
|
||||||
source = "git+https://git.flemming.ws/fai/module-sdk.git?branch=main#f209fbc86f531b53e576f3cdbdf75750eef3b07a"
|
|
||||||
dependencies = [
|
|
||||||
"fai-module-sdk-macros",
|
|
||||||
"serde",
|
|
||||||
"serde_json",
|
|
||||||
"thiserror",
|
|
||||||
"wit-bindgen",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "fai-module-sdk-macros"
|
|
||||||
version = "0.1.2"
|
|
||||||
source = "git+https://git.flemming.ws/fai/module-sdk.git?branch=main#f209fbc86f531b53e576f3cdbdf75750eef3b07a"
|
|
||||||
dependencies = [
|
|
||||||
"proc-macro2",
|
|
||||||
"quote",
|
|
||||||
"syn",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "flate2"
|
name = "flate2"
|
||||||
version = "1.1.9"
|
version = "1.1.9"
|
||||||
|
|
@ -508,7 +508,7 @@ dependencies = [
|
||||||
name = "text_extract"
|
name = "text_extract"
|
||||||
version = "0.1.5"
|
version = "0.1.5"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"fai-module-sdk",
|
"chain-module-sdk",
|
||||||
"pdf-extract",
|
"pdf-extract",
|
||||||
"quick-xml",
|
"quick-xml",
|
||||||
"serde",
|
"serde",
|
||||||
|
|
|
||||||
|
|
@ -7,18 +7,18 @@
|
||||||
name = "text_extract"
|
name = "text_extract"
|
||||||
version = "0.1.5"
|
version = "0.1.5"
|
||||||
edition = "2024"
|
edition = "2024"
|
||||||
authors = ["Dr. Stefan Flemming <platform@flemming.ai>"]
|
authors = ["Dr. Stefan Flemming <chain@flemming.ai>"]
|
||||||
license = "Apache-2.0"
|
license = "Apache-2.0"
|
||||||
publish = false
|
publish = false
|
||||||
description = "F∆I module — extract plain text from PDF and DOCX documents"
|
description = "F∆I module — extract plain text from PDF and DOCX documents"
|
||||||
repository = "https://git.flemming.ws/fai-modules/text-extract"
|
repository = "https://git.flemming.ws/chain-modules/text-extract"
|
||||||
rust-version = "1.85"
|
rust-version = "1.85"
|
||||||
|
|
||||||
[lib]
|
[lib]
|
||||||
crate-type = ["cdylib", "rlib"]
|
crate-type = ["cdylib", "rlib"]
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
fai-module-sdk = { git = "https://git.flemming.ws/fai/module-sdk.git", branch = "main" }
|
chain-module-sdk = { git = "https://git.flemming.ai/fai/chain-module-sdk-rust.git", branch = "main" }
|
||||||
serde = { version = "1", features = ["derive"] }
|
serde = { version = "1", features = ["derive"] }
|
||||||
pdf-extract = "0.7"
|
pdf-extract = "0.7"
|
||||||
zip = { version = "2", default-features = false, features = ["deflate"] }
|
zip = { version = "2", default-features = false, features = ["deflate"] }
|
||||||
|
|
|
||||||
2
LICENSE
2
LICENSE
|
|
@ -58,7 +58,7 @@ APPENDIX: How to apply the Apache License to your work.
|
||||||
|
|
||||||
To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives.
|
To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives.
|
||||||
|
|
||||||
Copyright 2026 fai-modules
|
Copyright 2026 Flemming.AI
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
you may not use this file except in compliance with the License.
|
you may not use this file except in compliance with the License.
|
||||||
|
|
|
||||||
96
MODULE.de.md
Normal file
96
MODULE.de.md
Normal file
|
|
@ -0,0 +1,96 @@
|
||||||
|
# text.extract
|
||||||
|
|
||||||
|
Reine Textextraktion aus PDF- und DOCX-Dokumenten. Pure-Rust,
|
||||||
|
in-WASM, ohne deklarierte Berechtigungen. Der kanonische erste
|
||||||
|
Schritt in Flows, die Rohtext brauchen — anonymisieren,
|
||||||
|
zusammenfassen, übersetzen, per LLM klassifizieren,
|
||||||
|
Audit-redigieren.
|
||||||
|
|
||||||
|
## Capability
|
||||||
|
|
||||||
|
- `text.extract@0.1.0`
|
||||||
|
|
||||||
|
## Eingaben
|
||||||
|
|
||||||
|
| Name | Typ | Beschreibung |
|
||||||
|
| ---------- | ----- | ------------------------------------------------------- |
|
||||||
|
| `document` | bytes | Das zu extrahierende Dokument. MIME-Typen siehe unten. |
|
||||||
|
|
||||||
|
Akzeptierte MIME-Typen (deklariert in
|
||||||
|
`module.yaml#accepts_mime`, von Studio als File-Picker-Filter
|
||||||
|
gelesen):
|
||||||
|
|
||||||
|
- `application/pdf`
|
||||||
|
- `application/x-pdf`
|
||||||
|
- `application/vnd.openxmlformats-officedocument.wordprocessingml.document`
|
||||||
|
|
||||||
|
Andere MIME-Typen werden mit `invalid_input` abgelehnt — vor
|
||||||
|
jedem Parsen.
|
||||||
|
|
||||||
|
## Ausgabe
|
||||||
|
|
||||||
|
| Name | Typ | Beschreibung |
|
||||||
|
| ----------- | ---- | --------------------------------------------- |
|
||||||
|
| `extracted` | json | `ExtractedDocument`-Schema siehe unten. |
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"engine": "pdf-extract",
|
||||||
|
"engine_version": "0.7",
|
||||||
|
"pages": [
|
||||||
|
{ "number": 1, "text": "…" }
|
||||||
|
]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
`confidence` und `bbox` pro Seite sind im Schema für eine
|
||||||
|
spätere Service-Mode-Variante (pdfium / mupdf via
|
||||||
|
Host-Services) reserviert. v0.1.0 setzt diese Felder nicht;
|
||||||
|
das Schema akzeptiert sie als optional.
|
||||||
|
|
||||||
|
## Berechtigungen
|
||||||
|
|
||||||
|
Keine. Reine Rust-Pipeline (`pdf-extract`, `zip`, `quick-xml`).
|
||||||
|
Kein Dateisystem, kein Netz. `permissions: []` in
|
||||||
|
`module.yaml` macht das explizit; der Hub erzwingt es.
|
||||||
|
|
||||||
|
## Grenzen in v0.1.0
|
||||||
|
|
||||||
|
- **Layout-naives PDF.** Mehrspaltige oder tabellenlastige
|
||||||
|
Dokumente können umsortierten Text liefern. Ein-Spalten-
|
||||||
|
Fließtext ist solide.
|
||||||
|
- **Kein OCR.** Gescannte (nur-Bild-)PDFs liefern pro Seite
|
||||||
|
leeren Text. Künftiges `text.ocr@^0` als Vor-Schritt.
|
||||||
|
- **Nur DOCX bei Office.** `.doc` (Altformat), `.rtf`, `.odt`
|
||||||
|
noch nicht unterstützt.
|
||||||
|
|
||||||
|
`1.0.0` setzt entweder die Service-Mode-Variante
|
||||||
|
(pdfium-Qualität + Layout) voraus oder einen
|
||||||
|
nachgewiesen-guten Extraktions-Score auf einem
|
||||||
|
repräsentativen Korpus.
|
||||||
|
|
||||||
|
## Beispiel-Flow
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
name: extract-anonymise
|
||||||
|
inputs:
|
||||||
|
document: bytes
|
||||||
|
steps:
|
||||||
|
- id: extract
|
||||||
|
use: text.extract@^0
|
||||||
|
with:
|
||||||
|
document: $inputs.document
|
||||||
|
- id: redact
|
||||||
|
use: text.anonymize@^0
|
||||||
|
with:
|
||||||
|
text: $extract.extracted.pages[*].text
|
||||||
|
outputs:
|
||||||
|
redacted: $redact.anonymized
|
||||||
|
```
|
||||||
|
|
||||||
|
## Build
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cargo build --release --target wasm32-wasip2
|
||||||
|
# Ausgabe: target/wasm32-wasip2/release/text_extract.wasm
|
||||||
|
```
|
||||||
92
MODULE.md
Normal file
92
MODULE.md
Normal file
|
|
@ -0,0 +1,92 @@
|
||||||
|
# text.extract
|
||||||
|
|
||||||
|
Plain-text extraction from PDF and DOCX documents. Pure-Rust,
|
||||||
|
in-WASM, no permissions required. The canonical first step in
|
||||||
|
any flow that needs raw text — anonymise, summarise, translate,
|
||||||
|
LLM-classify, audit-redact.
|
||||||
|
|
||||||
|
## Capability
|
||||||
|
|
||||||
|
- `text.extract@0.1.0`
|
||||||
|
|
||||||
|
## Inputs
|
||||||
|
|
||||||
|
| Name | Type | Description |
|
||||||
|
| ---------- | ----- | ------------------------------------------------------ |
|
||||||
|
| `document` | bytes | The document to extract. Accepted MIME types below. |
|
||||||
|
|
||||||
|
Accepted MIME types (declared in `module.yaml#accepts_mime`,
|
||||||
|
read by Studio as the file-picker filter):
|
||||||
|
|
||||||
|
- `application/pdf`
|
||||||
|
- `application/x-pdf`
|
||||||
|
- `application/vnd.openxmlformats-officedocument.wordprocessingml.document`
|
||||||
|
|
||||||
|
Other MIME types are rejected with a `invalid_input` error
|
||||||
|
before any parsing happens.
|
||||||
|
|
||||||
|
## Output
|
||||||
|
|
||||||
|
| Name | Type | Description |
|
||||||
|
| ----------- | ---- | ------------------------------------------------- |
|
||||||
|
| `extracted` | json | `ExtractedDocument` schema below. |
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"engine": "pdf-extract",
|
||||||
|
"engine_version": "0.7",
|
||||||
|
"pages": [
|
||||||
|
{ "number": 1, "text": "…" }
|
||||||
|
]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Per-page `confidence` and `bbox` fields are reserved in the
|
||||||
|
schema for a future service-mode variant (pdfium / mupdf via
|
||||||
|
host services) that surfaces layout information. v0.1.0 does
|
||||||
|
not populate them; the schema accepts them as optional.
|
||||||
|
|
||||||
|
## Permissions
|
||||||
|
|
||||||
|
None. Pure-Rust pipeline (`pdf-extract`, `zip`, `quick-xml`).
|
||||||
|
No filesystem, no network. The empty `permissions: []` in
|
||||||
|
`module.yaml` makes that explicit; the hub enforces it.
|
||||||
|
|
||||||
|
## Limits in v0.1.0
|
||||||
|
|
||||||
|
- **Layout-naive PDF.** Multi-column or table-heavy documents
|
||||||
|
may produce reordered text. Single-column body text is solid.
|
||||||
|
- **No OCR.** Scanned (image-only) PDFs return empty text per
|
||||||
|
page. Pair with a future `text.ocr@^0` step for those.
|
||||||
|
- **DOCX-only Office.** `.doc` (legacy binary), `.rtf`, `.odt`
|
||||||
|
are not yet supported.
|
||||||
|
|
||||||
|
Bumping to `1.0.0` requires either the service-mode variant
|
||||||
|
(pdfium-quality output + layout) or a confirmed-good extraction
|
||||||
|
score on a representative corpus.
|
||||||
|
|
||||||
|
## Example flow
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
name: extract-anonymise
|
||||||
|
inputs:
|
||||||
|
document: bytes
|
||||||
|
steps:
|
||||||
|
- id: extract
|
||||||
|
use: text.extract@^0
|
||||||
|
with:
|
||||||
|
document: $inputs.document
|
||||||
|
- id: redact
|
||||||
|
use: text.anonymize@^0
|
||||||
|
with:
|
||||||
|
text: $extract.extracted.pages[*].text
|
||||||
|
outputs:
|
||||||
|
redacted: $redact.anonymized
|
||||||
|
```
|
||||||
|
|
||||||
|
## Build
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cargo build --release --target wasm32-wasip2
|
||||||
|
# Output: target/wasm32-wasip2/release/text_extract.wasm
|
||||||
|
```
|
||||||
|
|
@ -45,7 +45,7 @@ The build depends on a sibling checkout of `fai/module-sdk`:
|
||||||
└── text-extract/ # this repo
|
└── text-extract/ # this repo
|
||||||
```
|
```
|
||||||
|
|
||||||
This is a temporary path-based dependency. Once `fai-module-sdk`
|
This is a temporary path-based dependency. Once `chain-module-sdk`
|
||||||
is published to a registry or a stable git tag, this module's
|
is published to a registry or a stable git tag, this module's
|
||||||
`Cargo.toml` will switch over. The Forgejo CI workflow checks out
|
`Cargo.toml` will switch over. The Forgejo CI workflow checks out
|
||||||
the SDK repo into a sibling directory before building.
|
the SDK repo into a sibling directory before building.
|
||||||
|
|
@ -81,5 +81,5 @@ integration test.
|
||||||
|
|
||||||
Apache-2.0. See `LICENSE`.
|
Apache-2.0. See `LICENSE`.
|
||||||
|
|
||||||
Author: Dr. Stefan Flemming, Flemming.AI <platform@flemming.ai>
|
Author: Dr. Stefan Flemming, Flemming.AI <chain@flemming.ai>
|
||||||
Repository: https://git.flemming.ws/fai-modules/text-extract
|
Repository: https://git.flemming.ws/chain-modules/text-extract
|
||||||
|
|
|
||||||
BIN
module.wasm
Normal file
BIN
module.wasm
Normal file
Binary file not shown.
49
module.yaml
49
module.yaml
|
|
@ -1,4 +1,5 @@
|
||||||
schema_version: 1
|
schema_version: 3
|
||||||
|
provider: chain
|
||||||
name: text-extract
|
name: text-extract
|
||||||
version: 0.1.0
|
version: 0.1.0
|
||||||
|
|
||||||
|
|
@ -9,19 +10,35 @@ provides:
|
||||||
|
|
||||||
# Inputs the invoke function accepts.
|
# Inputs the invoke function accepts.
|
||||||
inputs:
|
inputs:
|
||||||
# The document to extract text from. Supported MIME types:
|
document:
|
||||||
# application/pdf
|
type: bytes
|
||||||
# application/x-pdf
|
description:
|
||||||
# application/vnd.openxmlformats-officedocument.wordprocessingml.document
|
en: |
|
||||||
document: bytes
|
The document to extract text from. Supported MIME types:
|
||||||
|
application/pdf
|
||||||
|
application/x-pdf
|
||||||
|
application/vnd.openxmlformats-officedocument.wordprocessingml.document
|
||||||
|
de: |
|
||||||
|
Das Dokument, aus dem Text extrahiert werden soll. Unterstützte MIME-Typen:
|
||||||
|
application/pdf
|
||||||
|
application/x-pdf
|
||||||
|
application/vnd.openxmlformats-officedocument.wordprocessingml.document
|
||||||
|
|
||||||
# Outputs produced.
|
# Outputs produced.
|
||||||
outputs:
|
outputs:
|
||||||
# JSON-encoded ExtractedDocument:
|
extracted:
|
||||||
# { engine, engine_version, pages: [{number, text, confidence?, bbox?}] }
|
type: json
|
||||||
# confidence and bbox are reserved for a future service-mode variant
|
description:
|
||||||
# (pdfium / mupdf via host services). v0.1.0 omits them.
|
en: |
|
||||||
extracted: json
|
JSON-encoded ExtractedDocument:
|
||||||
|
{ engine, engine_version, pages: [{number, text, confidence?, bbox?}] }
|
||||||
|
confidence and bbox are reserved for a future service-mode variant
|
||||||
|
(pdfium / mupdf via host services). v0.1.0 omits them.
|
||||||
|
de: |
|
||||||
|
JSON-codiertes ExtractedDocument:
|
||||||
|
{ engine, engine_version, pages: [{number, text, confidence?, bbox?}] }
|
||||||
|
confidence und bbox sind für eine künftige Service-Mode-Variante reserviert
|
||||||
|
(pdfium / mupdf über Host-Services). v0.1.0 liefert sie nicht.
|
||||||
|
|
||||||
# Permissions required.
|
# Permissions required.
|
||||||
#
|
#
|
||||||
|
|
@ -29,3 +46,13 @@ outputs:
|
||||||
# zip, quick-xml). No filesystem, no network. The empty list makes
|
# zip, quick-xml). No filesystem, no network. The empty list makes
|
||||||
# this explicit; the hub enforces it.
|
# this explicit; the hub enforces it.
|
||||||
permissions: []
|
permissions: []
|
||||||
|
|
||||||
|
# MIME-type allow-list for the `document` input. Studio reads this
|
||||||
|
# through ModuleInfoResponse.accepts_mime and uses it as the OS
|
||||||
|
# file-picker filter so operators don't accidentally pick a `.png`
|
||||||
|
# for a PDF-extract step. Same list documented inline next to
|
||||||
|
# `inputs.document` above; kept synchronised by hand for now.
|
||||||
|
accepts_mime:
|
||||||
|
- application/pdf
|
||||||
|
- application/x-pdf
|
||||||
|
- application/vnd.openxmlformats-officedocument.wordprocessingml.document
|
||||||
|
|
|
||||||
|
|
@ -39,7 +39,7 @@
|
||||||
|
|
||||||
#![allow(clippy::result_large_err)]
|
#![allow(clippy::result_large_err)]
|
||||||
|
|
||||||
use fai_module_sdk::prelude::*;
|
use chain_module_sdk::prelude::*;
|
||||||
use serde::Serialize;
|
use serde::Serialize;
|
||||||
|
|
||||||
const PDF_MIME: &str = "application/pdf";
|
const PDF_MIME: &str = "application/pdf";
|
||||||
|
|
|
||||||
|
|
@ -16,7 +16,7 @@
|
||||||
|
|
||||||
use std::io::Write;
|
use std::io::Write;
|
||||||
|
|
||||||
use fai_module_sdk::prelude::*;
|
use chain_module_sdk::prelude::*;
|
||||||
use serde::Deserialize;
|
use serde::Deserialize;
|
||||||
use text_extract::invoke;
|
use text_extract::invoke;
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue