diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index 89d5a1d..56ea4e9 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -30,7 +30,7 @@ jobs: git fetch --depth=1 origin "$GITHUB_SHA" git checkout -q FETCH_HEAD - # The fai-module-sdk dependency is a git dep against a repo + # The chain-module-sdk dependency is a git dep against a repo # that lives in another Forgejo org. The repo-scoped # GITHUB_TOKEN cannot read it, and the Forgejo instance has # REQUIRE_SIGNIN_VIEW=true so anonymous read is also denied. diff --git a/.forgejo/workflows/sign.yml b/.forgejo/workflows/sign.yml new file mode 100644 index 0000000..e0cf27f --- /dev/null +++ b/.forgejo/workflows/sign.yml @@ -0,0 +1,223 @@ +# F∆I module bundle signing workflow (cosign key-pinning model). +# +# Drop-in template for any module repo under chain-modules/. +# Triggers on tag push (vX.Y.Z) — builds the module, packs the +# .fai bundle, signs the bundle bytes with the Flemming.AI +# signing key (ECDSA P-256), and attaches bundle + .sig sidecar +# to the matching Forgejo Release. +# +# The hub-side verifier ships in 0.12.0+: when an operator sets +# `require_signatures: true`, install downloads the .sig sidecar +# from `.sig` and verifies it against the pinned store +# public key (built-in for `bundled` / `official`, per-store +# config for everything else). +# +# Adopt by copying this file to `.forgejo/workflows/sign.yml` in +# the module repo and replacing the MODULE_NAME placeholder. +# Add the signing key to Forgejo secrets as FAI_SIGNING_KEY (PEM, +# unencrypted, ECDSA P-256). Rotation is a 5-minute key-roll + +# binary re-release; see infra/cosign/README.md. + +name: sign-bundle + +on: + push: + tags: + - 'v*.*.*' + +env: + CARGO_TERM_COLOR: always + RUST_TOOLCHAIN: "1.86" + CARGO_NET_GIT_FETCH_WITH_CLI: "true" + +jobs: + sign: + runs-on: ubuntu-latest + permissions: + contents: write # attach signature to release + + env: + # The canonical module name as written in module.yaml. + # Used in the bundle filename + wasm path. + MODULE_NAME: text-extract + + steps: + # See fai/platform CI for background on the manual external + # checkout: the DinD runner cannot resolve forgejo:3000, so + # we clone via the external URL with $GITHUB_TOKEN. + - name: Checkout via external URL + run: | + set -eu + mkdir -p "$GITHUB_WORKSPACE" + cd "$GITHUB_WORKSPACE" + git init -q + git remote add origin \ + "https://x-access-token:${GITHUB_TOKEN}@git.flemming.ai/${GITHUB_REPOSITORY}.git" + git fetch --depth=1 origin "$GITHUB_SHA" + git checkout -q FETCH_HEAD + + # The chain-module-sdk dependency lives in another Forgejo org + # behind REQUIRE_SIGNIN_VIEW. MODULE_SDK_PAT (org-level + # secret, read-only on fai/module-sdk) authenticates cargo's + # git fetch via insteadOf. + - name: Configure git URL rewrite for SDK fetch + env: + SDK_PAT: ${{ secrets.MODULE_SDK_PAT }} + run: | + git config --global \ + "url.https://x-access-token:${SDK_PAT}@git.flemming.ai/.insteadOf" \ + "https://git.flemming.ai/" + + - name: Install system dependencies + run: | + apt-get update -qq + apt-get install -y --no-install-recommends \ + curl \ + ca-certificates \ + build-essential \ + pkg-config \ + libssl-dev \ + git \ + jq \ + openssl + + - name: Install Rust toolchain + run: | + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \ + | sh -s -- -y \ + --profile minimal \ + --default-toolchain "$RUST_TOOLCHAIN" \ + --target wasm32-wasip2 + echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" + + - name: Build module (wasm32-wasip2) + run: cargo build --release --target wasm32-wasip2 + + - name: Install fai CLI + # Pinned to the production channel; signing requires the + # `fai pack` command. Override FAI_VERSION when bundle + # format compatibility matters. + run: | + curl -fsSL https://get.chain.flemming.ai | sh -s -- --no-bootstrap + # The installer drops fai at $HOME/.local/bin + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + export PATH="$HOME/.local/bin:$PATH" + fai --version + + - name: Stage bundle inputs + # The wasm artefact is named after the Cargo package + # (underscored), which may differ from the module name + # in module.yaml (dashed). Resolve it from Cargo.toml so + # the template stays agnostic to the project's naming + # convention. + run: | + set -eu + CARGO_NAME=$(grep -E '^name *= *"' Cargo.toml \ + | head -1 | sed 's/.*"\(.*\)".*/\1/') + mkdir -p staging + cp module.yaml staging/ + cp "target/wasm32-wasip2/release/${CARGO_NAME}.wasm" staging/module.wasm + if [ -f sbom.cdx.json ]; then cp sbom.cdx.json staging/; fi + if [ -f MODULE.md ]; then cp MODULE.md staging/; fi + if [ -f MODULE.de.md ]; then cp MODULE.de.md staging/; fi + + - name: Pack bundle + id: pack + run: | + export PATH="$HOME/.local/bin:$PATH" + BUNDLE="${{ env.MODULE_NAME }}-${{ github.ref_name }}.fai" + fai pack staging --output "$BUNDLE" + echo "bundle=$BUNDLE" >> "$GITHUB_OUTPUT" + ls -la "$BUNDLE" + + - name: Sign bundle (Flemming.AI signing key) + env: + FAI_SIGNING_KEY_PEM: ${{ secrets.FAI_SIGNING_KEY }} + run: | + if [ -z "${FAI_SIGNING_KEY_PEM:-}" ]; then + cat >&2 <<'EOF' + + FAI_SIGNING_KEY secret is empty or undefined. + + This workflow signs module bundles with the Flemming.AI + module-signing key (ECDSA P-256, PKCS#8 PEM, unencrypted). + The matching public key is pinned in the fai-hub binary at + infra/cosign/official.pub — hubs that require_signatures + will reject anything not signed with this key. + + To configure: + 1. Forgejo -> chain-modules org -> Settings -> Secrets + 2. Add secret FAI_SIGNING_KEY (org-level recommended) + 3. Paste the unencrypted PEM (BEGIN PRIVATE KEY ...) + The encrypted source-of-truth lives on Stefan's Mac at + ~/.fai-secrets/flemming-ai-signing.key (passphrase in + mSecure under "Ch∆In — Module Signing Key (Private)"). + + See fai/platform infra/cosign/OPERATOR-HANDOFF.md for the + full setup runbook and rotation process. + EOF + exit 1 + fi + umask 077 + echo "$FAI_SIGNING_KEY_PEM" > /tmp/signing-key.pem + openssl dgst -sha256 \ + -sign /tmp/signing-key.pem \ + -out "${{ steps.pack.outputs.bundle }}.sig.raw" \ + "${{ steps.pack.outputs.bundle }}" + base64 < "${{ steps.pack.outputs.bundle }}.sig.raw" \ + > "${{ steps.pack.outputs.bundle }}.sig" + rm "${{ steps.pack.outputs.bundle }}.sig.raw" + shred -u /tmp/signing-key.pem || rm -f /tmp/signing-key.pem + ls -la "${{ steps.pack.outputs.bundle }}.sig" + + - name: Round-trip verify + # Sanity-check: the freshly-signed bundle must verify + # against the well-known public key before we publish it. + run: | + curl -fsSL https://git.flemming.ai/fai/platform/raw/branch/main/infra/cosign/official.pub \ + -o /tmp/official.pub + openssl dgst -sha256 \ + -verify /tmp/official.pub \ + -signature <(base64 -d < "${{ steps.pack.outputs.bundle }}.sig") \ + "${{ steps.pack.outputs.bundle }}" + + - name: Attach bundle + signature to Forgejo Release + run: | + set -eu + BUNDLE="${{ steps.pack.outputs.bundle }}" + SIG="$BUNDLE.sig" + REL_JSON=$(curl -sS \ + -H "Authorization: token ${GITHUB_TOKEN}" \ + "https://git.flemming.ai/api/v1/repos/${GITHUB_REPOSITORY}/releases/tags/${GITHUB_REF_NAME}") + REL_ID=$(echo "$REL_JSON" | jq -r '.id // empty') + if [ -z "$REL_ID" ] || [ "$REL_ID" = "null" ]; then + REL_ID=$(curl -sS -X POST \ + -H "Authorization: token ${GITHUB_TOKEN}" \ + -H "Content-Type: application/json" \ + "https://git.flemming.ai/api/v1/repos/${GITHUB_REPOSITORY}/releases" \ + -d "{\"tag_name\":\"${GITHUB_REF_NAME}\",\"name\":\"${GITHUB_REF_NAME}\",\"draft\":false,\"prerelease\":false,\"body\":\"Auto-generated by sign-bundle workflow\"}" \ + | jq -r '.id') + fi + upload_one() { + local asset_path="$1" asset + asset=$(basename "$asset_path") + EXISTING=$(curl -sS \ + -H "Authorization: token ${GITHUB_TOKEN}" \ + "https://git.flemming.ai/api/v1/repos/${GITHUB_REPOSITORY}/releases/${REL_ID}/assets" \ + | jq -r ".[] | select(.name==\"${asset}\") | .id") + if [ -n "$EXISTING" ]; then + curl -sS -X DELETE \ + -H "Authorization: token ${GITHUB_TOKEN}" \ + "https://git.flemming.ai/api/v1/repos/${GITHUB_REPOSITORY}/releases/${REL_ID}/assets/${EXISTING}" \ + || true + fi + curl -sS -X POST \ + -H "Authorization: token ${GITHUB_TOKEN}" \ + -F "attachment=@${asset_path}" \ + "https://git.flemming.ai/api/v1/repos/${GITHUB_REPOSITORY}/releases/${REL_ID}/assets?name=${asset}" \ + > /dev/null + echo "uploaded $asset" + } + upload_one "$BUNDLE" + upload_one "$SIG" + echo "release ${GITHUB_REF_NAME} ready: https://git.flemming.ai/${GITHUB_REPOSITORY}/releases/tag/${GITHUB_REF_NAME}" diff --git a/Cargo.lock b/Cargo.lock index 1e84b68..6ca7df2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -77,6 +77,28 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "chain-module-sdk" +version = "0.1.2" +source = "git+https://git.flemming.ai/fai/chain-module-sdk-rust.git?branch=main#98e98e9371f7409560a1ef08bc0923d9a2506449" +dependencies = [ + "chain-module-sdk-macros", + "serde", + "serde_json", + "thiserror", + "wit-bindgen", +] + +[[package]] +name = "chain-module-sdk-macros" +version = "0.1.2" +source = "git+https://git.flemming.ai/fai/chain-module-sdk-rust.git?branch=main#98e98e9371f7409560a1ef08bc0923d9a2506449" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -167,28 +189,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "fai-module-sdk" -version = "0.1.2" -source = "git+https://git.flemming.ws/fai/module-sdk.git?branch=main#f209fbc86f531b53e576f3cdbdf75750eef3b07a" -dependencies = [ - "fai-module-sdk-macros", - "serde", - "serde_json", - "thiserror", - "wit-bindgen", -] - -[[package]] -name = "fai-module-sdk-macros" -version = "0.1.2" -source = "git+https://git.flemming.ws/fai/module-sdk.git?branch=main#f209fbc86f531b53e576f3cdbdf75750eef3b07a" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "flate2" version = "1.1.9" @@ -508,7 +508,7 @@ dependencies = [ name = "text_extract" version = "0.1.5" dependencies = [ - "fai-module-sdk", + "chain-module-sdk", "pdf-extract", "quick-xml", "serde", diff --git a/Cargo.toml b/Cargo.toml index a658fd5..708f2eb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,18 +7,18 @@ name = "text_extract" version = "0.1.5" edition = "2024" -authors = ["Dr. Stefan Flemming "] +authors = ["Dr. Stefan Flemming "] license = "Apache-2.0" publish = false description = "F∆I module — extract plain text from PDF and DOCX documents" -repository = "https://git.flemming.ws/fai-modules/text-extract" +repository = "https://git.flemming.ws/chain-modules/text-extract" rust-version = "1.85" [lib] crate-type = ["cdylib", "rlib"] [dependencies] -fai-module-sdk = { git = "https://git.flemming.ws/fai/module-sdk.git", branch = "main" } +chain-module-sdk = { git = "https://git.flemming.ai/fai/chain-module-sdk-rust.git", branch = "main" } serde = { version = "1", features = ["derive"] } pdf-extract = "0.7" zip = { version = "2", default-features = false, features = ["deflate"] } diff --git a/LICENSE b/LICENSE index 9536d06..cdde5b2 100644 --- a/LICENSE +++ b/LICENSE @@ -58,7 +58,7 @@ APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. -Copyright 2026 fai-modules +Copyright 2026 Flemming.AI Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/MODULE.de.md b/MODULE.de.md new file mode 100644 index 0000000..bd60e14 --- /dev/null +++ b/MODULE.de.md @@ -0,0 +1,96 @@ +# text.extract + +Reine Textextraktion aus PDF- und DOCX-Dokumenten. Pure-Rust, +in-WASM, ohne deklarierte Berechtigungen. Der kanonische erste +Schritt in Flows, die Rohtext brauchen — anonymisieren, +zusammenfassen, übersetzen, per LLM klassifizieren, +Audit-redigieren. + +## Capability + +- `text.extract@0.1.0` + +## Eingaben + +| Name | Typ | Beschreibung | +| ---------- | ----- | ------------------------------------------------------- | +| `document` | bytes | Das zu extrahierende Dokument. MIME-Typen siehe unten. | + +Akzeptierte MIME-Typen (deklariert in +`module.yaml#accepts_mime`, von Studio als File-Picker-Filter +gelesen): + +- `application/pdf` +- `application/x-pdf` +- `application/vnd.openxmlformats-officedocument.wordprocessingml.document` + +Andere MIME-Typen werden mit `invalid_input` abgelehnt — vor +jedem Parsen. + +## Ausgabe + +| Name | Typ | Beschreibung | +| ----------- | ---- | --------------------------------------------- | +| `extracted` | json | `ExtractedDocument`-Schema siehe unten. | + +```json +{ + "engine": "pdf-extract", + "engine_version": "0.7", + "pages": [ + { "number": 1, "text": "…" } + ] +} +``` + +`confidence` und `bbox` pro Seite sind im Schema für eine +spätere Service-Mode-Variante (pdfium / mupdf via +Host-Services) reserviert. v0.1.0 setzt diese Felder nicht; +das Schema akzeptiert sie als optional. + +## Berechtigungen + +Keine. Reine Rust-Pipeline (`pdf-extract`, `zip`, `quick-xml`). +Kein Dateisystem, kein Netz. `permissions: []` in +`module.yaml` macht das explizit; der Hub erzwingt es. + +## Grenzen in v0.1.0 + +- **Layout-naives PDF.** Mehrspaltige oder tabellenlastige + Dokumente können umsortierten Text liefern. Ein-Spalten- + Fließtext ist solide. +- **Kein OCR.** Gescannte (nur-Bild-)PDFs liefern pro Seite + leeren Text. Künftiges `text.ocr@^0` als Vor-Schritt. +- **Nur DOCX bei Office.** `.doc` (Altformat), `.rtf`, `.odt` + noch nicht unterstützt. + +`1.0.0` setzt entweder die Service-Mode-Variante +(pdfium-Qualität + Layout) voraus oder einen +nachgewiesen-guten Extraktions-Score auf einem +repräsentativen Korpus. + +## Beispiel-Flow + +```yaml +name: extract-anonymise +inputs: + document: bytes +steps: + - id: extract + use: text.extract@^0 + with: + document: $inputs.document + - id: redact + use: text.anonymize@^0 + with: + text: $extract.extracted.pages[*].text +outputs: + redacted: $redact.anonymized +``` + +## Build + +```bash +cargo build --release --target wasm32-wasip2 +# Ausgabe: target/wasm32-wasip2/release/text_extract.wasm +``` diff --git a/MODULE.md b/MODULE.md new file mode 100644 index 0000000..5871e48 --- /dev/null +++ b/MODULE.md @@ -0,0 +1,92 @@ +# text.extract + +Plain-text extraction from PDF and DOCX documents. Pure-Rust, +in-WASM, no permissions required. The canonical first step in +any flow that needs raw text — anonymise, summarise, translate, +LLM-classify, audit-redact. + +## Capability + +- `text.extract@0.1.0` + +## Inputs + +| Name | Type | Description | +| ---------- | ----- | ------------------------------------------------------ | +| `document` | bytes | The document to extract. Accepted MIME types below. | + +Accepted MIME types (declared in `module.yaml#accepts_mime`, +read by Studio as the file-picker filter): + +- `application/pdf` +- `application/x-pdf` +- `application/vnd.openxmlformats-officedocument.wordprocessingml.document` + +Other MIME types are rejected with a `invalid_input` error +before any parsing happens. + +## Output + +| Name | Type | Description | +| ----------- | ---- | ------------------------------------------------- | +| `extracted` | json | `ExtractedDocument` schema below. | + +```json +{ + "engine": "pdf-extract", + "engine_version": "0.7", + "pages": [ + { "number": 1, "text": "…" } + ] +} +``` + +Per-page `confidence` and `bbox` fields are reserved in the +schema for a future service-mode variant (pdfium / mupdf via +host services) that surfaces layout information. v0.1.0 does +not populate them; the schema accepts them as optional. + +## Permissions + +None. Pure-Rust pipeline (`pdf-extract`, `zip`, `quick-xml`). +No filesystem, no network. The empty `permissions: []` in +`module.yaml` makes that explicit; the hub enforces it. + +## Limits in v0.1.0 + +- **Layout-naive PDF.** Multi-column or table-heavy documents + may produce reordered text. Single-column body text is solid. +- **No OCR.** Scanned (image-only) PDFs return empty text per + page. Pair with a future `text.ocr@^0` step for those. +- **DOCX-only Office.** `.doc` (legacy binary), `.rtf`, `.odt` + are not yet supported. + +Bumping to `1.0.0` requires either the service-mode variant +(pdfium-quality output + layout) or a confirmed-good extraction +score on a representative corpus. + +## Example flow + +```yaml +name: extract-anonymise +inputs: + document: bytes +steps: + - id: extract + use: text.extract@^0 + with: + document: $inputs.document + - id: redact + use: text.anonymize@^0 + with: + text: $extract.extracted.pages[*].text +outputs: + redacted: $redact.anonymized +``` + +## Build + +```bash +cargo build --release --target wasm32-wasip2 +# Output: target/wasm32-wasip2/release/text_extract.wasm +``` diff --git a/README.md b/README.md index caeea10..1fae560 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ The build depends on a sibling checkout of `fai/module-sdk`: └── text-extract/ # this repo ``` -This is a temporary path-based dependency. Once `fai-module-sdk` +This is a temporary path-based dependency. Once `chain-module-sdk` is published to a registry or a stable git tag, this module's `Cargo.toml` will switch over. The Forgejo CI workflow checks out the SDK repo into a sibling directory before building. @@ -81,5 +81,5 @@ integration test. Apache-2.0. See `LICENSE`. -Author: Dr. Stefan Flemming, Flemming.AI -Repository: https://git.flemming.ws/fai-modules/text-extract +Author: Dr. Stefan Flemming, Flemming.AI +Repository: https://git.flemming.ws/chain-modules/text-extract diff --git a/module.wasm b/module.wasm new file mode 100644 index 0000000..f96898b Binary files /dev/null and b/module.wasm differ diff --git a/module.yaml b/module.yaml index f7767b5..a01ed0d 100644 --- a/module.yaml +++ b/module.yaml @@ -1,4 +1,5 @@ -schema_version: 1 +schema_version: 3 +provider: chain name: text-extract version: 0.1.0 @@ -9,19 +10,35 @@ provides: # Inputs the invoke function accepts. inputs: - # The document to extract text from. Supported MIME types: - # application/pdf - # application/x-pdf - # application/vnd.openxmlformats-officedocument.wordprocessingml.document - document: bytes + document: + type: bytes + description: + en: | + The document to extract text from. Supported MIME types: + application/pdf + application/x-pdf + application/vnd.openxmlformats-officedocument.wordprocessingml.document + de: | + Das Dokument, aus dem Text extrahiert werden soll. Unterstützte MIME-Typen: + application/pdf + application/x-pdf + application/vnd.openxmlformats-officedocument.wordprocessingml.document # Outputs produced. outputs: - # JSON-encoded ExtractedDocument: - # { engine, engine_version, pages: [{number, text, confidence?, bbox?}] } - # confidence and bbox are reserved for a future service-mode variant - # (pdfium / mupdf via host services). v0.1.0 omits them. - extracted: json + extracted: + type: json + description: + en: | + JSON-encoded ExtractedDocument: + { engine, engine_version, pages: [{number, text, confidence?, bbox?}] } + confidence and bbox are reserved for a future service-mode variant + (pdfium / mupdf via host services). v0.1.0 omits them. + de: | + JSON-codiertes ExtractedDocument: + { engine, engine_version, pages: [{number, text, confidence?, bbox?}] } + confidence und bbox sind für eine künftige Service-Mode-Variante reserviert + (pdfium / mupdf über Host-Services). v0.1.0 liefert sie nicht. # Permissions required. # @@ -29,3 +46,13 @@ outputs: # zip, quick-xml). No filesystem, no network. The empty list makes # this explicit; the hub enforces it. permissions: [] + +# MIME-type allow-list for the `document` input. Studio reads this +# through ModuleInfoResponse.accepts_mime and uses it as the OS +# file-picker filter so operators don't accidentally pick a `.png` +# for a PDF-extract step. Same list documented inline next to +# `inputs.document` above; kept synchronised by hand for now. +accepts_mime: + - application/pdf + - application/x-pdf + - application/vnd.openxmlformats-officedocument.wordprocessingml.document diff --git a/src/lib.rs b/src/lib.rs index 04b724c..5918728 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -39,7 +39,7 @@ #![allow(clippy::result_large_err)] -use fai_module_sdk::prelude::*; +use chain_module_sdk::prelude::*; use serde::Serialize; const PDF_MIME: &str = "application/pdf"; diff --git a/tests/docx_roundtrip.rs b/tests/docx_roundtrip.rs index b3494dd..ad6b2ff 100644 --- a/tests/docx_roundtrip.rs +++ b/tests/docx_roundtrip.rs @@ -16,7 +16,7 @@ use std::io::Write; -use fai_module_sdk::prelude::*; +use chain_module_sdk::prelude::*; use serde::Deserialize; use text_extract::invoke;