From 3dede6d7d88709f17e98041c3fb872e14f8f3020 Mon Sep 17 00:00:00 2001 From: flemming-it Date: Sun, 10 May 2026 21:41:23 +0200 Subject: [PATCH] ci(security): add self-test harness (mirror of fai/platform) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 18 scenarios that verify the security script's rules still trip on every tripwire — PATs, AWS keys, PEM headers, env files, allowlist exempts, confidentiality terms, marketing phrases, .security-allow exclusion, plus message-mode (Conventional Commits, DCO, Claude trailer, banned phrases). Wired into the security workflow as a second-line gate after the diff-based check. Catches the refactor-weakens-a-rule class of regression: the diff scan can be green while a rule silently no-ops; the harness fails loudly when that happens. Signed-off-by: flemming-it --- .forgejo/workflows/security.yml | 6 + tools/security/test-check-staged.sh | 219 ++++++++++++++++++++++++++++ 2 files changed, 225 insertions(+) create mode 100755 tools/security/test-check-staged.sh diff --git a/.forgejo/workflows/security.yml b/.forgejo/workflows/security.yml index 4f716ec..863c899 100644 --- a/.forgejo/workflows/security.yml +++ b/.forgejo/workflows/security.yml @@ -43,3 +43,9 @@ jobs: BASE="origin/$GITHUB_BASE_REF" fi bash tools/security/check-staged.sh ci "$BASE" + + - name: Security script self-test + # Verifies every rule still trips on its tripwire — catches + # the case where a refactor of the script silently weakens + # a check. + run: bash tools/security/test-check-staged.sh diff --git a/tools/security/test-check-staged.sh b/tools/security/test-check-staged.sh new file mode 100755 index 0000000..80cba27 --- /dev/null +++ b/tools/security/test-check-staged.sh @@ -0,0 +1,219 @@ +#!/usr/bin/env bash +# F∆I Platform — smoke tests for `tools/security/check-staged.sh`. +# +# Spins up an isolated tempdir per scenario, stages a tripwire, +# and asserts the security script either passes or fails with the +# expected violation string. Never touches the operator's repo. +# +# Run manually: +# bash tools/security/test-check-staged.sh +# +# Wire into CI alongside the script-under-test if you want a +# second-line check that the rules themselves still trigger. + +set -uo pipefail + +SCRIPT="$(cd "$(dirname "$0")" && pwd)/check-staged.sh" +if [ ! -x "$SCRIPT" ]; then + echo "FATAL: $SCRIPT not found or not executable" >&2 + exit 2 +fi + +PASS=0 +FAIL=0 + +setup_repo() { + local d + d=$(mktemp -d -t fai_check_test.XXXXXX) + ( + cd "$d" + git init -q + git config user.email t@t.local + git config user.name "Test" + git commit --allow-empty -m "init" -q + ) + cp "$SCRIPT" "$d/check.sh" + chmod +x "$d/check.sh" + echo "$d" +} + +# Args: label, expected-substring-in-stderr +assert_content_fails() { + local label="$1" expected="$2" + local out + if out=$(./check.sh content 2>&1); then + echo " FAIL [$label]: expected fail, but passed" + FAIL=$((FAIL + 1)) + return + fi + if echo "$out" | grep -q -F -- "$expected"; then + echo " PASS [$label]" + PASS=$((PASS + 1)) + else + echo " FAIL [$label]: failed but did not mention '$expected'" + echo "$out" | head -5 | sed 's/^/ /' + FAIL=$((FAIL + 1)) + fi +} + +assert_content_passes() { + local label="$1" + if ./check.sh content >/dev/null 2>&1; then + echo " PASS [$label]" + PASS=$((PASS + 1)) + else + echo " FAIL [$label]: expected pass, but failed" + ./check.sh content 2>&1 | head -5 | sed 's/^/ /' + FAIL=$((FAIL + 1)) + fi +} + +assert_message_fails() { + local label="$1" msg="$2" expected="$3" + local f + f=$(mktemp -t fai_msg_test.XXXXXX) + printf "%s\n" "$msg" > "$f" + local out + if out=$(./check.sh message "$f" 2>&1); then + echo " FAIL [$label]: expected fail, but passed" + FAIL=$((FAIL + 1)) + rm -f "$f" + return + fi + if echo "$out" | grep -q -F -- "$expected"; then + echo " PASS [$label]" + PASS=$((PASS + 1)) + else + echo " FAIL [$label]: failed but did not mention '$expected'" + echo "$out" | head -5 | sed 's/^/ /' + FAIL=$((FAIL + 1)) + fi + rm -f "$f" +} + +assert_message_passes() { + local label="$1" msg="$2" + local f + f=$(mktemp -t fai_msg_test.XXXXXX) + printf "%s\n" "$msg" > "$f" + if ./check.sh message "$f" >/dev/null 2>&1; then + echo " PASS [$label]" + PASS=$((PASS + 1)) + else + echo " FAIL [$label]: expected pass, but failed" + ./check.sh message "$f" 2>&1 | head -5 | sed 's/^/ /' + FAIL=$((FAIL + 1)) + fi + rm -f "$f" +} + +# ─── content scenarios ────────────────────────────────────────────── + +echo "── content scans ──" + +d=$(setup_repo); cd "$d" +assert_content_passes "clean state" + +d=$(setup_repo); cd "$d" +echo "TOKEN=ghp_aBcDeFgHiJkLmNoPqRsTuVwXyZ0123456789" > leak.txt +git add leak.txt +assert_content_fails "github PAT detected" "ghp_" + +d=$(setup_repo); cd "$d" +echo "key: sk-ant-aBcDeFgHiJkLmNoPqRsTuVwXyZ0123456789aBcDeF" > anthropic.txt +git add anthropic.txt +assert_content_fails "anthropic key detected" "sk-ant-" + +d=$(setup_repo); cd "$d" +echo "AWS_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE" > aws.txt +git add aws.txt +assert_content_fails "AWS access key detected" "AKIA" + +d=$(setup_repo); cd "$d" +echo "-----BEGIN RSA PRIVATE KEY-----" > id.pem +echo "stuff" >> id.pem +git add id.pem +assert_content_fails "PEM private key detected" "PRIVATE KEY" + +d=$(setup_repo); cd "$d" +touch .env +echo "X=y" > .env +git add .env +assert_content_fails ".env in stage" "forbidden file" + +d=$(setup_repo); cd "$d" +echo "DEMO_VAR=value" > .env.example +git add .env.example +assert_content_passes ".env.example allowlisted" + +d=$(setup_repo); cd "$d" +mkdir tests +echo "fake cert content" > tests/server.pem +git add tests/server.pem +assert_content_passes "tests/*.pem allowlisted" + +d=$(setup_repo); cd "$d" +echo "Notes from the ITDZ Berlin meeting" > confidential.md +git add confidential.md +assert_content_fails "confidential term ITDZ" "ITDZ" + +d=$(setup_repo); cd "$d" +echo "Our viral release of a powerful tool that just works" > marketing.md +git add marketing.md +assert_content_fails "banned phrase viral" "viral" + +d=$(setup_repo); cd "$d" +echo "viral content" > marketing.md +mkdir -p tools/today +echo "this references viral and powerful intentionally" > tools/today/prompt.md +echo "tools/today/" > .security-allow +git add .security-allow tools/today/prompt.md +assert_content_passes ".security-allow excludes legit listing" +# But banned content outside the allow path must still trip: +git add marketing.md +assert_content_fails ".security-allow does not cover other paths" "viral" + +# ─── message scenarios ────────────────────────────────────────────── + +echo "" +echo "── message scans ──" + +# Need a working dir to run the script from (it cd's to git toplevel). +d=$(setup_repo); cd "$d" + +assert_message_passes \ + "well-formed Conventional Commits + DCO" \ + "$(printf 'feat(hub): add thing\n\nbody\n\nSigned-off-by: Test ')" + +assert_message_fails \ + "missing DCO" \ + "feat(hub): add thing" \ + "missing DCO" + +assert_message_fails \ + "non-conventional subject" \ + "$(printf 'added stuff\n\nSigned-off-by: Test ')" \ + "Conventional Commits" + +assert_message_fails \ + "Claude co-author trailer" \ + "$(printf 'feat(hub): add thing\n\nCo-Authored-By: Claude \nSigned-off-by: Test ')" \ + "Co-Authored-By: Claude" + +assert_message_fails \ + "banned phrase in message body" \ + "$(printf 'feat(hub): add thing\n\nThis is just a viral test.\n\nSigned-off-by: Test ')" \ + "banned phrase" + +assert_message_passes \ + "merge commits skip Conventional check" \ + "$(printf 'Merge branch feature-x into main\n\nSigned-off-by: Test ')" + +# ─── summary ──────────────────────────────────────────────────────── + +echo "" +echo "─────────────────────────────────" +echo "Tests: $PASS passed, $FAIL failed" +if [ "$FAIL" -ne 0 ]; then + exit 1 +fi