ci(security): add self-test harness (mirror of fai/platform)
Some checks failed
Security / Security check (push) Has been cancelled
Some checks failed
Security / Security check (push) Has been cancelled
18 scenarios that verify the security script's rules still trip on every tripwire — PATs, AWS keys, PEM headers, env files, allowlist exempts, confidentiality terms, marketing phrases, .security-allow exclusion, plus message-mode (Conventional Commits, DCO, Claude trailer, banned phrases). Wired into the security workflow as a second-line gate after the diff-based check. Catches the refactor-weakens-a-rule class of regression: the diff scan can be green while a rule silently no-ops; the harness fails loudly when that happens. Signed-off-by: flemming-it <stefan.a.flemming@googlemail.com>
This commit is contained in:
flemming-it
parent
7416c34d7c
commit
3dede6d7d8
2 changed files with 225 additions and 0 deletions
|
|
@ -43,3 +43,9 @@ jobs:
|
||||||
BASE="origin/$GITHUB_BASE_REF"
|
BASE="origin/$GITHUB_BASE_REF"
|
||||||
fi
|
fi
|
||||||
bash tools/security/check-staged.sh ci "$BASE"
|
bash tools/security/check-staged.sh ci "$BASE"
|
||||||
|
|
||||||
|
- name: Security script self-test
|
||||||
|
# Verifies every rule still trips on its tripwire — catches
|
||||||
|
# the case where a refactor of the script silently weakens
|
||||||
|
# a check.
|
||||||
|
run: bash tools/security/test-check-staged.sh
|
||||||
|
|
|
||||||
219
tools/security/test-check-staged.sh
Executable file
219
tools/security/test-check-staged.sh
Executable file
|
|
@ -0,0 +1,219 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
# F∆I Platform — smoke tests for `tools/security/check-staged.sh`.
|
||||||
|
#
|
||||||
|
# Spins up an isolated tempdir per scenario, stages a tripwire,
|
||||||
|
# and asserts the security script either passes or fails with the
|
||||||
|
# expected violation string. Never touches the operator's repo.
|
||||||
|
#
|
||||||
|
# Run manually:
|
||||||
|
# bash tools/security/test-check-staged.sh
|
||||||
|
#
|
||||||
|
# Wire into CI alongside the script-under-test if you want a
|
||||||
|
# second-line check that the rules themselves still trigger.
|
||||||
|
|
||||||
|
set -uo pipefail
|
||||||
|
|
||||||
|
SCRIPT="$(cd "$(dirname "$0")" && pwd)/check-staged.sh"
|
||||||
|
if [ ! -x "$SCRIPT" ]; then
|
||||||
|
echo "FATAL: $SCRIPT not found or not executable" >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
|
||||||
|
PASS=0
|
||||||
|
FAIL=0
|
||||||
|
|
||||||
|
setup_repo() {
|
||||||
|
local d
|
||||||
|
d=$(mktemp -d -t fai_check_test.XXXXXX)
|
||||||
|
(
|
||||||
|
cd "$d"
|
||||||
|
git init -q
|
||||||
|
git config user.email t@t.local
|
||||||
|
git config user.name "Test"
|
||||||
|
git commit --allow-empty -m "init" -q
|
||||||
|
)
|
||||||
|
cp "$SCRIPT" "$d/check.sh"
|
||||||
|
chmod +x "$d/check.sh"
|
||||||
|
echo "$d"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Args: label, expected-substring-in-stderr
|
||||||
|
assert_content_fails() {
|
||||||
|
local label="$1" expected="$2"
|
||||||
|
local out
|
||||||
|
if out=$(./check.sh content 2>&1); then
|
||||||
|
echo " FAIL [$label]: expected fail, but passed"
|
||||||
|
FAIL=$((FAIL + 1))
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
if echo "$out" | grep -q -F -- "$expected"; then
|
||||||
|
echo " PASS [$label]"
|
||||||
|
PASS=$((PASS + 1))
|
||||||
|
else
|
||||||
|
echo " FAIL [$label]: failed but did not mention '$expected'"
|
||||||
|
echo "$out" | head -5 | sed 's/^/ /'
|
||||||
|
FAIL=$((FAIL + 1))
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
assert_content_passes() {
|
||||||
|
local label="$1"
|
||||||
|
if ./check.sh content >/dev/null 2>&1; then
|
||||||
|
echo " PASS [$label]"
|
||||||
|
PASS=$((PASS + 1))
|
||||||
|
else
|
||||||
|
echo " FAIL [$label]: expected pass, but failed"
|
||||||
|
./check.sh content 2>&1 | head -5 | sed 's/^/ /'
|
||||||
|
FAIL=$((FAIL + 1))
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
assert_message_fails() {
|
||||||
|
local label="$1" msg="$2" expected="$3"
|
||||||
|
local f
|
||||||
|
f=$(mktemp -t fai_msg_test.XXXXXX)
|
||||||
|
printf "%s\n" "$msg" > "$f"
|
||||||
|
local out
|
||||||
|
if out=$(./check.sh message "$f" 2>&1); then
|
||||||
|
echo " FAIL [$label]: expected fail, but passed"
|
||||||
|
FAIL=$((FAIL + 1))
|
||||||
|
rm -f "$f"
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
if echo "$out" | grep -q -F -- "$expected"; then
|
||||||
|
echo " PASS [$label]"
|
||||||
|
PASS=$((PASS + 1))
|
||||||
|
else
|
||||||
|
echo " FAIL [$label]: failed but did not mention '$expected'"
|
||||||
|
echo "$out" | head -5 | sed 's/^/ /'
|
||||||
|
FAIL=$((FAIL + 1))
|
||||||
|
fi
|
||||||
|
rm -f "$f"
|
||||||
|
}
|
||||||
|
|
||||||
|
assert_message_passes() {
|
||||||
|
local label="$1" msg="$2"
|
||||||
|
local f
|
||||||
|
f=$(mktemp -t fai_msg_test.XXXXXX)
|
||||||
|
printf "%s\n" "$msg" > "$f"
|
||||||
|
if ./check.sh message "$f" >/dev/null 2>&1; then
|
||||||
|
echo " PASS [$label]"
|
||||||
|
PASS=$((PASS + 1))
|
||||||
|
else
|
||||||
|
echo " FAIL [$label]: expected pass, but failed"
|
||||||
|
./check.sh message "$f" 2>&1 | head -5 | sed 's/^/ /'
|
||||||
|
FAIL=$((FAIL + 1))
|
||||||
|
fi
|
||||||
|
rm -f "$f"
|
||||||
|
}
|
||||||
|
|
||||||
|
# ─── content scenarios ──────────────────────────────────────────────
|
||||||
|
|
||||||
|
echo "── content scans ──"
|
||||||
|
|
||||||
|
d=$(setup_repo); cd "$d"
|
||||||
|
assert_content_passes "clean state"
|
||||||
|
|
||||||
|
d=$(setup_repo); cd "$d"
|
||||||
|
echo "TOKEN=ghp_aBcDeFgHiJkLmNoPqRsTuVwXyZ0123456789" > leak.txt
|
||||||
|
git add leak.txt
|
||||||
|
assert_content_fails "github PAT detected" "ghp_"
|
||||||
|
|
||||||
|
d=$(setup_repo); cd "$d"
|
||||||
|
echo "key: sk-ant-aBcDeFgHiJkLmNoPqRsTuVwXyZ0123456789aBcDeF" > anthropic.txt
|
||||||
|
git add anthropic.txt
|
||||||
|
assert_content_fails "anthropic key detected" "sk-ant-"
|
||||||
|
|
||||||
|
d=$(setup_repo); cd "$d"
|
||||||
|
echo "AWS_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE" > aws.txt
|
||||||
|
git add aws.txt
|
||||||
|
assert_content_fails "AWS access key detected" "AKIA"
|
||||||
|
|
||||||
|
d=$(setup_repo); cd "$d"
|
||||||
|
echo "-----BEGIN RSA PRIVATE KEY-----" > id.pem
|
||||||
|
echo "stuff" >> id.pem
|
||||||
|
git add id.pem
|
||||||
|
assert_content_fails "PEM private key detected" "PRIVATE KEY"
|
||||||
|
|
||||||
|
d=$(setup_repo); cd "$d"
|
||||||
|
touch .env
|
||||||
|
echo "X=y" > .env
|
||||||
|
git add .env
|
||||||
|
assert_content_fails ".env in stage" "forbidden file"
|
||||||
|
|
||||||
|
d=$(setup_repo); cd "$d"
|
||||||
|
echo "DEMO_VAR=value" > .env.example
|
||||||
|
git add .env.example
|
||||||
|
assert_content_passes ".env.example allowlisted"
|
||||||
|
|
||||||
|
d=$(setup_repo); cd "$d"
|
||||||
|
mkdir tests
|
||||||
|
echo "fake cert content" > tests/server.pem
|
||||||
|
git add tests/server.pem
|
||||||
|
assert_content_passes "tests/*.pem allowlisted"
|
||||||
|
|
||||||
|
d=$(setup_repo); cd "$d"
|
||||||
|
echo "Notes from the ITDZ Berlin meeting" > confidential.md
|
||||||
|
git add confidential.md
|
||||||
|
assert_content_fails "confidential term ITDZ" "ITDZ"
|
||||||
|
|
||||||
|
d=$(setup_repo); cd "$d"
|
||||||
|
echo "Our viral release of a powerful tool that just works" > marketing.md
|
||||||
|
git add marketing.md
|
||||||
|
assert_content_fails "banned phrase viral" "viral"
|
||||||
|
|
||||||
|
d=$(setup_repo); cd "$d"
|
||||||
|
echo "viral content" > marketing.md
|
||||||
|
mkdir -p tools/today
|
||||||
|
echo "this references viral and powerful intentionally" > tools/today/prompt.md
|
||||||
|
echo "tools/today/" > .security-allow
|
||||||
|
git add .security-allow tools/today/prompt.md
|
||||||
|
assert_content_passes ".security-allow excludes legit listing"
|
||||||
|
# But banned content outside the allow path must still trip:
|
||||||
|
git add marketing.md
|
||||||
|
assert_content_fails ".security-allow does not cover other paths" "viral"
|
||||||
|
|
||||||
|
# ─── message scenarios ──────────────────────────────────────────────
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "── message scans ──"
|
||||||
|
|
||||||
|
# Need a working dir to run the script from (it cd's to git toplevel).
|
||||||
|
d=$(setup_repo); cd "$d"
|
||||||
|
|
||||||
|
assert_message_passes \
|
||||||
|
"well-formed Conventional Commits + DCO" \
|
||||||
|
"$(printf 'feat(hub): add thing\n\nbody\n\nSigned-off-by: Test <t@t.local>')"
|
||||||
|
|
||||||
|
assert_message_fails \
|
||||||
|
"missing DCO" \
|
||||||
|
"feat(hub): add thing" \
|
||||||
|
"missing DCO"
|
||||||
|
|
||||||
|
assert_message_fails \
|
||||||
|
"non-conventional subject" \
|
||||||
|
"$(printf 'added stuff\n\nSigned-off-by: Test <t@t.local>')" \
|
||||||
|
"Conventional Commits"
|
||||||
|
|
||||||
|
assert_message_fails \
|
||||||
|
"Claude co-author trailer" \
|
||||||
|
"$(printf 'feat(hub): add thing\n\nCo-Authored-By: Claude <claude@anthropic.com>\nSigned-off-by: Test <t@t.local>')" \
|
||||||
|
"Co-Authored-By: Claude"
|
||||||
|
|
||||||
|
assert_message_fails \
|
||||||
|
"banned phrase in message body" \
|
||||||
|
"$(printf 'feat(hub): add thing\n\nThis is just a viral test.\n\nSigned-off-by: Test <t@t.local>')" \
|
||||||
|
"banned phrase"
|
||||||
|
|
||||||
|
assert_message_passes \
|
||||||
|
"merge commits skip Conventional check" \
|
||||||
|
"$(printf 'Merge branch feature-x into main\n\nSigned-off-by: Test <t@t.local>')"
|
||||||
|
|
||||||
|
# ─── summary ────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "─────────────────────────────────"
|
||||||
|
echo "Tests: $PASS passed, $FAIL failed"
|
||||||
|
if [ "$FAIL" -ne 0 ]; then
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
Loading…
Add table
Add a link
Reference in a new issue