From 29f7313876297998d4bfdb4f7f1c2ea6d5127712 Mon Sep 17 00:00:00 2001 From: "Jonathan D.A. Jewell" <6759885+hyperpolymath@users.noreply.github.com> Date: Thu, 27 Aug 2026 14:02:16 +0100 Subject: [PATCH 1/5] fix(ci): the invisible-character gate never matched anything MEASURED 2026-08-27: this gate's pattern caught 0 OF 6 invisible-character test cases. It has never detected an NBSP, zero-width space, BOM, soft hyphen, bidi override or word joiner. ROOT CAUSE: the pattern used UTF-8 BYTE sequences (\xc2\xa0) while grep -P matches CHARACTERS. Bytes c2 a0 are ONE character U+00A0; \xc2\xa0 asks for TWO characters, U+00C2 then U+00A0, which is never present. grep -P '\xc2\xa0' -> miss grep -P '\x{a0}' -> MATCH Only \x00 worked, being single-byte in both readings. FIXED: codepoint escapes; C0 control characters \x01-\x08,\x0B,\x0C,\x0E-\x1F added (TAB/LF/CR excluded); and grep -a, without which grep skips any NUL-bearing file as binary. The C0 range matters: a stray BACKSPACE byte made a workflow unparseable in developer-ecosystem, so it never ran, and this linter called it clean. Canonical fix: hyperpolymath/empty-linter#70. 1 file(s) here. VERIFIED: YAML re-parsed, and the corrected pattern was confirmed to catch a real NBSP before the change was kept. --- .github/workflows/dogfood-gate.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dogfood-gate.yml b/.github/workflows/dogfood-gate.yml index 717fd82..1e6196a 100644 --- a/.github/workflows/dogfood-gate.yml +++ b/.github/workflows/dogfood-gate.yml @@ -115,7 +115,7 @@ jobs: # Checks for: zero-width spaces, zero-width joiners, BOM, soft hyphens, # non-breaking spaces, null bytes, and other invisible Unicode in source files. set +e - PATTERNS='\xc2\xa0|\xe2\x80\x8b|\xe2\x80\x8c|\xe2\x80\x8d|\xef\xbb\xbf|\xc2\xad|\xe2\x80\x8e|\xe2\x80\x8f|\xe2\x80\xaa|\xe2\x80\xab|\xe2\x80\xac|\xe2\x80\xad|\xe2\x80\xae|\x00' + PATTERNS='\x00|[\x01-\x08\x0B\x0C\x0E-\x1F]|\x{a0}|\x{ad}|\x{200b}|\x{200c}|\x{200d}|\x{200e}|\x{200f}|\x{202a}|\x{202b}|\x{202c}|\x{202d}|\x{202e}|\x{2060}|\x{feff}' find "$GITHUB_WORKSPACE" \ -not -path '*/.git/*' -not -path '*/node_modules/*' \ -not -path '*/.deno/*' -not -path '*/target/*' \ @@ -126,7 +126,7 @@ jobs: -o -name '*.yml' -o -name '*.yaml' -o -name '*.md' -o -name '*.adoc' \ -o -name '*.idr' -o -name '*.zig' -o -name '*.v' -o -name '*.jl' \ -o -name '*.gleam' -o -name '*.hs' -o -name '*.ml' -o -name '*.sh' \) \ - -exec grep -Prl "$PATTERNS" {} \; > /tmp/empty-lint-results.txt 2>/dev/null + -exec grep -aPrl "$PATTERNS" {} \; > /tmp/empty-lint-results.txt 2>/dev/null EL_EXIT=$? set -e From d29b287563994247b0489778951728ebd46741c0 Mon Sep 17 00:00:00 2001 From: "Jonathan D.A. Jewell" <6759885+hyperpolymath@users.noreply.github.com> Date: Fri, 28 Aug 2026 06:27:01 +0100 Subject: [PATCH 2/5] fix(ci): enforce C0/NUL corruption in-step; warn on invisible Unicode Second layer of the empty-linter fix, scoped by an owner ruling after a census. DETECTION (layer 1, earlier commit on this branch) sees everything the pattern covers. ENFORCEMENT (this commit) distinguishes two classes: BLOCKING C0 control characters and NUL. Never legitimate; proven damage - a backspace byte made a workflow unloadable (it never ran once), and LaTeX maths in wiki files was silently mangled where a generation step turned backslash-b commands into backspaces. ADVISORY NBSP, BOM, zero-width marks. A gate-lens census found ~2,100 first-party files carry these as legitimate typography in prose; blocking would fail 2,333 files estate-wide for no safety gain. Enforcement lives INSIDE the scan step: if the scanner crashes, the step fails the job directly, so empty counts can never drift into a separate check that passes silently (review finding). The blocking count re-greps only the files the full pattern already flagged, so the find expression is not duplicated and cannot drift. 1 file(s). YAML re-parsed per edit; reverted on any mis-apply. --- .github/workflows/dogfood-gate.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/.github/workflows/dogfood-gate.yml b/.github/workflows/dogfood-gate.yml index 1e6196a..f367088 100644 --- a/.github/workflows/dogfood-gate.yml +++ b/.github/workflows/dogfood-gate.yml @@ -135,12 +135,40 @@ jobs: echo "exit_code=$EL_EXIT" >> "$GITHUB_OUTPUT" echo "ready=true" >> "$GITHUB_OUTPUT" + # Blocking subset: C0 controls and NUL only (owner ruling 2026-08-28). + # Invisible Unicode (NBSP/BOM/zero-width) stays ADVISORY - about 2,100 + # estate files carry it as legitimate typography in prose. + blocking=0 + while IFS= read -r bf; do + [ -z "$bf" ] && continue + if grep -qaP '\x00|[\x01-\x08\x0B\x0C\x0E-\x1F]' "$bf"; then + blocking=$((blocking+1)) + echo "::error file=${bf#$GITHUB_WORKSPACE/}::C0 control characters or NUL bytes - file corruption, blocks the gate" + fi + done < /tmp/empty-lint-results.txt + echo "blocking=$blocking" >> "$GITHUB_OUTPUT" + # Emit annotations for each file with invisible chars while IFS= read -r filepath; do [ -z "$filepath" ] && continue REL_PATH="${filepath#$GITHUB_WORKSPACE/}" echo "::warning file=${REL_PATH}::Invisible Unicode characters detected (zero-width space, BOM, NBSP, etc.)" done < /tmp/empty-lint-results.txt + + # Enforce (owner ruling 2026-08-28): C0/NUL corruption BLOCKS; other + # invisible Unicode stays advisory. Enforcement lives inside this step + # so a crash above fails the job directly - counts can never arrive + # empty into a separate check that then passes silently. + if [ "$EL_EXIT" -ne 0 ]; then + echo "::warning::invisible-character scan exited $EL_EXIT - results may be incomplete" + fi + if [ "${blocking:-0}" -gt 0 ]; then + echo "## Empty-linter: BLOCKED - $blocking file(s) with C0/NUL corruption" >> "$GITHUB_STEP_SUMMARY" + echo "::error::$blocking file(s) contain C0 control characters or NUL bytes - corruption, not typography. See file annotations." + exit 1 + elif [ "${FINDINGS:-0}" -gt 0 ]; then + echo "::notice::$FINDINGS file(s) carry invisible Unicode (NBSP/BOM/zero-width) - advisory only" + fi - name: Write summary run: | if [ "${{ steps.lint.outputs.ready }}" = "true" ]; then From 4a37a3bc0a5312c9f685c9567566c42bc1acc22a Mon Sep 17 00:00:00 2001 From: "Jonathan D.A. Jewell" <6759885+hyperpolymath@users.noreply.github.com> Date: Fri, 28 Aug 2026 16:05:03 +0100 Subject: [PATCH 3/5] fix(ci): make invisible-character PCRE locale-independent --- .github/workflows/dogfood-gate.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dogfood-gate.yml b/.github/workflows/dogfood-gate.yml index f367088..bc52cc7 100644 --- a/.github/workflows/dogfood-gate.yml +++ b/.github/workflows/dogfood-gate.yml @@ -115,7 +115,7 @@ jobs: # Checks for: zero-width spaces, zero-width joiners, BOM, soft hyphens, # non-breaking spaces, null bytes, and other invisible Unicode in source files. set +e - PATTERNS='\x00|[\x01-\x08\x0B\x0C\x0E-\x1F]|\x{a0}|\x{ad}|\x{200b}|\x{200c}|\x{200d}|\x{200e}|\x{200f}|\x{202a}|\x{202b}|\x{202c}|\x{202d}|\x{202e}|\x{2060}|\x{feff}' + PATTERNS='(*UTF)[\x00-\x08\x0B\x0C\x0E-\x1F\x{a0}\x{ad}\x{200b}-\x{200f}\x{202a}-\x{202f}\x{2060}\x{2066}-\x{2069}\x{feff}]' find "$GITHUB_WORKSPACE" \ -not -path '*/.git/*' -not -path '*/node_modules/*' \ -not -path '*/.deno/*' -not -path '*/target/*' \ From 4f11d2d68c32db9dc54ed8352ac0c94484ac7e66 Mon Sep 17 00:00:00 2001 From: "Jonathan D.A. Jewell" <6759885+hyperpolymath@users.noreply.github.com> Date: Sun, 30 Aug 2026 15:04:14 +0100 Subject: [PATCH 4/5] fix(ci): harden invisible-character scan --- .github/workflows/dogfood-gate.yml | 87 ++++++++++--------------- scripts/check-invisible-characters.sh | 70 ++++++++++++++++++++ tests/invisible-characters-test.sh | 92 +++++++++++++++++++++++++++ 3 files changed, 197 insertions(+), 52 deletions(-) create mode 100755 scripts/check-invisible-characters.sh create mode 100755 tests/invisible-characters-test.sh diff --git a/.github/workflows/dogfood-gate.yml b/.github/workflows/dogfood-gate.yml index bc52cc7..4ea0f9e 100644 --- a/.github/workflows/dogfood-gate.yml +++ b/.github/workflows/dogfood-gate.yml @@ -110,64 +110,47 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Scan for invisible characters id: lint + shell: bash run: | - # Inline invisible character detection (from empty-linter's core patterns). - # Checks for: zero-width spaces, zero-width joiners, BOM, soft hyphens, - # non-breaking spaces, null bytes, and other invisible Unicode in source files. - set +e - PATTERNS='(*UTF)[\x00-\x08\x0B\x0C\x0E-\x1F\x{a0}\x{ad}\x{200b}-\x{200f}\x{202a}-\x{202f}\x{2060}\x{2066}-\x{2069}\x{feff}]' - find "$GITHUB_WORKSPACE" \ - -not -path '*/.git/*' -not -path '*/node_modules/*' \ - -not -path '*/.deno/*' -not -path '*/target/*' \ - -not -path '*/_build/*' -not -path '*/deps/*' \ - -not -path '*/external_corpora/*' -not -path '*/.lake/*' \ - -type f \( -name '*.rs' -o -name '*.ex' -o -name '*.exs' -o -name '*.res' \ - -o -name '*.js' -o -name '*.ts' -o -name '*.json' -o -name '*.toml' \ - -o -name '*.yml' -o -name '*.yaml' -o -name '*.md' -o -name '*.adoc' \ - -o -name '*.idr' -o -name '*.zig' -o -name '*.v' -o -name '*.jl' \ - -o -name '*.gleam' -o -name '*.hs' -o -name '*.ml' -o -name '*.sh' \) \ - -exec grep -aPrl "$PATTERNS" {} \; > /tmp/empty-lint-results.txt 2>/dev/null - EL_EXIT=$? - set -e + RESULTS_FILE="$RUNNER_TEMP/empty-lint-results.bin" + BLOCKING_FILE="$RUNNER_TEMP/empty-lint-blocking-results.bin" + if ! scripts/check-invisible-characters.sh \ + "$GITHUB_WORKSPACE" "$RESULTS_FILE" "$BLOCKING_FILE"; then + echo "::error::Invisible-character scanner failed; refusing a partial pass" + exit 2 + fi - FINDINGS=$(wc -l < /tmp/empty-lint-results.txt 2>/dev/null || echo 0) - echo "findings=$FINDINGS" >> "$GITHUB_OUTPUT" - echo "exit_code=$EL_EXIT" >> "$GITHUB_OUTPUT" - echo "ready=true" >> "$GITHUB_OUTPUT" + FINDINGS=0 + while IFS= read -r -d '' filepath; do + FINDINGS=$((FINDINGS + 1)) + REL_PATH="${filepath#"$GITHUB_WORKSPACE"/}" + SAFE_PATH="${REL_PATH//'%'/'%25'}" + SAFE_PATH="${SAFE_PATH//$'\r'/'%0D'}" + SAFE_PATH="${SAFE_PATH//$'\n'/'%0A'}" + SAFE_PATH="${SAFE_PATH//':'/'%3A'}" + SAFE_PATH="${SAFE_PATH//','/'%2C'}" + echo "::warning file=${SAFE_PATH}::Invisible Unicode or C0 characters detected" + done < "$RESULTS_FILE" - # Blocking subset: C0 controls and NUL only (owner ruling 2026-08-28). - # Invisible Unicode (NBSP/BOM/zero-width) stays ADVISORY - about 2,100 - # estate files carry it as legitimate typography in prose. - blocking=0 - while IFS= read -r bf; do - [ -z "$bf" ] && continue - if grep -qaP '\x00|[\x01-\x08\x0B\x0C\x0E-\x1F]' "$bf"; then - blocking=$((blocking+1)) - echo "::error file=${bf#$GITHUB_WORKSPACE/}::C0 control characters or NUL bytes - file corruption, blocks the gate" - fi - done < /tmp/empty-lint-results.txt - echo "blocking=$blocking" >> "$GITHUB_OUTPUT" + BLOCKING=0 + while IFS= read -r -d '' filepath; do + BLOCKING=$((BLOCKING + 1)) + REL_PATH="${filepath#"$GITHUB_WORKSPACE"/}" + SAFE_PATH="${REL_PATH//'%'/'%25'}" + SAFE_PATH="${SAFE_PATH//$'\r'/'%0D'}" + SAFE_PATH="${SAFE_PATH//$'\n'/'%0A'}" + SAFE_PATH="${SAFE_PATH//':'/'%3A'}" + SAFE_PATH="${SAFE_PATH//','/'%2C'}" + echo "::error file=${SAFE_PATH}::C0 control character or NUL byte detected" + done < "$BLOCKING_FILE" - # Emit annotations for each file with invisible chars - while IFS= read -r filepath; do - [ -z "$filepath" ] && continue - REL_PATH="${filepath#$GITHUB_WORKSPACE/}" - echo "::warning file=${REL_PATH}::Invisible Unicode characters detected (zero-width space, BOM, NBSP, etc.)" - done < /tmp/empty-lint-results.txt + echo "findings=$FINDINGS" >> "$GITHUB_OUTPUT" + echo "blocking=$BLOCKING" >> "$GITHUB_OUTPUT" + echo "ready=true" >> "$GITHUB_OUTPUT" - # Enforce (owner ruling 2026-08-28): C0/NUL corruption BLOCKS; other - # invisible Unicode stays advisory. Enforcement lives inside this step - # so a crash above fails the job directly - counts can never arrive - # empty into a separate check that then passes silently. - if [ "$EL_EXIT" -ne 0 ]; then - echo "::warning::invisible-character scan exited $EL_EXIT - results may be incomplete" - fi - if [ "${blocking:-0}" -gt 0 ]; then - echo "## Empty-linter: BLOCKED - $blocking file(s) with C0/NUL corruption" >> "$GITHUB_STEP_SUMMARY" - echo "::error::$blocking file(s) contain C0 control characters or NUL bytes - corruption, not typography. See file annotations." + if [ "$BLOCKING" -gt 0 ]; then + echo "## Empty-linter: BLOCKED — $BLOCKING file(s) contain C0/NUL corruption" >> "$GITHUB_STEP_SUMMARY" exit 1 - elif [ "${FINDINGS:-0}" -gt 0 ]; then - echo "::notice::$FINDINGS file(s) carry invisible Unicode (NBSP/BOM/zero-width) - advisory only" fi - name: Write summary run: | diff --git a/scripts/check-invisible-characters.sh b/scripts/check-invisible-characters.sh new file mode 100755 index 0000000..bf77ac3 --- /dev/null +++ b/scripts/check-invisible-characters.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +# Byte-safe scanner for invisible Unicode encodings and forbidden C0 controls. +set -u + +scan_root="${1:-}" +results_file="${2:-}" +blocking_results_file="${3:-}" +grep_bin="${INVISIBLE_GREP_BIN:-grep}" +find_bin="${INVISIBLE_FIND_BIN:-find}" + +if [[ -z "$scan_root" || ! -d "$scan_root" || -z "$results_file" ]]; then + echo "usage: $0 SCAN_ROOT RESULTS_FILE" >&2 + exit 2 +fi + +# Scan bytes under the C locale. This detects UTF-8 encodings even when another +# byte in the file is invalid UTF-8, while excluding permitted TAB/LF/CR bytes. +pattern='[\x00-\x08\x0B\x0C\x0E-\x1F]|\xC2(?:\xA0|\xAD)|\xE2\x80[\x8B-\x8F\xAA-\xAF]|\xE2\x81(?:\xA0|[\xA6-\xA9])|\xEF\xBB\xBF' +blocking_pattern='[\x00-\x08\x0B\x0C\x0E-\x1F]' +: > "$results_file" || exit 2 +if [[ -n "$blocking_results_file" ]]; then + : > "$blocking_results_file" || exit 2 +fi +scan_error=0 +enumeration_file="$(mktemp /tmp/rsr-invisible-files.XXXXXX)" || exit 2 +# Invoked indirectly by the EXIT trap. +# shellcheck disable=SC2329 +cleanup() { + rm -f -- "$enumeration_file" +} +trap cleanup EXIT + +if ! "$find_bin" "$scan_root" \ + -not -path '*/.git/*' -not -path '*/node_modules/*' \ + -not -path '*/.deno/*' -not -path '*/target/*' \ + -not -path '*/_build/*' -not -path '*/deps/*' \ + -not -path '*/external_corpora/*' -not -path '*/.lake/*' \ + -type f \( -name '*.rs' -o -name '*.ex' -o -name '*.exs' -o -name '*.res' \ + -o -name '*.js' -o -name '*.ts' -o -name '*.json' -o -name '*.toml' \ + -o -name '*.yml' -o -name '*.yaml' -o -name '*.md' -o -name '*.adoc' \ + -o -name '*.idr' -o -name '*.zig' -o -name '*.v' -o -name '*.jl' \ + -o -name '*.gleam' -o -name '*.hs' -o -name '*.ml' -o -name '*.sh' \) \ + -print0 > "$enumeration_file"; then + echo "file enumeration failed: $scan_root" >&2 + exit 1 +fi + +while IFS= read -r -d '' filepath; do + LC_ALL=C "$grep_bin" -aPq "$pattern" "$filepath" + status=$? + case "$status" in + 0) + printf '%s\0' "$filepath" >> "$results_file" || scan_error=1 + if [[ -n "$blocking_results_file" ]]; then + LC_ALL=C "$grep_bin" -aPq "$blocking_pattern" "$filepath" + blocking_status=$? + case "$blocking_status" in + 0) printf '%s\0' "$filepath" >> "$blocking_results_file" || scan_error=1 ;; + 1) ;; + *) echo "blocking-classifier error ($blocking_status): $filepath" >&2; scan_error=1 ;; + esac + fi + ;; + 1) ;; + *) echo "scanner error ($status): $filepath" >&2; scan_error=1 ;; + esac +done < "$enumeration_file" + +exit "$scan_error" diff --git a/tests/invisible-characters-test.sh b/tests/invisible-characters-test.sh new file mode 100755 index 0000000..91f02bf --- /dev/null +++ b/tests/invisible-characters-test.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +fixture_root="$(mktemp -d /tmp/rsr-invisible-test.XXXXXX)" +cleanup() { + case "$fixture_root" in + /tmp/rsr-invisible-test.*) rm -rf -- "$fixture_root" ;; + *) echo "refusing unsafe cleanup target: $fixture_root" >&2 ;; + esac +} +trap cleanup EXIT + +scanner="$repo_root/scripts/check-invisible-characters.sh" +results="$fixture_root/results.bin" +blocking_results="$fixture_root/blocking-results.bin" +fixtures="$fixture_root/fixtures" +mkdir -p "$fixtures" + +printf 'tab\tline\ncarriage\rreturn\n' > "$fixtures/safe.md" +printf 'nbsp:\302\240\n' > "$fixtures/nbsp.md" +printf 'soft-hyphen:\302\255\n' > "$fixtures/soft-hyphen.adoc" +printf 'zero-width:\342\200\213\n' > "$fixtures/zero-width.json" +printf 'bidi:\342\200\256\n' > "$fixtures/bidi.toml" +printf 'word-joiner:\342\201\240\n' > "$fixtures/word-joiner.yml" +printf '\357\273\277leading bom\n' > "$fixtures/bom.sh" +printf 'nul:\000byte\n' > "$fixtures/nul.rs" +printf 'backspace:\010byte\n' > "$fixtures/backspace.rs" +printf 'invalid:\377 then nbsp:\302\240\n' > "$fixtures/invalid-utf8.md" +printf 'newline name:\302\240\n' > "$fixtures/with +newline.md" + +"$scanner" "$fixtures" "$results" "$blocking_results" + +count=0 +safe_seen=false +newline_seen=false +while IFS= read -r -d '' filepath; do + count=$((count + 1)) + [[ "$filepath" == "$fixtures/safe.md" ]] && safe_seen=true + [[ "$filepath" == "$fixtures/with"$'\n'"newline.md" ]] && newline_seen=true +done < "$results" + +[[ "$count" -eq 10 ]] || { + echo "expected 10 findings, got $count" >&2 + exit 1 +} +[[ "$safe_seen" == false ]] || { + echo "TAB/LF/CR-only safe fixture was incorrectly reported" >&2 + exit 1 +} +[[ "$newline_seen" == true ]] || { + echo "newline-containing filename was not preserved as one record" >&2 + exit 1 +} + +blocking_count=0 +nul_blocked=false +backspace_blocked=false +while IFS= read -r -d '' filepath; do + blocking_count=$((blocking_count + 1)) + [[ "$filepath" == "$fixtures/nul.rs" ]] && nul_blocked=true + [[ "$filepath" == "$fixtures/backspace.rs" ]] && backspace_blocked=true +done < "$blocking_results" +[[ "$blocking_count" -eq 2 && "$nul_blocked" == true && "$backspace_blocked" == true ]] || { + echo "expected only NUL and backspace fixtures in the blocking set" >&2 + exit 1 +} + +if "$scanner" "$fixture_root/missing" "$results"; then + echo "missing scan root did not fail closed" >&2 + exit 1 +fi + +failing_grep="$fixture_root/failing-grep" +printf '#!/usr/bin/env sh\nexit 2\n' > "$failing_grep" +chmod +x "$failing_grep" +if INVISIBLE_GREP_BIN="$failing_grep" "$scanner" "$fixtures" "$results"; then + echo "grep execution errors did not fail closed" >&2 + exit 1 +fi + +failing_find="$fixture_root/failing-find" +printf '#!/usr/bin/env sh\nexit 2\n' > "$failing_find" +chmod +x "$failing_find" +if INVISIBLE_FIND_BIN="$failing_find" "$scanner" "$fixtures" "$results"; then + echo "find execution errors did not fail closed" >&2 + exit 1 +fi + +echo "invisible-character scanner positive and negative controls passed" From 2e2fb708342ae0689dff273af67ac858d72f091c Mon Sep 17 00:00:00 2001 From: "coderabbitai[bot]" <136622811+coderabbitai[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 10:58:12 +0000 Subject: [PATCH 5/5] fix: apply CodeRabbit auto-fixes Fixed 3 file(s) based on 2 failed pre-merge checks. Co-authored-by: CodeRabbit --- scripts/check-invisible-characters.sh | 16 ++- src/fairness_nodocs.jl | 157 ++++++++++++++++++++++++++ tests/invisible-characters-test.sh | 26 ++++- 3 files changed, 194 insertions(+), 5 deletions(-) diff --git a/scripts/check-invisible-characters.sh b/scripts/check-invisible-characters.sh index bf77ac3..6c6e052 100755 --- a/scripts/check-invisible-characters.sh +++ b/scripts/check-invisible-characters.sh @@ -6,11 +6,12 @@ set -u scan_root="${1:-}" results_file="${2:-}" blocking_results_file="${3:-}" +leading_bom_results_file="${4:-}" grep_bin="${INVISIBLE_GREP_BIN:-grep}" find_bin="${INVISIBLE_FIND_BIN:-find}" if [[ -z "$scan_root" || ! -d "$scan_root" || -z "$results_file" ]]; then - echo "usage: $0 SCAN_ROOT RESULTS_FILE" >&2 + echo "usage: $0 SCAN_ROOT RESULTS_FILE [BLOCKING_RESULTS_FILE] [LEADING_BOM_RESULTS_FILE]" >&2 exit 2 fi @@ -18,10 +19,14 @@ fi # byte in the file is invalid UTF-8, while excluding permitted TAB/LF/CR bytes. pattern='[\x00-\x08\x0B\x0C\x0E-\x1F]|\xC2(?:\xA0|\xAD)|\xE2\x80[\x8B-\x8F\xAA-\xAF]|\xE2\x81(?:\xA0|[\xA6-\xA9])|\xEF\xBB\xBF' blocking_pattern='[\x00-\x08\x0B\x0C\x0E-\x1F]' +leading_bom_pattern='\A\xEF\xBB\xBF' : > "$results_file" || exit 2 if [[ -n "$blocking_results_file" ]]; then : > "$blocking_results_file" || exit 2 fi +if [[ -n "$leading_bom_results_file" ]]; then + : > "$leading_bom_results_file" || exit 2 +fi scan_error=0 enumeration_file="$(mktemp /tmp/rsr-invisible-files.XXXXXX)" || exit 2 # Invoked indirectly by the EXIT trap. @@ -61,6 +66,15 @@ while IFS= read -r -d '' filepath; do *) echo "blocking-classifier error ($blocking_status): $filepath" >&2; scan_error=1 ;; esac fi + if [[ -n "$leading_bom_results_file" ]]; then + LC_ALL=C "$grep_bin" -aPzoq "$leading_bom_pattern" "$filepath" + leading_bom_status=$? + case "$leading_bom_status" in + 0) printf '%s\0' "$filepath" >> "$leading_bom_results_file" || scan_error=1 ;; + 1) ;; + *) echo "leading-bom-classifier error ($leading_bom_status): $filepath" >&2; scan_error=1 ;; + esac + fi ;; 1) ;; *) echo "scanner error ($status): $filepath" >&2; scan_error=1 ;; diff --git a/src/fairness_nodocs.jl b/src/fairness_nodocs.jl index b0cbdcd..c5e3a62 100644 --- a/src/fairness_nodocs.jl +++ b/src/fairness_nodocs.jl @@ -1,6 +1,33 @@ # SPDX-License-Identifier: MPL-2.0 # Copyright (c) 2026 Jonathan D.A. Jewell +""" + demographic_parity(predictions::AbstractVector, protected_attributes::AbstractVector)::Float64 + +Compute the demographic parity disparity across protected groups. + +Demographic parity requires that the positive prediction rate (the proportion of +individuals receiving a favorable outcome) is equal across all protected groups. +This function calculates the maximum difference in positive prediction rates +between any two groups, with a value of 0.0 indicating perfect parity. + +# Arguments +- `predictions::AbstractVector`: Binary predictions or scores for each individual. +- `protected_attributes::AbstractVector`: Group membership for each individual + (e.g., gender, race). + +# Returns +- `Float64`: The maximum difference in positive prediction rates between groups. + Returns 0.0 if there are fewer than 2 groups. + +# Example + +```julia +predictions = [1, 0, 1, 1, 0, 1] +protected = [:A, :A, :B, :B, :B, :A] +disparity = demographic_parity(predictions, protected) +``` +""" function demographic_parity(predictions::AbstractVector, protected_attributes::AbstractVector)::Float64 @assert length(predictions) == length(protected_attributes) "Lengths must match." unique_groups = unique(protected_attributes) @@ -17,6 +44,38 @@ function demographic_parity(predictions::AbstractVector, protected_attributes::A return maximum(rates) - minimum(rates) end +""" + equalized_odds(predictions::AbstractVector{<:Real}, labels::AbstractVector{<:Real}, + protected_attributes::AbstractVector)::Float64 + +Compute the equalized odds disparity across protected groups. + +Equalized odds requires that both the true positive rate (TPR) and false positive +rate (FPR) are equal across all protected groups. This function calculates the +maximum disparity in either TPR or FPR between groups, with a value of 0.0 +indicating perfect equalized odds. + +# Arguments +- `predictions::AbstractVector{<:Real}`: Binary predictions (typically 0 or 1) for + each individual. +- `labels::AbstractVector{<:Real}`: True binary labels (typically 0 or 1) for each + individual. +- `protected_attributes::AbstractVector`: Group membership for each individual + (e.g., gender, race). + +# Returns +- `Float64`: The maximum disparity in TPR or FPR between groups. Returns 0.0 if + there are fewer than 2 groups. + +# Example + +```julia +predictions = [1, 0, 1, 1, 0, 1] +labels = [1, 0, 0, 1, 0, 1] +protected = [:A, :A, :B, :B, :B, :A] +disparity = equalized_odds(predictions, labels, protected) +``` +""" function equalized_odds(predictions::AbstractVector{<:Real}, labels::AbstractVector{<:Real}, protected_attributes::AbstractVector)::Float64 @assert length(predictions) == length(labels) == length(protected_attributes) "Lengths must match." @@ -44,6 +103,39 @@ function equalized_odds(predictions::AbstractVector{<:Real}, labels::AbstractVec return max(max_tpr_disparity, max_fpr_disparity) end +""" + equal_opportunity(predictions::AbstractVector{<:Real}, labels::AbstractVector{<:Real}, + protected_attributes::AbstractVector)::Float64 + +Compute the equal opportunity disparity across protected groups. + +Equal opportunity is a weaker form of equalized odds that focuses only on the +true positive rate (TPR). It requires that individuals who truly deserve a +positive outcome have an equal chance of receiving it, regardless of their +protected group membership. This function calculates the maximum difference in +TPR between groups, with a value of 0.0 indicating perfect equal opportunity. + +# Arguments +- `predictions::AbstractVector{<:Real}`: Binary predictions (typically 0 or 1) for + each individual. +- `labels::AbstractVector{<:Real}`: True binary labels (typically 0 or 1) for each + individual. +- `protected_attributes::AbstractVector`: Group membership for each individual + (e.g., gender, race). + +# Returns +- `Float64`: The maximum difference in TPR between groups. Returns 0.0 if there + are fewer than 2 groups. + +# Example + +```julia +predictions = [1, 0, 1, 1, 0, 1] +labels = [1, 0, 0, 1, 0, 1] +protected = [:A, :A, :B, :B, :B, :A] +disparity = equal_opportunity(predictions, labels, protected) +``` +""" function equal_opportunity(predictions::AbstractVector{<:Real}, labels::AbstractVector{<:Real}, protected_attributes::AbstractVector)::Float64 @assert length(predictions) == length(labels) == length(protected_attributes) "Lengths must match." @@ -64,6 +156,36 @@ function equal_opportunity(predictions::AbstractVector{<:Real}, labels::Abstract return maximum(tprs) - minimum(tprs) end +""" + disparate_impact(predictions::AbstractVector, protected_attributes::AbstractVector)::Float64 + +Compute the disparate impact ratio across protected groups. + +Disparate impact measures whether the selection rate (positive prediction rate) +for a protected group is substantially less than for other groups. This function +returns the ratio of the minimum selection rate to the maximum selection rate +across all groups. A value of 1.0 indicates no disparate impact, while values +closer to 0.0 indicate greater disparity. The "80% rule" commonly used in hiring +suggests that a ratio below 0.8 may indicate adverse impact. + +# Arguments +- `predictions::AbstractVector`: Binary predictions or scores for each individual. +- `protected_attributes::AbstractVector`: Group membership for each individual + (e.g., gender, race). + +# Returns +- `Float64`: The ratio of minimum to maximum selection rates across groups. + Returns 1.0 if there are fewer than 2 groups or if the maximum rate is 0.0. + +# Example + +```julia +predictions = [1, 0, 1, 1, 0, 1] +protected = [:A, :A, :B, :B, :B, :A] +ratio = disparate_impact(predictions, protected) +# A ratio < 0.8 may indicate disparate impact under the 80% rule +``` +""" function disparate_impact(predictions::AbstractVector, protected_attributes::AbstractVector)::Float64 @assert length(predictions) == length(protected_attributes) "Lengths must match." unique_groups = unique(protected_attributes) @@ -82,6 +204,41 @@ function disparate_impact(predictions::AbstractVector, protected_attributes::Abs return max_rate > 0.0 ? min_rate / max_rate : 1.0 end +""" + individual_fairness(predictions::AbstractVector, similarity_matrix::AbstractMatrix; + similarity_threshold::Float64 = 0.8)::Float64 + +Compute the individual fairness metric based on similarity between individuals. + +Individual fairness requires that similar individuals receive similar treatment. +This function measures the average absolute difference in predictions between +pairs of individuals whose similarity exceeds a given threshold. Lower values +indicate better individual fairness (more similar predictions for similar +individuals). + +# Arguments +- `predictions::AbstractVector`: Predictions or scores for each individual. +- `similarity_matrix::AbstractMatrix`: An n×n matrix where `similarity_matrix[i, j]` + indicates the similarity between individuals i and j. + Values should typically be in [0, 1]. +- `similarity_threshold::Float64`: The minimum similarity required for two individuals + to be considered "similar" and compared. Defaults to 0.8. + +# Returns +- `Float64`: The average absolute difference in predictions between similar individuals. + Returns 0.0 if no pairs of individuals exceed the similarity threshold. + +# Example + +```julia +predictions = [0.8, 0.2, 0.7, 0.3] +similarity = [1.0 0.9 0.1 0.2; + 0.9 1.0 0.2 0.1; + 0.1 0.2 1.0 0.85; + 0.2 0.1 0.85 1.0] +fairness = individual_fairness(predictions, similarity, similarity_threshold=0.85) +``` +""" function individual_fairness(predictions::AbstractVector, similarity_matrix::AbstractMatrix; similarity_threshold::Float64 = 0.8)::Float64 n = length(predictions) diff --git a/tests/invisible-characters-test.sh b/tests/invisible-characters-test.sh index 91f02bf..4ae1335 100755 --- a/tests/invisible-characters-test.sh +++ b/tests/invisible-characters-test.sh @@ -15,6 +15,7 @@ trap cleanup EXIT scanner="$repo_root/scripts/check-invisible-characters.sh" results="$fixture_root/results.bin" blocking_results="$fixture_root/blocking-results.bin" +leading_bom_results="$fixture_root/leading-bom-results.bin" fixtures="$fixture_root/fixtures" mkdir -p "$fixtures" @@ -24,14 +25,16 @@ printf 'soft-hyphen:\302\255\n' > "$fixtures/soft-hyphen.adoc" printf 'zero-width:\342\200\213\n' > "$fixtures/zero-width.json" printf 'bidi:\342\200\256\n' > "$fixtures/bidi.toml" printf 'word-joiner:\342\201\240\n' > "$fixtures/word-joiner.yml" -printf '\357\273\277leading bom\n' > "$fixtures/bom.sh" +printf '\357\273\277leading bom\n' > "$fixtures/bom-leading.sh" +printf 'mid\357\273\277bom\n' > "$fixtures/bom-mid.sh" +printf 'first\n\357\273\277second\n' > "$fixtures/bom-after-newline.sh" printf 'nul:\000byte\n' > "$fixtures/nul.rs" printf 'backspace:\010byte\n' > "$fixtures/backspace.rs" printf 'invalid:\377 then nbsp:\302\240\n' > "$fixtures/invalid-utf8.md" printf 'newline name:\302\240\n' > "$fixtures/with newline.md" -"$scanner" "$fixtures" "$results" "$blocking_results" +"$scanner" "$fixtures" "$results" "$blocking_results" "$leading_bom_results" count=0 safe_seen=false @@ -42,8 +45,8 @@ while IFS= read -r -d '' filepath; do [[ "$filepath" == "$fixtures/with"$'\n'"newline.md" ]] && newline_seen=true done < "$results" -[[ "$count" -eq 10 ]] || { - echo "expected 10 findings, got $count" >&2 +[[ "$count" -eq 12 ]] || { + echo "expected 12 findings, got $count" >&2 exit 1 } [[ "$safe_seen" == false ]] || { @@ -68,6 +71,21 @@ done < "$blocking_results" exit 1 } +leading_bom_count=0 +leading_bom_found=false +mid_bom_found=false +bom_after_newline_found=false +while IFS= read -r -d '' filepath; do + leading_bom_count=$((leading_bom_count + 1)) + [[ "$filepath" == "$fixtures/bom-leading.sh" ]] && leading_bom_found=true + [[ "$filepath" == "$fixtures/bom-mid.sh" ]] && mid_bom_found=true + [[ "$filepath" == "$fixtures/bom-after-newline.sh" ]] && bom_after_newline_found=true +done < "$leading_bom_results" +[[ "$leading_bom_count" -eq 1 && "$leading_bom_found" == true && "$mid_bom_found" == false && "$bom_after_newline_found" == false ]] || { + echo "expected only leading BOM fixture in the leading-bom set, got $leading_bom_count (leading: $leading_bom_found, mid: $mid_bom_found, after-newline: $bom_after_newline_found)" >&2 + exit 1 +} + if "$scanner" "$fixture_root/missing" "$results"; then echo "missing scan root did not fail closed" >&2 exit 1