fix(ci): the invisible-character gate never matched anything

.github/workflows/dogfood-gate.yml

@@ -110,37 +110,48 @@ jobs:
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       - name: Scan for invisible characters
         id: lint
+        shell: bash
         run: |
-          # Inline invisible character detection (from empty-linter's core patterns).
-          # Checks for: zero-width spaces, zero-width joiners, BOM, soft hyphens,
-          # non-breaking spaces, null bytes, and other invisible Unicode in source files.
-          set +e
-          PATTERNS='\xc2\xa0|\xe2\x80\x8b|\xe2\x80\x8c|\xe2\x80\x8d|\xef\xbb\xbf|\xc2\xad|\xe2\x80\x8e|\xe2\x80\x8f|\xe2\x80\xaa|\xe2\x80\xab|\xe2\x80\xac|\xe2\x80\xad|\xe2\x80\xae|\x00'
-          find "$GITHUB_WORKSPACE" \
-            -not -path '*/.git/*' -not -path '*/node_modules/*' \
-            -not -path '*/.deno/*' -not -path '*/target/*' \
-            -not -path '*/_build/*' -not -path '*/deps/*' \
-            -not -path '*/external_corpora/*' -not -path '*/.lake/*' \
-            -type f \( -name '*.rs' -o -name '*.ex' -o -name '*.exs' -o -name '*.res' \
-              -o -name '*.js' -o -name '*.ts' -o -name '*.json' -o -name '*.toml' \
-              -o -name '*.yml' -o -name '*.yaml' -o -name '*.md' -o -name '*.adoc' \
-              -o -name '*.idr' -o -name '*.zig' -o -name '*.v' -o -name '*.jl' \
-              -o -name '*.gleam' -o -name '*.hs' -o -name '*.ml' -o -name '*.sh' \) \
-            -exec grep -Prl "$PATTERNS" {} \; > /tmp/empty-lint-results.txt 2>/dev/null
-          EL_EXIT=$?
-          set -e
+          RESULTS_FILE="$RUNNER_TEMP/empty-lint-results.bin"
+          BLOCKING_FILE="$RUNNER_TEMP/empty-lint-blocking-results.bin"
+          if ! scripts/check-invisible-characters.sh \
+              "$GITHUB_WORKSPACE" "$RESULTS_FILE" "$BLOCKING_FILE"; then
+            echo "::error::Invisible-character scanner failed; refusing a partial pass"
+            exit 2
+          fi
+
+          FINDINGS=0
+          while IFS= read -r -d '' filepath; do
+            FINDINGS=$((FINDINGS + 1))
+            REL_PATH="${filepath#"$GITHUB_WORKSPACE"/}"
+            SAFE_PATH="${REL_PATH//'%'/'%25'}"
+            SAFE_PATH="${SAFE_PATH//$'\r'/'%0D'}"
+            SAFE_PATH="${SAFE_PATH//$'\n'/'%0A'}"
+            SAFE_PATH="${SAFE_PATH//':'/'%3A'}"
+            SAFE_PATH="${SAFE_PATH//','/'%2C'}"
+            echo "::warning file=${SAFE_PATH}::Invisible Unicode or C0 characters detected"
+          done < "$RESULTS_FILE"
+
+          BLOCKING=0
+          while IFS= read -r -d '' filepath; do
+            BLOCKING=$((BLOCKING + 1))
+            REL_PATH="${filepath#"$GITHUB_WORKSPACE"/}"
+            SAFE_PATH="${REL_PATH//'%'/'%25'}"
+            SAFE_PATH="${SAFE_PATH//$'\r'/'%0D'}"
+            SAFE_PATH="${SAFE_PATH//$'\n'/'%0A'}"
+            SAFE_PATH="${SAFE_PATH//':'/'%3A'}"
+            SAFE_PATH="${SAFE_PATH//','/'%2C'}"
+            echo "::error file=${SAFE_PATH}::C0 control character or NUL byte detected"
+          done < "$BLOCKING_FILE"
 
-          FINDINGS=$(wc -l < /tmp/empty-lint-results.txt 2>/dev/null || echo 0)
           echo "findings=$FINDINGS" >> "$GITHUB_OUTPUT"
-          echo "exit_code=$EL_EXIT" >> "$GITHUB_OUTPUT"
+          echo "blocking=$BLOCKING" >> "$GITHUB_OUTPUT"
           echo "ready=true" >> "$GITHUB_OUTPUT"
 
-          # Emit annotations for each file with invisible chars
-          while IFS= read -r filepath; do
-            [ -z "$filepath" ] && continue
-            REL_PATH="${filepath#$GITHUB_WORKSPACE/}"
-            echo "::warning file=${REL_PATH}::Invisible Unicode characters detected (zero-width space, BOM, NBSP, etc.)"
-          done < /tmp/empty-lint-results.txt
+          if [ "$BLOCKING" -gt 0 ]; then
+            echo "## Empty-linter: BLOCKED — $BLOCKING file(s) contain C0/NUL corruption" >> "$GITHUB_STEP_SUMMARY"
+            exit 1
+          fi
       - name: Write summary
         run: |
           if [ "${{ steps.lint.outputs.ready }}" = "true" ]; then

scripts/check-invisible-characters.sh

@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: MPL-2.0
+# Byte-safe scanner for invisible Unicode encodings and forbidden C0 controls.
+set -u
+
+scan_root="${1:-}"
+results_file="${2:-}"
+blocking_results_file="${3:-}"
+leading_bom_results_file="${4:-}"
+grep_bin="${INVISIBLE_GREP_BIN:-grep}"
+find_bin="${INVISIBLE_FIND_BIN:-find}"
+
+if [[ -z "$scan_root" || ! -d "$scan_root" || -z "$results_file" ]]; then
+  echo "usage: $0 SCAN_ROOT RESULTS_FILE [BLOCKING_RESULTS_FILE] [LEADING_BOM_RESULTS_FILE]" >&2
+  exit 2
+fi
+
+# Scan bytes under the C locale. This detects UTF-8 encodings even when another
+# byte in the file is invalid UTF-8, while excluding permitted TAB/LF/CR bytes.
+pattern='[\x00-\x08\x0B\x0C\x0E-\x1F]|\xC2(?:\xA0|\xAD)|\xE2\x80[\x8B-\x8F\xAA-\xAF]|\xE2\x81(?:\xA0|[\xA6-\xA9])|\xEF\xBB\xBF'
+blocking_pattern='[\x00-\x08\x0B\x0C\x0E-\x1F]'
+leading_bom_pattern='\A\xEF\xBB\xBF'
+: > "$results_file" || exit 2
+if [[ -n "$blocking_results_file" ]]; then
+  : > "$blocking_results_file" || exit 2
+fi
+if [[ -n "$leading_bom_results_file" ]]; then
+  : > "$leading_bom_results_file" || exit 2
+fi
+scan_error=0
+enumeration_file="$(mktemp /tmp/rsr-invisible-files.XXXXXX)" || exit 2
+# Invoked indirectly by the EXIT trap.
+# shellcheck disable=SC2329
+cleanup() {
+  rm -f -- "$enumeration_file"
+}
+trap cleanup EXIT
+
+if ! "$find_bin" "$scan_root" \
+    -not -path '*/.git/*' -not -path '*/node_modules/*' \
+    -not -path '*/.deno/*' -not -path '*/target/*' \
+    -not -path '*/_build/*' -not -path '*/deps/*' \
+    -not -path '*/external_corpora/*' -not -path '*/.lake/*' \
+    -type f \( -name '*.rs' -o -name '*.ex' -o -name '*.exs' -o -name '*.res' \
+      -o -name '*.js' -o -name '*.ts' -o -name '*.json' -o -name '*.toml' \
+      -o -name '*.yml' -o -name '*.yaml' -o -name '*.md' -o -name '*.adoc' \
+      -o -name '*.idr' -o -name '*.zig' -o -name '*.v' -o -name '*.jl' \
+      -o -name '*.gleam' -o -name '*.hs' -o -name '*.ml' -o -name '*.sh' \) \
+    -print0 > "$enumeration_file"; then
+  echo "file enumeration failed: $scan_root" >&2
+  exit 1
+fi
+
+while IFS= read -r -d '' filepath; do
+  LC_ALL=C "$grep_bin" -aPq "$pattern" "$filepath"
+  status=$?
+  case "$status" in
+    0)
+      printf '%s\0' "$filepath" >> "$results_file" || scan_error=1
+      if [[ -n "$blocking_results_file" ]]; then
+        LC_ALL=C "$grep_bin" -aPq "$blocking_pattern" "$filepath"
+        blocking_status=$?
+        case "$blocking_status" in
+          0) printf '%s\0' "$filepath" >> "$blocking_results_file" || scan_error=1 ;;
+          1) ;;
+          *) echo "blocking-classifier error ($blocking_status): $filepath" >&2; scan_error=1 ;;
+        esac
+      fi
+      if [[ -n "$leading_bom_results_file" ]]; then
+        LC_ALL=C "$grep_bin" -aPzoq "$leading_bom_pattern" "$filepath"
+        leading_bom_status=$?
+        case "$leading_bom_status" in
+          0) printf '%s\0' "$filepath" >> "$leading_bom_results_file" || scan_error=1 ;;
+          1) ;;
+          *) echo "leading-bom-classifier error ($leading_bom_status): $filepath" >&2; scan_error=1 ;;
+        esac
+      fi
+      ;;
+    1) ;;
+    *) echo "scanner error ($status): $filepath" >&2; scan_error=1 ;;
+  esac
+done < "$enumeration_file"
+
+exit "$scan_error"

src/fairness_nodocs.jl

@@ -1,6 +1,33 @@
 # SPDX-License-Identifier: MPL-2.0
 # Copyright (c) 2026 Jonathan D.A. Jewell <j.d.a.jewell@open.ac.uk>
 
+"""
+    demographic_parity(predictions::AbstractVector, protected_attributes::AbstractVector)::Float64
+
+Compute the demographic parity disparity across protected groups.
+
+Demographic parity requires that the positive prediction rate (the proportion of
+individuals receiving a favorable outcome) is equal across all protected groups.
+This function calculates the maximum difference in positive prediction rates
+between any two groups, with a value of 0.0 indicating perfect parity.
+
+# Arguments
+- `predictions::AbstractVector`: Binary predictions or scores for each individual.
+- `protected_attributes::AbstractVector`: Group membership for each individual
+                                         (e.g., gender, race).
+
+# Returns
+- `Float64`: The maximum difference in positive prediction rates between groups.
+             Returns 0.0 if there are fewer than 2 groups.
+
+# Example
+
+```julia
+predictions = [1, 0, 1, 1, 0, 1]
+protected = [:A, :A, :B, :B, :B, :A]
+disparity = demographic_parity(predictions, protected)
+```
+"""
 function demographic_parity(predictions::AbstractVector, protected_attributes::AbstractVector)::Float64
     @assert length(predictions) == length(protected_attributes) "Lengths must match."
     unique_groups = unique(protected_attributes)
@@ -17,6 +44,38 @@ function demographic_parity(predictions::AbstractVector, protected_attributes::A
     return maximum(rates) - minimum(rates)
 end
 
+"""
+    equalized_odds(predictions::AbstractVector{<:Real}, labels::AbstractVector{<:Real},
+                   protected_attributes::AbstractVector)::Float64
+
+Compute the equalized odds disparity across protected groups.
+
+Equalized odds requires that both the true positive rate (TPR) and false positive
+rate (FPR) are equal across all protected groups. This function calculates the
+maximum disparity in either TPR or FPR between groups, with a value of 0.0
+indicating perfect equalized odds.
+
+# Arguments
+- `predictions::AbstractVector{<:Real}`: Binary predictions (typically 0 or 1) for
+                                         each individual.
+- `labels::AbstractVector{<:Real}`: True binary labels (typically 0 or 1) for each
+                                    individual.
+- `protected_attributes::AbstractVector`: Group membership for each individual
+                                         (e.g., gender, race).
+
+# Returns
+- `Float64`: The maximum disparity in TPR or FPR between groups. Returns 0.0 if
+             there are fewer than 2 groups.
+
+# Example
+
+```julia
+predictions = [1, 0, 1, 1, 0, 1]
+labels = [1, 0, 0, 1, 0, 1]
+protected = [:A, :A, :B, :B, :B, :A]
+disparity = equalized_odds(predictions, labels, protected)
+```
+"""
 function equalized_odds(predictions::AbstractVector{<:Real}, labels::AbstractVector{<:Real},
                         protected_attributes::AbstractVector)::Float64
     @assert length(predictions) == length(labels) == length(protected_attributes) "Lengths must match."
@@ -44,6 +103,39 @@ function equalized_odds(predictions::AbstractVector{<:Real}, labels::AbstractVec
     return max(max_tpr_disparity, max_fpr_disparity)
 end
 
+"""
+    equal_opportunity(predictions::AbstractVector{<:Real}, labels::AbstractVector{<:Real},
+                     protected_attributes::AbstractVector)::Float64
+
+Compute the equal opportunity disparity across protected groups.
+
+Equal opportunity is a weaker form of equalized odds that focuses only on the
+true positive rate (TPR). It requires that individuals who truly deserve a
+positive outcome have an equal chance of receiving it, regardless of their
+protected group membership. This function calculates the maximum difference in
+TPR between groups, with a value of 0.0 indicating perfect equal opportunity.
+
+# Arguments
+- `predictions::AbstractVector{<:Real}`: Binary predictions (typically 0 or 1) for
+                                         each individual.
+- `labels::AbstractVector{<:Real}`: True binary labels (typically 0 or 1) for each
+                                    individual.
+- `protected_attributes::AbstractVector`: Group membership for each individual
+                                         (e.g., gender, race).
+
+# Returns
+- `Float64`: The maximum difference in TPR between groups. Returns 0.0 if there
+             are fewer than 2 groups.
+
+# Example
+
+```julia
+predictions = [1, 0, 1, 1, 0, 1]
+labels = [1, 0, 0, 1, 0, 1]
+protected = [:A, :A, :B, :B, :B, :A]
+disparity = equal_opportunity(predictions, labels, protected)
+```
+"""
 function equal_opportunity(predictions::AbstractVector{<:Real}, labels::AbstractVector{<:Real},
                           protected_attributes::AbstractVector)::Float64
     @assert length(predictions) == length(labels) == length(protected_attributes) "Lengths must match."
@@ -64,6 +156,36 @@ function equal_opportunity(predictions::AbstractVector{<:Real}, labels::Abstract
     return maximum(tprs) - minimum(tprs)
 end
 
+"""
+    disparate_impact(predictions::AbstractVector, protected_attributes::AbstractVector)::Float64
+
+Compute the disparate impact ratio across protected groups.
+
+Disparate impact measures whether the selection rate (positive prediction rate)
+for a protected group is substantially less than for other groups. This function
+returns the ratio of the minimum selection rate to the maximum selection rate
+across all groups. A value of 1.0 indicates no disparate impact, while values
+closer to 0.0 indicate greater disparity. The "80% rule" commonly used in hiring
+suggests that a ratio below 0.8 may indicate adverse impact.
+
+# Arguments
+- `predictions::AbstractVector`: Binary predictions or scores for each individual.
+- `protected_attributes::AbstractVector`: Group membership for each individual
+                                         (e.g., gender, race).
+
+# Returns
+- `Float64`: The ratio of minimum to maximum selection rates across groups.
+             Returns 1.0 if there are fewer than 2 groups or if the maximum rate is 0.0.
+
+# Example
+
+```julia
+predictions = [1, 0, 1, 1, 0, 1]
+protected = [:A, :A, :B, :B, :B, :A]
+ratio = disparate_impact(predictions, protected)
+# A ratio < 0.8 may indicate disparate impact under the 80% rule
+```
+"""
 function disparate_impact(predictions::AbstractVector, protected_attributes::AbstractVector)::Float64
     @assert length(predictions) == length(protected_attributes) "Lengths must match."
     unique_groups = unique(protected_attributes)
@@ -82,6 +204,41 @@ function disparate_impact(predictions::AbstractVector, protected_attributes::Abs
     return max_rate > 0.0 ? min_rate / max_rate : 1.0
 end
 
+"""
+    individual_fairness(predictions::AbstractVector, similarity_matrix::AbstractMatrix;
+                       similarity_threshold::Float64 = 0.8)::Float64
+
+Compute the individual fairness metric based on similarity between individuals.
+
+Individual fairness requires that similar individuals receive similar treatment.
+This function measures the average absolute difference in predictions between
+pairs of individuals whose similarity exceeds a given threshold. Lower values
+indicate better individual fairness (more similar predictions for similar
+individuals).
+
+# Arguments
+- `predictions::AbstractVector`: Predictions or scores for each individual.
+- `similarity_matrix::AbstractMatrix`: An n×n matrix where `similarity_matrix[i, j]`
+                                       indicates the similarity between individuals i and j.
+                                       Values should typically be in [0, 1].
+- `similarity_threshold::Float64`: The minimum similarity required for two individuals
+                                   to be considered "similar" and compared. Defaults to 0.8.
+
+# Returns
+- `Float64`: The average absolute difference in predictions between similar individuals.
+             Returns 0.0 if no pairs of individuals exceed the similarity threshold.
+
+# Example
+
+```julia
+predictions = [0.8, 0.2, 0.7, 0.3]
+similarity = [1.0 0.9 0.1 0.2;
+              0.9 1.0 0.2 0.1;
+              0.1 0.2 1.0 0.85;
+              0.2 0.1 0.85 1.0]
+fairness = individual_fairness(predictions, similarity, similarity_threshold=0.85)
+```
+"""
 function individual_fairness(predictions::AbstractVector, similarity_matrix::AbstractMatrix;
                             similarity_threshold::Float64 = 0.8)::Float64
     n = length(predictions)