From 02a5baebaaf39ee4e26c967d32710c1917bcb7fd Mon Sep 17 00:00:00 2001 From: "Jonathan D.A. Jewell" <6759885+hyperpolymath@users.noreply.github.com> Date: Thu, 27 Aug 2026 14:49:39 +0100 Subject: [PATCH 1/5] fix(ci): the invisible-character gate never matched anything MEASURED 2026-08-27: this gate's pattern caught 0 OF 6 invisible-character test cases. It has never detected an NBSP, zero-width space, BOM, soft hyphen, bidi override or word joiner. ROOT CAUSE: the pattern used UTF-8 BYTE sequences (\xc2\xa0) while grep -P matches CHARACTERS. Bytes c2 a0 are ONE character U+00A0; \xc2\xa0 asks for TWO characters, U+00C2 then U+00A0, which is never present. grep -P '\xc2\xa0' -> miss grep -P '\x{a0}' -> MATCH Only \x00 worked, being single-byte in both readings. FIXED: codepoint escapes; C0 control characters \x01-\x08,\x0B,\x0C,\x0E-\x1F added (TAB/LF/CR excluded); and grep -a, without which grep skips any NUL-bearing file as binary. The C0 range matters: a stray BACKSPACE byte made a workflow unparseable in developer-ecosystem, so it never ran, and this linter called it clean. Canonical fix: hyperpolymath/empty-linter#70. 1 file(s) here. VERIFIED: YAML re-parsed, and the corrected pattern was confirmed to catch a real NBSP before the change was kept. --- .github/workflows/dogfood-gate.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dogfood-gate.yml b/.github/workflows/dogfood-gate.yml index 341fb4f..a134dff 100644 --- a/.github/workflows/dogfood-gate.yml +++ b/.github/workflows/dogfood-gate.yml @@ -110,7 +110,7 @@ jobs: # Checks for: zero-width spaces, zero-width joiners, BOM, soft hyphens, # non-breaking spaces, null bytes, and other invisible Unicode in source files. set +e - PATTERNS='\xc2\xa0|\xe2\x80\x8b|\xe2\x80\x8c|\xe2\x80\x8d|\xef\xbb\xbf|\xc2\xad|\xe2\x80\x8e|\xe2\x80\x8f|\xe2\x80\xaa|\xe2\x80\xab|\xe2\x80\xac|\xe2\x80\xad|\xe2\x80\xae|\x00' + PATTERNS='\x00|[\x01-\x08\x0B\x0C\x0E-\x1F]|\x{a0}|\x{ad}|\x{200b}|\x{200c}|\x{200d}|\x{200e}|\x{200f}|\x{202a}|\x{202b}|\x{202c}|\x{202d}|\x{202e}|\x{2060}|\x{feff}' find "$GITHUB_WORKSPACE" \ -not -path '*/.git/*' -not -path '*/node_modules/*' \ -not -path '*/.deno/*' -not -path '*/target/*' \ @@ -121,7 +121,7 @@ jobs: -o -name '*.yml' -o -name '*.yaml' -o -name '*.md' -o -name '*.adoc' \ -o -name '*.idr' -o -name '*.zig' -o -name '*.v' -o -name '*.jl' \ -o -name '*.gleam' -o -name '*.hs' -o -name '*.ml' -o -name '*.sh' \) \ - -exec grep -Prl "$PATTERNS" {} \; > /tmp/empty-lint-results.txt 2>/dev/null + -exec grep -aPrl "$PATTERNS" {} \; > /tmp/empty-lint-results.txt 2>/dev/null EL_EXIT=$? set -e From 32973dbfad8efa369c9b9fe6062bcd69ad5bfafd Mon Sep 17 00:00:00 2001 From: "Jonathan D.A. Jewell" <6759885+hyperpolymath@users.noreply.github.com> Date: Fri, 28 Aug 2026 16:12:06 +0100 Subject: [PATCH 2/5] fix(ci): make invisible-character PCRE locale-independent --- .github/workflows/dogfood-gate.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dogfood-gate.yml b/.github/workflows/dogfood-gate.yml index a134dff..e5c46a9 100644 --- a/.github/workflows/dogfood-gate.yml +++ b/.github/workflows/dogfood-gate.yml @@ -110,7 +110,7 @@ jobs: # Checks for: zero-width spaces, zero-width joiners, BOM, soft hyphens, # non-breaking spaces, null bytes, and other invisible Unicode in source files. set +e - PATTERNS='\x00|[\x01-\x08\x0B\x0C\x0E-\x1F]|\x{a0}|\x{ad}|\x{200b}|\x{200c}|\x{200d}|\x{200e}|\x{200f}|\x{202a}|\x{202b}|\x{202c}|\x{202d}|\x{202e}|\x{2060}|\x{feff}' + PATTERNS='(*UTF)[\x00-\x08\x0B\x0C\x0E-\x1F\x{a0}\x{ad}\x{200b}-\x{200f}\x{202a}-\x{202f}\x{2060}\x{2066}-\x{2069}\x{feff}]' find "$GITHUB_WORKSPACE" \ -not -path '*/.git/*' -not -path '*/node_modules/*' \ -not -path '*/.deno/*' -not -path '*/target/*' \ From 17781da5782f2145546baccb0591fb0e3bec93a9 Mon Sep 17 00:00:00 2001 From: "Jonathan D.A. Jewell" <6759885+hyperpolymath@users.noreply.github.com> Date: Fri, 28 Aug 2026 19:55:08 +0100 Subject: [PATCH 3/5] Update .github/workflows/dogfood-gate.yml Co-authored-by: codacy-production[bot] <61871480+codacy-production[bot]@users.noreply.github.com> Signed-off-by: Jonathan D.A. Jewell <6759885+hyperpolymath@users.noreply.github.com> --- .github/workflows/dogfood-gate.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dogfood-gate.yml b/.github/workflows/dogfood-gate.yml index e5c46a9..9c5df68 100644 --- a/.github/workflows/dogfood-gate.yml +++ b/.github/workflows/dogfood-gate.yml @@ -121,7 +121,7 @@ jobs: -o -name '*.yml' -o -name '*.yaml' -o -name '*.md' -o -name '*.adoc' \ -o -name '*.idr' -o -name '*.zig' -o -name '*.v' -o -name '*.jl' \ -o -name '*.gleam' -o -name '*.hs' -o -name '*.ml' -o -name '*.sh' \) \ - -exec grep -aPrl "$PATTERNS" {} \; > /tmp/empty-lint-results.txt 2>/dev/null + -exec grep -aPl "$PATTERNS" {} + > /tmp/empty-lint-results.txt EL_EXIT=$? set -e From 979297b0410b579197daad40976298ba4dfbf103 Mon Sep 17 00:00:00 2001 From: "Jonathan D.A. Jewell" <6759885+hyperpolymath@users.noreply.github.com> Date: Sat, 29 Aug 2026 13:24:30 +0100 Subject: [PATCH 4/5] fix(ci): make invisible scan byte-safe --- .github/workflows/dogfood-gate.yml | 96 ++++++++++++++++++++++-------- 1 file changed, 71 insertions(+), 25 deletions(-) diff --git a/.github/workflows/dogfood-gate.yml b/.github/workflows/dogfood-gate.yml index 9c5df68..ae6ed15 100644 --- a/.github/workflows/dogfood-gate.yml +++ b/.github/workflows/dogfood-gate.yml @@ -109,37 +109,83 @@ jobs: # Inline invisible character detection (from empty-linter's core patterns). # Checks for: zero-width spaces, zero-width joiners, BOM, soft hyphens, # non-breaking spaces, null bytes, and other invisible Unicode in source files. - set +e - PATTERNS='(*UTF)[\x00-\x08\x0B\x0C\x0E-\x1F\x{a0}\x{ad}\x{200b}-\x{200f}\x{202a}-\x{202f}\x{2060}\x{2066}-\x{2069}\x{feff}]' - find "$GITHUB_WORKSPACE" \ - -not -path '*/.git/*' -not -path '*/node_modules/*' \ - -not -path '*/.deno/*' -not -path '*/target/*' \ - -not -path '*/_build/*' -not -path '*/deps/*' \ - -not -path '*/external_corpora/*' -not -path '*/.lake/*' \ - -type f \( -name '*.rs' -o -name '*.ex' -o -name '*.exs' -o -name '*.res' \ - -o -name '*.js' -o -name '*.ts' -o -name '*.json' -o -name '*.toml' \ - -o -name '*.yml' -o -name '*.yaml' -o -name '*.md' -o -name '*.adoc' \ - -o -name '*.idr' -o -name '*.zig' -o -name '*.v' -o -name '*.jl' \ - -o -name '*.gleam' -o -name '*.hs' -o -name '*.ml' -o -name '*.sh' \) \ - -exec grep -aPl "$PATTERNS" {} + > /tmp/empty-lint-results.txt - EL_EXIT=$? - set -e + python3 - <<'PY' + import os + from pathlib import Path - FINDINGS=$(wc -l < /tmp/empty-lint-results.txt 2>/dev/null || echo 0) - echo "findings=$FINDINGS" >> "$GITHUB_OUTPUT" - echo "exit_code=$EL_EXIT" >> "$GITHUB_OUTPUT" - echo "ready=true" >> "$GITHUB_OUTPUT" + root = Path(os.environ["GITHUB_WORKSPACE"]) + skipped_dirs = { + ".git", ".deno", ".lake", "_build", "deps", + "external_corpora", "node_modules", "target", + } + source_suffixes = { + ".adoc", ".ex", ".exs", ".gleam", ".hs", ".idr", ".jl", + ".js", ".json", ".md", ".ml", ".res", ".rs", ".sh", + ".toml", ".ts", ".v", ".yaml", ".yml", ".zig", + } + invisible_codepoints = { + 0x00A0, 0x00AD, 0x2060, 0xFEFF, + *range(0x200B, 0x2010), + *range(0x202A, 0x2030), + *range(0x2066, 0x206A), + } - # Emit annotations for each file with invisible chars - while IFS= read -r filepath; do - [ -z "$filepath" ] && continue - REL_PATH="${filepath#$GITHUB_WORKSPACE/}" - echo "::warning file=${REL_PATH}::Invisible Unicode characters detected (zero-width space, BOM, NBSP, etc.)" - done < /tmp/empty-lint-results.txt + def annotation_escape(value): + return str(value).replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A") + + findings = [] + errors = [] + for path in root.rglob("*"): + relative = path.relative_to(root) + if ( + path.is_symlink() + or not path.is_file() + or path.suffix.lower() not in source_suffixes + or any(part in skipped_dirs for part in relative.parts[:-1]) + ): + continue + try: + data = path.read_bytes() + except OSError as error: + errors.append((relative, f"could not read file: {error}")) + continue + + reasons = set() + if data.startswith(b"\xef\xbb\xbf"): + reasons.add("leading UTF-8 BOM") + if any(byte <= 0x08 or byte in (0x0B, 0x0C) or 0x0E <= byte <= 0x1F for byte in data): + reasons.add("C0 control character") + try: + text_content = data.decode("utf-8", errors="strict") + except UnicodeDecodeError as error: + errors.append((relative, f"invalid UTF-8 at byte {error.start}")) + continue + if any(ord(character) in invisible_codepoints for character in text_content): + reasons.add("invisible Unicode code point") + if reasons: + findings.append((relative, ", ".join(sorted(reasons)))) + + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output: + output.write(f"findings={len(findings)}\n") + output.write(f"exit_code={2 if errors else 0}\n") + output.write("ready=true\n") + + for relative, reasons in findings: + print(f"::warning file={annotation_escape(relative)}::Invisible characters detected: {reasons}") + for relative, reason in errors: + print(f"::error file={annotation_escape(relative)}::Invisible-character scan failed: {annotation_escape(reason)}") + PY - name: Write summary run: | if [ "${{ steps.lint.outputs.ready }}" = "true" ]; then FINDINGS="${{ steps.lint.outputs.findings }}" + EXIT_CODE="${{ steps.lint.outputs.exit_code }}" + if [ "$EXIT_CODE" -ne 0 ] 2>/dev/null; then + echo "## Empty-Linter Results" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo ":x: Scanner execution failed; see error annotations above." >> "$GITHUB_STEP_SUMMARY" + exit 1 + fi if [ "$FINDINGS" -gt 0 ] 2>/dev/null; then echo "## Empty-Linter Results" >> "$GITHUB_STEP_SUMMARY" echo "" >> "$GITHUB_STEP_SUMMARY" From b4036f3f910b10ba8480561fc88b0b0889a871fb Mon Sep 17 00:00:00 2001 From: "Jonathan D.A. Jewell" <6759885+hyperpolymath@users.noreply.github.com> Date: Sat, 29 Aug 2026 13:47:18 +0100 Subject: [PATCH 5/5] fix(ci): enforce invisible-character findings --- .github/workflows/dogfood-gate.yml | 91 ++++++++++++++++++------------ CHANGELOG.adoc | 2 +- TEST-NEEDS.adoc | 2 +- 3 files changed, 58 insertions(+), 37 deletions(-) diff --git a/.github/workflows/dogfood-gate.yml b/.github/workflows/dogfood-gate.yml index ae6ed15..ca81d3f 100644 --- a/.github/workflows/dogfood-gate.yml +++ b/.github/workflows/dogfood-gate.yml @@ -115,12 +115,20 @@ jobs: root = Path(os.environ["GITHUB_WORKSPACE"]) skipped_dirs = { - ".git", ".deno", ".lake", "_build", "deps", - "external_corpora", "node_modules", "target", + ".cache", ".deno", ".elixir_ls", ".git", ".lake", ".zig-cache", + "_build", "build", "coverage", "deps", "dist", "external_corpora", + "node_modules", "out", "target", "vendor", "zig-cache", "zig-out", + } + intentional_fixture_dirs = { + ("tests", "fixtures", "bom-detection"), + ("tests", "fixtures", "empty-linter"), } source_suffixes = { - ".adoc", ".ex", ".exs", ".gleam", ".hs", ".idr", ".jl", - ".js", ".json", ".md", ".ml", ".res", ".rs", ".sh", + ".adoc", ".adb", ".ads", ".agda", ".c", ".cc", ".clj", ".cljs", + ".cpp", ".erl", ".ex", ".exs", ".fs", ".fsi", ".fsx", ".gleam", + ".h", ".hh", ".hpp", ".hrl", ".hs", ".idr", ".java", ".jl", + ".js", ".json", ".kt", ".kts", ".lean", ".lua", ".md", ".ml", + ".php", ".r", ".rb", ".res", ".rs", ".scala", ".sh", ".swift", ".toml", ".ts", ".v", ".yaml", ".yml", ".zig", } invisible_codepoints = { @@ -130,40 +138,52 @@ jobs: *range(0x2066, 0x206A), } - def annotation_escape(value): + def command_escape(value): return str(value).replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A") + def property_escape(value): + return command_escape(value).replace(":", "%3A").replace(",", "%2C") + + # Runtime regression for GitHub workflow-command property delimiters. + assert property_escape("docs/a,b::c.md") == "docs/a%2Cb%3A%3Ac.md" + + def intentionally_invalid_fixture(relative): + return any(relative.parts[:len(prefix)] == prefix for prefix in intentional_fixture_dirs) + findings = [] errors = [] - for path in root.rglob("*"): - relative = path.relative_to(root) - if ( - path.is_symlink() - or not path.is_file() - or path.suffix.lower() not in source_suffixes - or any(part in skipped_dirs for part in relative.parts[:-1]) - ): - continue - try: - data = path.read_bytes() - except OSError as error: - errors.append((relative, f"could not read file: {error}")) - continue + for directory, dirnames, filenames in os.walk(root, topdown=True): + dirnames[:] = [name for name in dirnames if name not in skipped_dirs] + directory_path = Path(directory) + for filename in filenames: + path = directory_path / filename + relative = path.relative_to(root) + if ( + path.is_symlink() + or path.suffix.lower() not in source_suffixes + or intentionally_invalid_fixture(relative) + ): + continue + try: + data = path.read_bytes() + except OSError as error: + errors.append((relative, f"could not read file: {error}")) + continue - reasons = set() - if data.startswith(b"\xef\xbb\xbf"): - reasons.add("leading UTF-8 BOM") - if any(byte <= 0x08 or byte in (0x0B, 0x0C) or 0x0E <= byte <= 0x1F for byte in data): - reasons.add("C0 control character") - try: - text_content = data.decode("utf-8", errors="strict") - except UnicodeDecodeError as error: - errors.append((relative, f"invalid UTF-8 at byte {error.start}")) - continue - if any(ord(character) in invisible_codepoints for character in text_content): - reasons.add("invisible Unicode code point") - if reasons: - findings.append((relative, ", ".join(sorted(reasons)))) + reasons = set() + if data.startswith(b"\xef\xbb\xbf"): + reasons.add("leading UTF-8 BOM") + if any(byte <= 0x08 or byte in (0x0B, 0x0C) or 0x0E <= byte <= 0x1F for byte in data): + reasons.add("C0 control character") + try: + text_content = data.decode("utf-8", errors="strict") + except UnicodeDecodeError as error: + errors.append((relative, f"invalid UTF-8 at byte {error.start}")) + continue + if any(ord(character) in invisible_codepoints for character in text_content): + reasons.add("invisible Unicode code point") + if reasons: + findings.append((relative, ", ".join(sorted(reasons)))) with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output: output.write(f"findings={len(findings)}\n") @@ -171,9 +191,9 @@ jobs: output.write("ready=true\n") for relative, reasons in findings: - print(f"::warning file={annotation_escape(relative)}::Invisible characters detected: {reasons}") + print(f"::warning file={property_escape(relative)}::Invisible characters detected: {command_escape(reasons)}") for relative, reason in errors: - print(f"::error file={annotation_escape(relative)}::Invisible-character scan failed: {annotation_escape(reason)}") + print(f"::error file={property_escape(relative)}::Invisible-character scan failed: {command_escape(reason)}") PY - name: Write summary run: | @@ -190,6 +210,7 @@ jobs: echo "## Empty-Linter Results" >> "$GITHUB_STEP_SUMMARY" echo "" >> "$GITHUB_STEP_SUMMARY" echo "Found **${FINDINGS}** invisible character issue(s). See annotations above." >> "$GITHUB_STEP_SUMMARY" + exit 1 else echo "## Empty-Linter Results" >> "$GITHUB_STEP_SUMMARY" echo "" >> "$GITHUB_STEP_SUMMARY" diff --git a/CHANGELOG.adoc b/CHANGELOG.adoc index 3b2b735..55bb3fc 100644 --- a/CHANGELOG.adoc +++ b/CHANGELOG.adoc @@ -43,7 +43,7 @@ required files stay manual (`+plasma-engine/src/{action,apply}.rs+`) * feat(engine): overlay `+override-rules+` semantics — an overlay removes base rules by id from the effective set, and may pair an override with an `+add-rules+` reusing the same id to _replace_ a rule -(e.g. relax a severity or exempt a zone). `+override-rules+` targeting +(e.g. relax a severity or exempt a zone). `+override-rules+` targeting an unknown base id is a load error; `+modify-rules+` remains reserved. The evaluator and the action planner share one `+effective_rules+` set, so a finding never resolves to an overridden rule diff --git a/TEST-NEEDS.adoc b/TEST-NEEDS.adoc index 3f966ea..fe13653 100644 --- a/TEST-NEEDS.adoc +++ b/TEST-NEEDS.adoc @@ -28,7 +28,7 @@ Run everything with `+cargo test --workspace+`. === What’s Covered * SPDX expression parsing (lexer, parser, catalog -incl. Palimpsest-family ids) +incl. Palimpsest-family ids) * Zone assignment and `+.plasma.toml+` parsing * Repository scanning (`+scan_repo+`): counts, score, deterministic ordering