From 281e68c06f9f4d04f41ff1145f3dcbca46886586 Mon Sep 17 00:00:00 2001 From: hyperpolymath <6759885+hyperpolymath@users.noreply.github.com> Date: Sun, 14 Jun 2026 03:15:33 +0100 Subject: [PATCH 01/18] test(affine): add 66-test suite; fix byte_to_hex, transform ordering MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add tests/ covering ByteDetector (17), PathHandler (17), SafeWhitespace (13), TextTransform (19) — all 66 pass - Fix byte_to_hex to handle multi-byte codepoints (>0xFF) - Move normalize_line_endings to run after trim_lines_fn so JS .trim() cannot strip \r from CRLF-normalized lines - Add scripts/build-all.sh; deno.json build-all delegates to it (deno task shell does not support POSIX for-in loops) - Inject missing cross-module symbols into TextTransform.deno.js via post-compile Python patch (compiler issue #122 workaround): LF/CRLF/CR constants, is_invisible helper, concat alias Co-Authored-By: Claude Sonnet 4.6 --- deno.json | 2 +- deno.lock | 23 +++++++ scripts/build-all.sh | 42 +++++++++++++ src/core/ByteDetector.affine | 8 ++- src/core/TextTransform.affine | 6 +- tests/ByteDetector_test.js | Bin 0 -> 3627 bytes tests/PathHandler_test.js | 100 ++++++++++++++++++++++++++++++ tests/SafeWhitespace_test.js | 77 +++++++++++++++++++++++ tests/TextTransform_test.js | 111 ++++++++++++++++++++++++++++++++++ 9 files changed, 364 insertions(+), 5 deletions(-) create mode 100755 scripts/build-all.sh create mode 100644 tests/ByteDetector_test.js create mode 100644 tests/PathHandler_test.js create mode 100644 tests/SafeWhitespace_test.js create mode 100644 tests/TextTransform_test.js diff --git a/deno.json b/deno.json index 504b5b2..fb5c684 100644 --- a/deno.json +++ b/deno.json @@ -4,7 +4,7 @@ "exports": "./EmptyLinter.deno.js", "tasks": { "build": "affinescript compile --deno-esm EmptyLinter.affine -o EmptyLinter.deno.js && affinescript compile --deno-esm src/cli/Main.affine -o src/cli/Main.deno.js", - "build-all": "for f in stdlib/SafeHex.affine stdlib/SafeWhitespace.affine stdlib/SafePath.affine stdlib/SafeString.affine src/core/ByteDetector.affine src/core/TextTransform.affine src/core/PathHandler.affine EmptyLinter.affine src/cli/Main.affine; do affinescript compile --deno-esm $f -o ${f%.affine}.deno.js; done", + "build-all": "bash scripts/build-all.sh", "clean": "find . -name '*.deno.js' ! -path './stdlib/*' -delete", "dev": "while true; do affinescript compile --deno-esm EmptyLinter.affine -o EmptyLinter.deno.js 2>&1; sleep 2; done", "test": "deno test --allow-read --allow-write tests/", diff --git a/deno.lock b/deno.lock index e69de29..b6ea69d 100644 --- a/deno.lock +++ b/deno.lock @@ -0,0 +1,23 @@ +{ + "version": "5", + "specifiers": { + "jsr:@std/assert@*": "1.0.19", + "jsr:@std/internal@^1.0.12": "1.0.14" + }, + "jsr": { + "@std/assert@1.0.19": { + "integrity": "eaada96ee120cb980bc47e040f82814d786fe8162ecc53c91d8df60b8755991e", + "dependencies": [ + "jsr:@std/internal" + ] + }, + "@std/internal@1.0.14": { + "integrity": "291516b3d4c35024d6ffbc0a9df5bf4c64116e05b50012cf846710152d2ffdf7" + } + }, + "workspace": { + "dependencies": [ + "jsr:@std/assert@1" + ] + } +} diff --git a/scripts/build-all.sh b/scripts/build-all.sh new file mode 100755 index 0000000..b9ecd40 --- /dev/null +++ b/scripts/build-all.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +# SPDX-FileCopyrightText: 2026 hyperpolymath +set -euo pipefail + +SOURCES=( + stdlib/SafeHex.affine + stdlib/SafeWhitespace.affine + stdlib/SafePath.affine + stdlib/SafeString.affine + src/core/ByteDetector.affine + src/core/TextTransform.affine + src/core/PathHandler.affine + EmptyLinter.affine + src/cli/Main.affine +) + +for f in "${SOURCES[@]}"; do + affinescript compile --deno-esm "$f" -o "${f%.affine}.deno.js" +done + +# Workaround: AffineScript alpha compiler (issue #122) does not fully inline +# cross-module dependencies into TextTransform.deno.js. Inject missing symbols +# after compilation: LF/CRLF/CR (zero-arg enum constructors), is_invisible +# (private helper from SafeWhitespace), concat (string stdlib fn). +python3 - << 'PYEOF' +PATCH = """\ +const LF={tag:"LF"};const CRLF={tag:"CRLF"};const CR={tag:"CR"}; +function is_invisible(c){return(c===0||c===160||c===8203||c===65279||c===173||c===8206||c===8207||c===8204||c===8205||c===8288);} +function concat(a,b){return __as_concat(a,b);} +""" +MARKER = "// ---- end runtime ----" +path = "src/core/TextTransform.deno.js" +with open(path) as fh: + content = fh.read() +if PATCH.strip().split("\n")[0] not in content: + content = content.replace(MARKER, MARKER + "\n" + PATCH, 1) + with open(path, "w") as fh: + fh.write(content) +PYEOF + +echo "build-all complete" diff --git a/src/core/ByteDetector.affine b/src/core/ByteDetector.affine index 50bc757..0d229c2 100644 --- a/src/core/ByteDetector.affine +++ b/src/core/ByteDetector.affine @@ -52,7 +52,13 @@ pub fn get_artifact_def(byte_val: Int) -> Option { } pub fn byte_to_hex(v: Int) -> String { - encode_byte(v & 255) + if v <= 255 { + encode_byte(v) + } else if v <= 65535 { + encode_byte((v >> 8) & 255) ++ encode_byte(v & 255) + } else { + encode_byte((v >> 16) & 255) ++ encode_byte((v >> 8) & 255) ++ encode_byte(v & 255) + } } pub fn scan(content: String) -> [Artifact] { diff --git a/src/core/TextTransform.affine b/src/core/TextTransform.affine index 734a50d..e279336 100644 --- a/src/core/TextTransform.affine +++ b/src/core/TextTransform.affine @@ -52,9 +52,6 @@ pub fn transform(content: String, options: TransformOptions) -> String { if options.remove_invisibles_opt { s = remove_invisibles(s); } - if options.normalize_line_endings_opt { - s = normalize_line_endings(s, options.target_line_ending); - } if options.collapse_spaces_opt { s = collapse_spaces(s); } @@ -65,6 +62,9 @@ pub fn transform(content: String, options: TransformOptions) -> String { if options.trim_document { s = trim(s); } + if options.normalize_line_endings_opt { + s = normalize_line_endings(s, options.target_line_ending); + } if options.ensure_final_newline_opt { s = ensure_final_newline(s); } diff --git a/tests/ByteDetector_test.js b/tests/ByteDetector_test.js new file mode 100644 index 0000000000000000000000000000000000000000..91719f4f18261b1ed6b75ea8d7b720bcbd5aa6ee GIT binary patch literal 3627 zcmb_eQEuBt5cRiDG2oX3&=jo}`EVK|b)2FJ9LIoNxTs?YG&z*kp?2BbrEMVz(EnaS z_vl9f9iFa%M=VCJaOX{eT(gv++=F)~ zM?2#n+#Z!SKd}s7iF_g1bf(YnPVd1u96y7%f=7B5aX1Wq4B#!^VwS-V*Fh3Q!L@yQ zDRSgN6bHq}{cSeSh1Bo~BBfC3*S{B0ru_2hSd=f|3#3xa!Mj#+@1@eo$eDZ3s{^(H zR2*?~>$AN!cCJIVsCD%E{GYqVNVK(+eM_$WcZbClbxwK6y@2Pa;X7MzNkGnEG(F zAOkHXGraTd!RrUIensc7ILdrD#hces3hBdnBst?#AKvg(_}go|K`>G>9>qf9=zxwq zM2)c)G9YF}aIGkS?L+ggMx_T{-DJ;R3yJkmaC~rj;`uNb!i#u;Ml_~xJuf}XUk6JiqggoMnC zOtU=0`yzZ)s7F*JyQ}3b(=%JuAGmolri9s7Ym$5GcKkTB5_KWD0-Wc10jZD>XGr=) z{i8|V8ju^EV&tT3Czq8E{A|V)(4RX&lj;TDuxg~R7f?-AAeSO3Vx+@_lq3p3QuB2% zHP1nK7llLW$SCYcY}0UT4afb4p*0Nq4b#+Iou=mMhij#V1ZW^68O3T#X2*PinsEuWiI2Sim4adJ69oa;8mBED>;mIZOT5~P?aqDX;X?+65hjfL!7qG891>!qb+2$Tr1;nGwaU3MzGro;PYons@bN>ihz9xebBP#=e>ob)YdsF z6-^OtXULs&H@I&Ib?bz|#ify`V*!lcFvYIw6RO;&Fd^aYE7|r|NFlmWvQ})`6d9*o zQ1Avg*WC2c^4#1kop^6_6B!3OqV>@`6?1gr(umO*SjB9wV}0e)yuJ$ISa7t(Qt*#l z+;ZE7jVKz|x2d2ozTX{|kj+{BY?=xym+1SdHCpc~w{f>)&<%o`-@tPPN~=N#MS78= zNX-}}8kLsR0{y7W#tOWcUm*!`BoTIZsC$r +import { assertEquals } from "jsr:@std/assert"; +import { + validate, unwrap_path, path_join, sanitize, + is_within, get_parent, filename, has_extension, + is_excluded, from_trusted, + TraversalDetected, +} from "../src/core/PathHandler.deno.js"; + +Deno.test("PathHandler: validate accepts relative paths", () => { + const p = validate("src/main.affine"); + assertEquals(p.tag, "Some"); + assertEquals(unwrap_path(p.value), "src/main.affine"); +}); + +Deno.test("PathHandler: validate rejects absolute paths", () => { + assertEquals(validate("/etc/passwd").tag, "None"); +}); + +Deno.test("PathHandler: validate rejects path traversal", () => { + assertEquals(validate("../../etc/passwd").tag, "None"); +}); + +Deno.test("PathHandler: validate rejects embedded traversal", () => { + assertEquals(validate("src/../../../etc").tag, "None"); +}); + +Deno.test("PathHandler: sanitize removes dangerous characters", () => { + const clean = sanitize("file.txt"); + assertEquals(clean.includes("<"), false); + assertEquals(clean.includes(">"), false); +}); + +Deno.test("PathHandler: sanitize replaces slashes", () => { + const clean = sanitize("path/to/file"); + assertEquals(clean.includes("/"), false); +}); + +Deno.test("PathHandler: path_join creates valid joined path", () => { + const base = from_trusted("docs"); + const result = path_join(base, ["notes", "file.txt"]); + assertEquals(result.tag, "Ok"); + assertEquals(unwrap_path(result.value), "docs/notes/file.txt"); +}); + +Deno.test("PathHandler: path_join rejects traversal in components", () => { + const base = from_trusted("home"); + const result = path_join(base, ["..", "..", "etc"]); + assertEquals(result.tag, "Err"); + assertEquals(result.error.tag, "TraversalDetected"); +}); + +Deno.test("PathHandler: filename extracts basename", () => { + const p = from_trusted("docs/reports/file.pdf"); + assertEquals(filename(p), "file.pdf"); +}); + +Deno.test("PathHandler: filename handles no directory", () => { + assertEquals(filename(from_trusted("file.txt")), "file.txt"); +}); + +Deno.test("PathHandler: has_extension checks extension", () => { + const p = from_trusted("src/main.affine"); + assertEquals(has_extension(p, ".affine"), true); + assertEquals(has_extension(p, ".js"), false); +}); + +Deno.test("PathHandler: get_parent extracts directory", () => { + const p = from_trusted("home/user/docs/file.txt"); + const parent = get_parent(p); + assertEquals(parent.tag, "Some"); + assertEquals(unwrap_path(parent.value), "home/user/docs"); +}); + +Deno.test("PathHandler: get_parent returns None for no directory", () => { + assertEquals(get_parent(from_trusted("file.txt")).tag, "None"); +}); + +Deno.test("PathHandler: is_within checks path containment", () => { + const p = from_trusted("home/user/docs"); + const base = from_trusted("home/user"); + assertEquals(is_within(p, base), true); +}); + +Deno.test("PathHandler: is_within rejects unrelated paths", () => { + const p = from_trusted("etc/passwd"); + const base = from_trusted("home/user"); + assertEquals(is_within(p, base), false); +}); + +Deno.test("PathHandler: is_excluded matches excluded dirs", () => { + const p = from_trusted("project/node_modules/pkg/index.js"); + assertEquals(is_excluded(p, ["node_modules", ".git"]), true); +}); + +Deno.test("PathHandler: is_excluded allows non-excluded paths", () => { + const p = from_trusted("project/src/main.affine"); + assertEquals(is_excluded(p, ["node_modules", ".git"]), false); +}); diff --git a/tests/SafeWhitespace_test.js b/tests/SafeWhitespace_test.js new file mode 100644 index 0000000..a818528 --- /dev/null +++ b/tests/SafeWhitespace_test.js @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: MPL-2.0 +// SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell +import { assertEquals } from "jsr:@std/assert"; +import { + LF, CRLF, CR, + remove_invisibles, normalize_line_endings, + collapse_spaces, collapse_blank_lines, + trim_start, trim_end, ensure_final_newline, + detect_invisibles, +} from "../stdlib/SafeWhitespace.deno.js"; + +Deno.test("SafeWhitespace: trim_start removes leading whitespace", () => { + assertEquals(trim_start(" hello"), "hello"); + assertEquals(trim_start("\t\nhello"), "hello"); + assertEquals(trim_start("hello"), "hello"); +}); + +Deno.test("SafeWhitespace: trim_end removes trailing whitespace", () => { + assertEquals(trim_end("hello "), "hello"); + assertEquals(trim_end("hello\t\n"), "hello"); + assertEquals(trim_end("hello"), "hello"); +}); + +Deno.test("SafeWhitespace: collapse_spaces reduces multiple spaces", () => { + assertEquals(collapse_spaces("hello world"), "hello world"); + assertEquals(collapse_spaces("a b c"), "a b c"); +}); + +Deno.test("SafeWhitespace: collapse_spaces preserves single spaces", () => { + assertEquals(collapse_spaces("hello world"), "hello world"); +}); + +Deno.test("SafeWhitespace: collapse_blank_lines reduces excess blank lines", () => { + const result = collapse_blank_lines("para1\n\n\n\npara2", 1); + assertEquals(result.includes("\n\n\n"), false); +}); + +Deno.test("SafeWhitespace: normalize_line_endings converts CRLF to LF", () => { + const result = normalize_line_endings("line1\r\nline2", LF); + assertEquals(result.includes("\r"), false); +}); + +Deno.test("SafeWhitespace: normalize_line_endings converts LF to CRLF", () => { + const result = normalize_line_endings("line1\nline2", CRLF); + assertEquals(result.includes("\r\n"), true); +}); + +Deno.test("SafeWhitespace: ensure_final_newline adds newline when missing", () => { + assertEquals(ensure_final_newline("hello").endsWith("\n"), true); +}); + +Deno.test("SafeWhitespace: ensure_final_newline idempotent when present", () => { + const result = ensure_final_newline("hello\n"); + assertEquals(result, "hello\n"); +}); + +Deno.test("SafeWhitespace: remove_invisibles strips known invisible chars", () => { + const result = remove_invisibles("​hello"); + assertEquals(result.includes("​"), false); + assertEquals(result.includes(""), false); +}); + +Deno.test("SafeWhitespace: detect_invisibles finds NBSP", () => { + const found = detect_invisibles("hello world"); + assertEquals(found.length, 1); + assertEquals(found[0], 0xa0); +}); + +Deno.test("SafeWhitespace: detect_invisibles empty for clean string", () => { + assertEquals(detect_invisibles("hello world").length, 0); +}); + +Deno.test("SafeWhitespace: LineEnding constants have correct tags", () => { + assertEquals(LF.tag, "LF"); + assertEquals(CRLF.tag, "CRLF"); + assertEquals(CR.tag, "CR"); +}); diff --git a/tests/TextTransform_test.js b/tests/TextTransform_test.js new file mode 100644 index 0000000..94bb8eb --- /dev/null +++ b/tests/TextTransform_test.js @@ -0,0 +1,111 @@ +// SPDX-License-Identifier: MPL-2.0 +// SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell +import { assertEquals, assertNotEquals } from "jsr:@std/assert"; +import { + default_options, transform, transform_default, + get_metrics, metrics_to_string, + check_constraints, format_for_html, format_for_js, +} from "../src/core/TextTransform.deno.js"; +import { LF, CRLF } from "../stdlib/SafeWhitespace.deno.js"; + +Deno.test("TextTransform: transform trims lines when option set", () => { + const opts = { ...default_options(), trim_document: false, ensure_final_newline: false }; + const result = transform(" hello \n world ", opts); + assertEquals(result.includes(" hello"), false); +}); + +Deno.test("TextTransform: transform collapses spaces", () => { + const opts = { ...default_options(), trim_document: false, ensure_final_newline_opt: false, collapse_spaces_opt: true }; + const result = transform("hello world", opts); + assertEquals(result.includes(" "), false); +}); + +Deno.test("TextTransform: transform normalizes CRLF to LF", () => { + const opts = { ...default_options(), target_line_ending: LF }; + const result = transform("line1\r\nline2\r\nline3", opts); + assertEquals(result.includes("\r\n"), false); + assertEquals(result.includes("\r"), false); +}); + +Deno.test("TextTransform: transform normalizes LF to CRLF", () => { + const opts = { ...default_options(), target_line_ending: CRLF, ensure_final_newline_opt: false }; + const result = transform("line1\nline2", opts); + assertEquals(result.includes("\r\n"), true); +}); + +Deno.test("TextTransform: transform collapses excess blank lines", () => { + const opts = { ...default_options(), max_blank_lines: 1, ensure_final_newline: false }; + const result = transform("para1\n\n\n\n\npara2", opts); + assertEquals(result.includes("\n\n\n"), false); +}); + +Deno.test("TextTransform: transform ensures final newline", () => { + const opts = { ...default_options(), ensure_final_newline: true }; + assertEquals(transform("no newline", opts).endsWith("\n"), true); +}); + +Deno.test("TextTransform: transform_default returns a string", () => { + const result = transform_default(" test "); + assertEquals(typeof result, "string"); +}); + +Deno.test("TextTransform: get_metrics counts chars", () => { + assertEquals(get_metrics("Hello World").chars, 11); +}); + +Deno.test("TextTransform: get_metrics counts words", () => { + assertEquals(get_metrics("Hello World Test").words, 3); +}); + +Deno.test("TextTransform: get_metrics counts lines", () => { + assertEquals(get_metrics("Line 1\nLine 2\nLine 3").lines, 3); +}); + +Deno.test("TextTransform: metrics_to_string includes char count", () => { + const m = get_metrics("Hello World"); + const s = metrics_to_string(m); + assertEquals(s.includes("11"), true); +}); + +Deno.test("TextTransform: check_constraints detects char limit exceeded", () => { + const c = { max_chars: { tag: "Some", value: 5 }, max_words: { tag: "None" }, max_lines: { tag: "None" }, max_bytes: { tag: "None" } }; + const violations = check_constraints("This is a long string", c); + assertEquals(violations.length > 0, true); +}); + +Deno.test("TextTransform: check_constraints passes when within limit", () => { + const c = { max_chars: { tag: "Some", value: 100 }, max_words: { tag: "None" }, max_lines: { tag: "None" }, max_bytes: { tag: "None" } }; + assertEquals(check_constraints("Short", c).length, 0); +}); + +Deno.test("TextTransform: check_constraints detects word limit exceeded", () => { + const c = { max_chars: { tag: "None" }, max_words: { tag: "Some", value: 3 }, max_lines: { tag: "None" }, max_bytes: { tag: "None" } }; + const violations = check_constraints("one two three four five", c); + assertEquals(violations.length > 0, true); +}); + +Deno.test("TextTransform: check_constraints detects line limit exceeded", () => { + const c = { max_chars: { tag: "None" }, max_words: { tag: "None" }, max_lines: { tag: "Some", value: 2 }, max_bytes: { tag: "None" } }; + const violations = check_constraints("a\nb\nc\nd", c); + assertEquals(violations.length > 0, true); +}); + +Deno.test("TextTransform: format_for_html escapes < and >", () => { + const result = format_for_html(""); + assertEquals(result.includes("