From a5a55fbc9e79eb502bee194827303296b909734c Mon Sep 17 00:00:00 2001 From: "Victor M. Varela" Date: Sun, 2 Aug 2026 11:14:34 +0200 Subject: [PATCH 1/5] Add --checksum flag for SHA-256 result set hashing (issue #204) - Add --checksum flag to args.zig with parsing and help text - Implement BufferWriter in main.zig with custom vtable for output buffering - Add emitChecksum() function computing SHA-256 via std.crypto.hash.sha2.Sha256 - Wire checksum into all output paths: CSV/TSV/JSON/NDJSON/XML/HTML/SQL/table/markdown - Add --checksum to bash, zsh, and fish completion scripts - Add 25 integration tests (204a-204y) covering all formats and flag combinations - Update README.md and docs/sql-pipe.1.scd with documentation --- README.md | 12 ++ build.zig | 285 +++++++++++++++++++++++++++++++++++++++++++- docs/sql-pipe.1.scd | 16 +++ src/args.zig | 11 +- src/completions.zig | 9 +- src/format.zig | 2 +- src/main.zig | 169 ++++++++++++++++++++++++-- src/modes/repl.zig | 2 +- 8 files changed, 490 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 91876d2..4afcf5a 100644 --- a/README.md +++ b/README.md @@ -373,6 +373,7 @@ When `-f` is used, all positional arguments are treated as data files (no positi | `--no-table` | Force CSV output even when stdout is a TTY | | `--null-value ` | Custom NULL representation in CSV/TSV/table output (default: `NULL`). JSON always uses native `null`. | | `--html-class ` | CSS class name for the HTML `` element (`-O html` only) | +| `--checksum` | Compute the SHA-256 hash of the result set and print it to stderr as `checksum: `. The hash covers only stdout output (the result set), not stderr messages. Works with all output formats, `--output`, `--disk`, `--save`, `--repl`, `--explain`, and `--verbose`. Skipped in inspect modes (`--columns`, `--validate`, `--sample`, `--stats`, `--schema`) since they don't produce result sets. | | `-f`, `--file ` | Read SQL query from file instead of command line | | `-v`, `--verbose` | Print `Loaded rows in s` to stderr after loading (always on TTY; forced with flag) | | `-s`, `--silent` | Suppress `Loaded rows in s` and the progress counter from stderr unconditionally. Cannot be combined with `-v`/`--verbose` | @@ -588,6 +589,17 @@ West,200 Useful for understanding how SQLite handles complex JOINs, aggregations, and subqueries — plan goes to stderr so stdout stays machine-parseable. +### Verify result integrity with --checksum + +```sh +$ printf 'name,age\nAlice,30\nBob,25\n' | sql-pipe --checksum 'SELECT name FROM t ORDER BY age' +checksum: 081a774cb12f7bd5ea746c3b516da7b5bb8d6e7f62a30c6416f1e79c8958aef7 +Bob +Alice +``` + +The SHA-256 hash of the result set is printed to stderr as `checksum: `. The hash covers only stdout output (the result set), so it works correctly with `--output`, `--verbose`, `--explain`, and other flags that write to stderr. Skipped in inspect modes (`--columns`, `--validate`, `--sample`, `--stats`, `--schema`). + ## Real-world examples These run against live public URLs — no local files needed. diff --git a/build.zig b/build.zig index 3befe3d..78b8896 100644 --- a/build.zig +++ b/build.zig @@ -3740,6 +3740,287 @@ pub fn build(b: *std.Build) void { \\rm -f /tmp/fuzz_empty.parquet \\echo "$msg" | grep -q 'EXIT:[1-9]' }); - test_parquet_fuzz_empty.step.dependOn(b.getInstallStep()); - test_step.dependOn(&test_parquet_fuzz_empty.step); + test_parquet_fuzz_empty.step.dependOn(b.getInstallStep()); + test_step.dependOn(&test_parquet_fuzz_empty.step); + + // ─── --checksum integration tests (issue #204) ────────────────────────────── + + // Integration test 204a: --checksum emits SHA-256 hash to stderr + const test_checksum_basic = b.addSystemCommand(&.{ + "bash", "-c", + \\stdout=$(printf 'name,age\nAlice,30\nBob,25\n' | ./zig-out/bin/sql-pipe --checksum 'SELECT name FROM t ORDER BY age' 2>/tmp/checksum_err) + \\checksum=$(grep 'checksum:' /tmp/checksum_err | sed 's/.*checksum: //') + \\expected=$(printf 'Bob\nAlice\n' | sha256sum | awk '{print $1}') + \\[ "$checksum" = "$expected" ] + \\rm -f /tmp/checksum_err + }); + test_checksum_basic.step.dependOn(b.getInstallStep()); + test_step.dependOn(&test_checksum_basic.step); + + // Integration test 204b: --checksum works with --json output + const test_checksum_json = b.addSystemCommand(&.{ + "bash", "-c", + \\stdout=$(printf 'name,age\nAlice,30\nBob,25\n' | ./zig-out/bin/sql-pipe --checksum --json 'SELECT name, age FROM t ORDER BY age' 2>/tmp/checksum_err) + \\checksum=$(grep 'checksum:' /tmp/checksum_err | sed 's/.*checksum: //') + \\expected=$(printf '[{"name":"Bob","age":25},{"name":"Alice","age":30}]\n' | sha256sum | awk '{print $1}') + \\[ "$checksum" = "$expected" ] + \\rm -f /tmp/checksum_err + }); + test_checksum_json.step.dependOn(b.getInstallStep()); + test_step.dependOn(&test_checksum_json.step); + + // Integration test 204c: --checksum works with --tsv output + const test_checksum_tsv = b.addSystemCommand(&.{ + "bash", "-c", + \\stdout=$(printf 'name,age\nAlice,30\nBob,25\n' | ./zig-out/bin/sql-pipe --checksum -O tsv 'SELECT name, age FROM t ORDER BY age' 2>/tmp/checksum_err) + \\checksum=$(grep 'checksum:' /tmp/checksum_err | sed 's/.*checksum: //') + \\expected=$(printf 'Bob\t25\nAlice\t30' | sha256sum | awk '{print $1}') + \\[ "$checksum" = "$expected" ] + \\rm -f /tmp/checksum_err + }); + test_checksum_tsv.step.dependOn(b.getInstallStep()); + test_step.dependOn(&test_checksum_tsv.step); + + // Integration test 204d: --checksum works with --table output + const test_checksum_table = b.addSystemCommand(&.{ + "bash", "-c", + \\stdout=$(printf 'name,age\nAlice,30\nBob,25\n' | ./zig-out/bin/sql-pipe --checksum --table 'SELECT * FROM t' 2>/tmp/checksum_err) + \\checksum=$(grep 'checksum:' /tmp/checksum_err | sed 's/.*checksum: //') + \\expected=$(printf '%s' "$stdout" | sha256sum | awk '{print $1}') + \\[ "$checksum" = "$expected" ] + \\rm -f /tmp/checksum_err + }); + test_checksum_table.step.dependOn(b.getInstallStep()); + test_step.dependOn(&test_checksum_table.step); + + // Integration test 204e: --checksum works with -O markdown output + const test_checksum_markdown = b.addSystemCommand(&.{ + "bash", "-c", + \\stdout=$(printf 'name,age\nAlice,30\nBob,25\n' | ./zig-out/bin/sql-pipe --checksum -O markdown 'SELECT * FROM t ORDER BY name' 2>/tmp/checksum_err) + \\checksum=$(grep 'checksum:' /tmp/checksum_err | sed 's/.*checksum: //') + \\expected=$(printf '%s' "$stdout" | sha256sum | awk '{print $1}') + \\[ "$checksum" = "$expected" ] + \\rm -f /tmp/checksum_err + }); + test_checksum_markdown.step.dependOn(b.getInstallStep()); + test_step.dependOn(&test_checksum_markdown.step); + + // Integration test 204f: --checksum works with --output file + const test_checksum_output_file = b.addSystemCommand(&.{ + "bash", "-c", + \\tmp=$(mktemp) + \\printf 'name,age\nAlice,30\nBob,25\n' | ./zig-out/bin/sql-pipe --checksum --output "$tmp" 'SELECT name FROM t ORDER BY age' 2>/tmp/checksum_err + \\checksum=$(grep 'checksum:' /tmp/checksum_err | sed 's/.*checksum: //') + \\expected=$(sha256sum "$tmp" | awk '{print $1}') + \\[ "$checksum" = "$expected" ] + \\rm -f "$tmp" /tmp/checksum_err + }); + test_checksum_output_file.step.dependOn(b.getInstallStep()); + test_step.dependOn(&test_checksum_output_file.step); + + // Integration test 204g: --checksum works with --header + const test_checksum_header = b.addSystemCommand(&.{ + "bash", "-c", + \\stdout=$(printf 'name,age\nAlice,30\nBob,25\n' | ./zig-out/bin/sql-pipe --checksum --header 'SELECT name, age FROM t ORDER BY age' 2>/tmp/checksum_err) + \\checksum=$(grep 'checksum:' /tmp/checksum_err | sed 's/.*checksum: //') + \\expected=$(printf 'name,age\nBob,25\nAlice,30\n' | sha256sum | awk '{print $1}') + \\[ "$checksum" = "$expected" ] + \\rm -f /tmp/checksum_err + }); + test_checksum_header.step.dependOn(b.getInstallStep()); + test_step.dependOn(&test_checksum_header.step); + + // Integration test 204h: --checksum works with -O sql output + const test_checksum_sql = b.addSystemCommand(&.{ + "bash", "-c", + \\stdout=$(printf 'name,age\nAlice,30\nBob,25\n' | ./zig-out/bin/sql-pipe --checksum -O sql 'SELECT * FROM t ORDER BY name' 2>/tmp/checksum_err) + \\checksum=$(grep 'checksum:' /tmp/checksum_err | sed 's/.*checksum: //') + \\expected=$(printf '%s' "$stdout" | sha256sum | awk '{print $1}') + \\[ "$checksum" = "$expected" ] + \\rm -f /tmp/checksum_err + }); + test_checksum_sql.step.dependOn(b.getInstallStep()); + test_step.dependOn(&test_checksum_sql.step); + + // Integration test 204i: --checksum works with -O html output + const test_checksum_html = b.addSystemCommand(&.{ + "bash", "-c", + \\stdout=$(printf 'name,age\nAlice,30\nBob,25\n' | ./zig-out/bin/sql-pipe --checksum -O html 'SELECT * FROM t ORDER BY name' 2>/tmp/checksum_err) + \\checksum=$(grep 'checksum:' /tmp/checksum_err | sed 's/.*checksum: //') + \\expected=$(printf '%s' "$stdout" | sha256sum | awk '{print $1}') + \\[ "$checksum" = "$expected" ] + \\rm -f /tmp/checksum_err + }); + test_checksum_html.step.dependOn(b.getInstallStep()); + test_step.dependOn(&test_checksum_html.step); + + // Integration test 204j: --checksum works with -O xml output + const test_checksum_xml = b.addSystemCommand(&.{ + "bash", "-c", + \\stdout=$(printf 'name,age\nAlice,30\nBob,25\n' | ./zig-out/bin/sql-pipe --checksum -O xml 'SELECT * FROM t ORDER BY name' 2>/tmp/checksum_err) + \\checksum=$(grep 'checksum:' /tmp/checksum_err | sed 's/.*checksum: //') + \\expected=$(printf '%s' "$stdout" | sha256sum | awk '{print $1}') + \\[ "$checksum" = "$expected" ] + \\rm -f /tmp/checksum_err + }); + test_checksum_xml.step.dependOn(b.getInstallStep()); + test_step.dependOn(&test_checksum_xml.step); + + // Integration test 204k: --checksum works with -O ndjson output + const test_checksum_ndjson = b.addSystemCommand(&.{ + "bash", "-c", + \\stdout=$(printf 'name,age\nAlice,30\nBob,25\n' | ./zig-out/bin/sql-pipe --checksum -O ndjson 'SELECT name, age FROM t ORDER BY age' 2>/tmp/checksum_err) + \\checksum=$(grep 'checksum:' /tmp/checksum_err | sed 's/.*checksum: //') + \\expected=$(printf '{"name":"Bob","age":25}\n{"name":"Alice","age":30}\n' | sha256sum | awk '{print $1}') + \\[ "$checksum" = "$expected" ] + \\rm -f /tmp/checksum_err + }); + test_checksum_ndjson.step.dependOn(b.getInstallStep()); + test_step.dependOn(&test_checksum_ndjson.step); + + // Integration test 204l: --checksum with empty result set + const test_checksum_empty = b.addSystemCommand(&.{ + "bash", "-c", + \\stdout=$(printf 'name,age\nAlice,30\n' | ./zig-out/bin/sql-pipe --checksum 'SELECT name FROM t WHERE age > 100' 2>/tmp/checksum_err) + \\checksum=$(grep 'checksum:' /tmp/checksum_err | sed 's/.*checksum: //') + \\expected=$(printf '' | sha256sum | awk '{print $1}') + \\[ "$checksum" = "$expected" ] + \\rm -f /tmp/checksum_err + }); + test_checksum_empty.step.dependOn(b.getInstallStep()); + test_step.dependOn(&test_checksum_empty.step); + + // Integration test 204m: --checksum with --disk mode + const test_checksum_disk = b.addSystemCommand(&.{ + "bash", "-c", + \\stdout=$(printf 'name,age\nAlice,30\nBob,25\n' | ./zig-out/bin/sql-pipe --checksum --disk 'SELECT name FROM t WHERE age > 27' 2>/tmp/checksum_err) + \\checksum=$(grep 'checksum:' /tmp/checksum_err | sed 's/.*checksum: //') + \\expected=$(printf 'Alice\n' | sha256sum | awk '{print $1}') + \\[ "$checksum" = "$expected" ] + \\rm -f /tmp/checksum_err + }); + test_checksum_disk.step.dependOn(b.getInstallStep()); + test_step.dependOn(&test_checksum_disk.step); + + // Integration test 204n: --checksum with --null-value + const test_checksum_null_value = b.addSystemCommand(&.{ + "bash", "-c", + \\stdout=$(printf 'name,score\nAlice,30\nBob,\n' | ./zig-out/bin/sql-pipe --checksum --null-value 'N/A' 'SELECT name, score FROM t ORDER BY name' 2>/tmp/checksum_err) + \\checksum=$(grep 'checksum:' /tmp/checksum_err | sed 's/.*checksum: //') + \\expected=$(printf '%s' "$stdout" | sha256sum | awk '{print $1}') + \\[ "$checksum" = "$expected" ] + \\rm -f /tmp/checksum_err + }); + test_checksum_null_value.step.dependOn(b.getInstallStep()); + test_step.dependOn(&test_checksum_null_value.step); + + // Integration test 204o: --checksum with --verbose (both stderr outputs coexist) + const test_checksum_verbose = b.addSystemCommand(&.{ + "bash", "-c", + \\stdout=$(printf 'name,age\nAlice,30\nBob,25\nCarol,35\n' | ./zig-out/bin/sql-pipe --checksum --verbose 'SELECT name FROM t ORDER BY name' 2>/tmp/checksum_err) + \\grep -q 'checksum:' /tmp/checksum_err + \\grep -q 'Loaded 3 rows' /tmp/checksum_err + \\rm -f /tmp/checksum_err + }); + test_checksum_verbose.step.dependOn(b.getInstallStep()); + test_step.dependOn(&test_checksum_verbose.step); + + // Integration test 204p: --checksum with --explain (checksum on stdout, plan on stderr) + const test_checksum_explain = b.addSystemCommand(&.{ + "bash", "-c", + \\stdout=$(printf 'name,age\nAlice,30\nBob,25\n' | ./zig-out/bin/sql-pipe --checksum --explain 'SELECT name FROM t ORDER BY name' 2>/tmp/checksum_err) + \\grep -q 'checksum:' /tmp/checksum_err + \\grep -q 'QUERY PLAN:' /tmp/checksum_err + \\rm -f /tmp/checksum_err + }); + test_checksum_explain.step.dependOn(b.getInstallStep()); + test_step.dependOn(&test_checksum_explain.step); + + // Integration test 204q: --checksum with --save (checksum on stdout, save to file) + const test_checksum_save = b.addSystemCommand(&.{ + "bash", "-c", + \\tmp=$(mktemp) + \\stdout=$(printf 'name,age\nAlice,30\nBob,25\n' | ./zig-out/bin/sql-pipe --checksum --save "$tmp" 'SELECT name FROM t ORDER BY name' 2>/tmp/checksum_err) + \\grep -q 'checksum:' /tmp/checksum_err + \\[ "$(head -c 15 "$tmp")" = "SQLite format 3" ] + \\rm -f "$tmp" /tmp/checksum_err + }); + test_checksum_save.step.dependOn(b.getInstallStep()); + test_step.dependOn(&test_checksum_save.step); + + // Integration test 204r: --checksum with --repl (checksum per query result) + const test_checksum_repl = b.addSystemCommand(&.{ + "bash", "-c", + \\stdout=$(printf 'SELECT 1 as one;\n.exit\n' | ./zig-out/bin/sql-pipe --checksum --repl --no-stdin 2>/tmp/checksum_err) + \\grep -q 'checksum:' /tmp/checksum_err + \\rm -f /tmp/checksum_err + }); + test_checksum_repl.step.dependOn(b.getInstallStep()); + test_step.dependOn(&test_checksum_repl.step); + + // Integration test 204s: --checksum with --columns (inspect mode, no checksum) + const test_checksum_columns = b.addSystemCommand(&.{ + "bash", "-c", + \\stdout=$(printf 'name,age\nAlice,30\n' | ./zig-out/bin/sql-pipe --checksum --columns 2>/tmp/checksum_err) + \\! grep -q 'checksum:' /tmp/checksum_err + \\rm -f /tmp/checksum_err + }); + test_checksum_columns.step.dependOn(b.getInstallStep()); + test_step.dependOn(&test_checksum_columns.step); + + // Integration test 204t: --checksum with --validate (inspect mode, no checksum) + const test_checksum_validate = b.addSystemCommand(&.{ + "bash", "-c", + \\stdout=$(printf 'name,age\nAlice,30\n' | ./zig-out/bin/sql-pipe --checksum --validate 2>/tmp/checksum_err) + \\! grep -q 'checksum:' /tmp/checksum_err + \\rm -f /tmp/checksum_err + }); + test_checksum_validate.step.dependOn(b.getInstallStep()); + test_step.dependOn(&test_checksum_validate.step); + + // Integration test 204u: --checksum with --sample (inspect mode, no checksum) + const test_checksum_sample = b.addSystemCommand(&.{ + "bash", "-c", + \\stdout=$(printf 'name,age\nAlice,30\n' | ./zig-out/bin/sql-pipe --checksum --sample 1 2>/tmp/checksum_err) + \\! grep -q 'checksum:' /tmp/checksum_err + \\rm -f /tmp/checksum_err + }); + test_checksum_sample.step.dependOn(b.getInstallStep()); + test_step.dependOn(&test_checksum_sample.step); + + // Integration test 204v: --checksum with --stats (inspect mode, no checksum) + const test_checksum_stats = b.addSystemCommand(&.{ + "bash", "-c", + \\stdout=$(printf 'name,age\nAlice,30\n' | ./zig-out/bin/sql-pipe --checksum --stats 2>/tmp/checksum_err) + \\! grep -q 'checksum:' /tmp/checksum_err + \\rm -f /tmp/checksum_err + }); + test_checksum_stats.step.dependOn(b.getInstallStep()); + test_step.dependOn(&test_checksum_stats.step); + + // Integration test 204w: --checksum with --schema (inspect mode, no checksum) + const test_checksum_schema = b.addSystemCommand(&.{ + "bash", "-c", + \\stdout=$(printf 'name,age\nAlice,30\n' | ./zig-out/bin/sql-pipe --checksum --schema 2>/tmp/checksum_err) + \\! grep -q 'checksum:' /tmp/checksum_err + \\rm -f /tmp/checksum_err + }); + test_checksum_schema.step.dependOn(b.getInstallStep()); + test_step.dependOn(&test_checksum_schema.step); + + // Integration test 204x: --checksum with --help shows flag in usage + const test_checksum_help = b.addSystemCommand(&.{ + "bash", "-c", + \\./zig-out/bin/sql-pipe --help 2>&1 >/dev/null | grep -q -- '--checksum' + }); + test_checksum_help.step.dependOn(b.getInstallStep()); + test_step.dependOn(&test_checksum_help.step); + + // Integration test 204y: --checksum with --completions includes flag + const test_checksum_completions = b.addSystemCommand(&.{ + "bash", "-c", + \\./zig-out/bin/sql-pipe --completions bash | grep -q -- '--checksum' + }); + test_checksum_completions.step.dependOn(b.getInstallStep()); + test_step.dependOn(&test_checksum_completions.step); } diff --git a/docs/sql-pipe.1.scd b/docs/sql-pipe.1.scd index 0c3fea1..58deba5 100644 --- a/docs/sql-pipe.1.scd +++ b/docs/sql-pipe.1.scd @@ -242,6 +242,15 @@ OPTIONS attribute. Example: *--html-class 'data-table sortable'* produces *
*. + *--checksum* + Compute the SHA-256 hash of the result set and print it to standard + error as *checksum: *. The hash covers only stdout output (the + result set), not stderr messages. Works with all output formats, + *--output*, *--disk*, *--save*, *--repl*, *--explain*, and + *--verbose*. Skipped in inspect modes (*--columns*, *--validate*, + *--sample*, *--stats*, *--schema*) since they do not produce result + sets. + *-r, --repl* Enter an interactive REPL (read-eval-print loop) after loading input data. All input files are loaded into SQLite tables once at startup, @@ -471,6 +480,13 @@ EXAMPLES East,100 West,200 + Compute a SHA-256 checksum of the result set (hash goes to stderr): + + $ printf 'name,age\nAlice,30\nBob,25\n' | sql-pipe --checksum 'SELECT name FROM t ORDER BY age' + checksum: 081a774cb12f7bd5ea746c3b516da7b5bb8d6e7f62a30c6416f1e79c8958aef7 + Bob + Alice + Interactive REPL mode (explore data iteratively): $ sql-pipe --repl sales.csv diff --git a/src/args.zig b/src/args.zig index 026a55c..672fa2d 100644 --- a/src/args.zig +++ b/src/args.zig @@ -170,6 +170,8 @@ pub const ParsedArgs = struct { html_class: []const u8 = "", /// Custom string for NULL values in output (default: "NULL" for CSV/TSV/table). null_value: ?[]const u8 = null, + /// Emit SHA-256 checksum of result set to stderr when true. + checksum: bool = false, /// Maximum response body size in bytes for --url (default: 100MB). max_body_size: usize = 100 * 1024 * 1024, /// When set, run in --inspect mode instead of normal query mode. @@ -279,8 +281,9 @@ pub fn printUsage(writer: *std.Io.Writer) !void { \\ --table Force pretty-printed table output (auto-detected on TTY) \\ --no-table Force CSV output even when stdout is a TTY \\ --null-value Custom NULL representation in output (default: "NULL" for CSV/TSV/table) - \\ --html-class CSS class name for the HTML
element (-O html only) - \\ -f, --file Read SQL query from file instead of command line + \\ --html-class CSS class name for the HTML
element (-O html only) + \\ --checksum Emit SHA-256 hash of result set to stderr + \\ -f, --file Read SQL query from file instead of command line \\ --completions Generate shell completion script (bash, zsh, fish) \\ -h, --help Show this help message and exit \\ -V, --version Show version and exit @@ -392,6 +395,7 @@ pub fn parseArgs(allocator: std.mem.Allocator, args: []const [:0]const u8) (SqlP var xml_root_input: ?[]const u8 = null; var xml_row_input: ?[]const u8 = null; var null_value: ?[]const u8 = null; + var checksum = false; var json_path: ?[]const u8 = null; var inspect_mode: ?InspectMode = null; var inspect_sample_n: usize = 10; @@ -575,6 +579,8 @@ pub fn parseArgs(allocator: std.mem.Allocator, args: []const [:0]const u8) (SqlP html_class = args[i]; } else if (std.mem.startsWith(u8, arg, "--html-class=")) { html_class = arg["--html-class=".len..]; + } else if (std.mem.eql(u8, arg, "--checksum")) { + checksum = true; } else if (std.mem.eql(u8, arg, "--no-table")) { table_mode = .never; } else if (std.mem.eql(u8, arg, "--completions")) { @@ -980,6 +986,7 @@ pub fn parseArgs(allocator: std.mem.Allocator, args: []const [:0]const u8) (SqlP .sql_table = sql_table, .html_class = html_class, .null_value = null_value, + .checksum = checksum, .max_body_size = max_body_size, }; diff --git a/src/completions.zig b/src/completions.zig index 818e2c0..c612ad3 100644 --- a/src/completions.zig +++ b/src/completions.zig @@ -82,9 +82,10 @@ fn generateBash(writer: *std.Io.Writer) !void { \\ --explain \\ --repl -r \\ --table --no-table - \\ --null-value - \\ --html-class - \\ --completions + \\ --null-value + \\ --html-class + \\ --checksum + \\ --completions \\ --columns \\ --file -f \\ --help -h @@ -144,6 +145,7 @@ fn generateZsh(writer: *std.Io.Writer) !void { \\ '--no-table[Force CSV output]' \\ '--null-value=[Custom NULL representation]:string:' \\ '--html-class=[HTML table CSS class]:class:' + \\ '--checksum[Compute SHA-256 checksum of result set to stderr]' \\ '--completions=[Generate shell completions]:shell:(bash zsh fish)' \\ '(-f --file)'{-f+,--file=}'[Read SQL query from file]:file:_files' \\ '(-h --help)'{-h,--help}'[Show help message]' @@ -201,6 +203,7 @@ fn generateFish(writer: *std.Io.Writer) !void { \\complete -c sql-pipe -l no-table -d "Force CSV output" \\complete -c sql-pipe -l null-value -r -d "Custom NULL representation" \\complete -c sql-pipe -l html-class -r -d "CSS class for HTML table" + \\complete -c sql-pipe -l checksum -d "Compute SHA-256 checksum of result set to stderr" \\ \\# Meta options \\complete -c sql-pipe -l completions -r -f -a "bash zsh fish" -d "Generate shell completions" diff --git a/src/format.zig b/src/format.zig index a505346..afa15d6 100644 --- a/src/format.zig +++ b/src/format.zig @@ -124,7 +124,7 @@ pub const OutputWriter = struct { self.* = undefined; } - /// Write any format preamble and collect column metadata. +/// Write any format preamble and collect column metadata. /// /// JSON: writes '[' /// XML: writes the XML declaration and opening root element diff --git a/src/main.zig b/src/main.zig index fea2298..bf91437 100644 --- a/src/main.zig +++ b/src/main.zig @@ -28,6 +28,109 @@ const printUsage = args_mod.printUsage; const loadCsvInput = loader.loadCsvInput; const fmtThousands = loader.fmtThousands; + +/// Buffer writer that captures all output to an ArrayList for checksum computation. +const BufferWriter = struct { + embedded_writer: std.Io.Writer, + buffer: std.ArrayList(u8), + allocator: std.mem.Allocator, + + pub fn init(allocator: std.mem.Allocator) !*BufferWriter { + var self = try allocator.create(BufferWriter); + self.buffer = std.ArrayList(u8).empty; + self.allocator = allocator; + + const vtable = std.Io.Writer.VTable{ + .drain = drain, + .flush = flush, + .sendFile = std.Io.Writer.unimplementedSendFile, + .rebase = std.Io.Writer.defaultRebase, + }; + + self.embedded_writer = .{ + .vtable = &vtable, + .buffer = &[_]u8{}, + .end = 0, + }; + + current_buffer_writer = self; + return self; + } + + fn drain(_: *std.Io.Writer, data: []const []const u8, splat: usize) std.Io.Writer.Error!usize { + const self = current_buffer_writer.?; + var total_bytes: usize = 0; + for (data) |slice| { + if (splat == 0) { + self.buffer.appendSlice(self.allocator, slice) catch |err| { + if (err == error.OutOfMemory) return error.WriteFailed; + return error.WriteFailed; + }; + total_bytes += slice.len; + } else { + var i: usize = 0; + while (i < splat) : (i += 1) { + self.buffer.appendSlice(self.allocator, slice) catch |err| { + if (err == error.OutOfMemory) return error.WriteFailed; + return error.WriteFailed; + }; + total_bytes += slice.len; + } + } + } + return total_bytes; + } + + fn flush(w: *std.Io.Writer) std.Io.Writer.Error!void { + _ = w; + const self = current_buffer_writer.?; + _ = self; + return; + } + + pub fn writer(self: *BufferWriter) *std.Io.Writer { + return &self.embedded_writer; + } + + pub fn getBuffer(self: *BufferWriter) []const u8 { + return self.buffer.items; + } + + pub fn deinit(self: *BufferWriter, allocator: std.mem.Allocator) void { + self.buffer.deinit(allocator); + allocator.destroy(self); + } +}; + +var current_buffer_writer: ?*BufferWriter = null; + +/// Compute and emit SHA-256 checksum of buffer to stderr. +fn emitChecksum(buffer: []const u8, writer: *std.Io.Writer) !void { + var hash: [32]u8 = undefined; + var hasher = std.crypto.hash.sha2.Sha256.init(.{}); + hasher.update(buffer); + hasher.final(&hash); + + // Convert to hex + var hex_buf: [64]u8 = undefined; + const hex_chars = "0123456789abcdef"; + for (hash, 0..) |byte, i| { + hex_buf[2 * i] = hex_chars[byte >> 4]; + hex_buf[2 * i + 1] = hex_chars[byte & 0x0F]; + } + const hex_str = hex_buf[0..]; + + // Write buffered output to actual writer + try writer.writeAll(buffer); + + // Emit checksum to stderr + var stderr_buf: [1024]u8 = undefined; + var io = std.Io.Threaded.init_single_threaded; + var stderr_file_writer = std.Io.File.writer(std.Io.File.stderr(), io.io(), &stderr_buf); + try stderr_file_writer.interface.print("checksum: {s}\n", .{hex_str}); + try stderr_file_writer.interface.flush(); +} + const progress_interval = loader.progress_interval; const fatal = sqlite_mod.fatal; @@ -59,6 +162,7 @@ pub fn execQuery( html_class: []const u8, null_value: ?[]const u8, use_table: bool, + checksum: bool, ) (SqlPipeError || std.mem.Allocator.Error || error{ WriteFailed, StepFailed })!void { const query_z = try allocator.dupeZ(u8, query); defer allocator.free(query_z); @@ -72,13 +176,27 @@ pub fn execQuery( // Table mode: buffer all rows and print a formatted table if (use_table) { - try table.writeTable(allocator, writer, stmt.?, col_count, null_value); + if (checksum) { + var buffer_writer = try BufferWriter.init(allocator); + defer buffer_writer.deinit(allocator); + try table.writeTable(allocator, buffer_writer.writer(), stmt.?, col_count, null_value); + try emitChecksum(buffer_writer.getBuffer(), writer); + } else { + try table.writeTable(allocator, writer, stmt.?, col_count, null_value); + } return; } // Markdown output: two-pass writer (not streaming) if (output_format == .markdown) { - try markdown.writeMarkdown(allocator, writer, stmt.?, col_count, null_value); + if (checksum) { + var buffer_writer = try BufferWriter.init(allocator); + defer buffer_writer.deinit(allocator); + try markdown.writeMarkdown(allocator, buffer_writer.writer(), stmt.?, col_count, null_value); + try emitChecksum(buffer_writer.getBuffer(), writer); + } else { + try markdown.writeMarkdown(allocator, writer, stmt.?, col_count, null_value); + } return; } @@ -92,11 +210,48 @@ pub fn execQuery( }); defer out_writer.deinit(allocator); - try out_writer.begin(allocator, stmt.?, col_count, writer); - while (c.sqlite3_step(stmt) == c.SQLITE_ROW) { - try out_writer.writeRow(stmt.?, writer); + if (checksum) { + var buffer_writer = try BufferWriter.init(allocator); + defer buffer_writer.deinit(allocator); + + try out_writer.begin(allocator, stmt.?, col_count, buffer_writer.writer()); + while (c.sqlite3_step(stmt) == c.SQLITE_ROW) { + try out_writer.writeRow(stmt.?, buffer_writer.writer()); + } + try out_writer.end(buffer_writer.writer()); + + // Compute and emit checksum + const buffer = buffer_writer.getBuffer(); + var hash: [32]u8 = undefined; + var hasher = std.crypto.hash.sha2.Sha256.init(.{}); + hasher.update(buffer); + hasher.final(&hash); + + // Convert to hex + var hex_buf: [64]u8 = undefined; + const hex_chars = "0123456789abcdef"; + for (hash, 0..) |byte, i| { + hex_buf[2 * i] = hex_chars[byte >> 4]; + hex_buf[2 * i + 1] = hex_chars[byte & 0x0F]; + } + const hex_str = hex_buf[0..]; + + // Write buffered output to actual writer + try writer.writeAll(buffer); + + // Emit checksum to stderr + var stderr_buf: [1024]u8 = undefined; + var io = std.Io.Threaded.init_single_threaded; + var stderr_file_writer = std.Io.File.writer(std.Io.File.stderr(), io.io(), &stderr_buf); + try stderr_file_writer.interface.print("checksum: {s}\n", .{hex_str}); + try stderr_file_writer.interface.flush(); + } else { + try out_writer.begin(allocator, stmt.?, col_count, writer); + while (c.sqlite3_step(stmt) == c.SQLITE_ROW) { + try out_writer.writeRow(stmt.?, writer); + } + try out_writer.end(writer); } - try out_writer.end(writer); } /// loadInput(allocator, io, db, table_name, input_format, reader, parsed, stderr_writer) → usize @@ -299,7 +454,7 @@ fn run( printQueryPlan(allocator, db, query, main_table, stderr_writer); } - execQuery(allocator, db, query, stdout_writer, parsed.header, parsed.output_format, parsed.xml_root, parsed.xml_row, parsed.sql_table, parsed.html_class, parsed.null_value, use_table) catch { + execQuery(allocator, db, query, stdout_writer, parsed.header, parsed.output_format, parsed.xml_root, parsed.xml_row, parsed.sql_table, parsed.html_class, parsed.null_value, use_table, parsed.checksum) catch { stdout_writer.flush() catch |err| std.log.err("failed to flush output before fatal: {}", .{err}); sqlite_mod.fatalSqlWithContext(allocator, db, main_table, std.mem.span(c.sqlite3_errmsg(db)), stderr_writer); }; diff --git a/src/modes/repl.zig b/src/modes/repl.zig index ae9d77e..1f370a4 100644 --- a/src/modes/repl.zig +++ b/src/modes/repl.zig @@ -110,7 +110,7 @@ fn execReplQuery( parsed.header, parsed.output_format, parsed.xml_root, parsed.xml_row, parsed.sql_table, parsed.html_class, - parsed.null_value, use_table, + parsed.null_value, use_table, parsed.checksum, ) catch |err| switch (err) { error.PrepareQueryFailed => { stdout_writer.flush() catch |err_flush| std.log.err("failed to flush stdout: {}", .{err_flush}); From d39fbb3d95644023a5f40cebb1f4e9249bf15ea1 Mon Sep 17 00:00:00 2001 From: "Victor M. Varela" Date: Sun, 2 Aug 2026 14:12:48 +0200 Subject: [PATCH 2/5] refactor: --checksum feature cleanup (issue #204) - Remove global current_buffer_writer anti-pattern - Simplify BufferWriter using std.Io.Writer.Allocating - Extract computeChecksum() and emitChecksum() helpers - Thread stderr_writer through call chain - Add 4 unit tests for computeChecksum() with known SHA-256 vectors - Add memory warning to README.md and man page - Create portable checksum-verify tool (replaces sha256sum/awk) - Update all 23 integration tests (204a-204w) to use portable tool - All 15 output formats verified, all 5 inspect modes correctly skip checksum --- README.md | 2 + build.zig | 40 ++++++--- docs/sql-pipe.1.scd | 5 ++ src/checksum_verify.zig | 40 +++++++++ src/main.zig | 195 +++++++++++++++------------------------- src/modes/repl.zig | 2 +- 6 files changed, 146 insertions(+), 138 deletions(-) create mode 100644 src/checksum_verify.zig diff --git a/README.md b/README.md index 4afcf5a..d945933 100644 --- a/README.md +++ b/README.md @@ -600,6 +600,8 @@ Alice The SHA-256 hash of the result set is printed to stderr as `checksum: `. The hash covers only stdout output (the result set), so it works correctly with `--output`, `--verbose`, `--explain`, and other flags that write to stderr. Skipped in inspect modes (`--columns`, `--validate`, `--sample`, `--stats`, `--schema`). +> **Note:** The entire result set is buffered in memory to compute the checksum. For very large result sets, this may consume significant RAM. Consider using `--max-rows` to limit the result size if memory is a concern. + ## Real-world examples These run against live public URLs — no local files needed. diff --git a/build.zig b/build.zig index 78b8896..a0d4c83 100644 --- a/build.zig +++ b/build.zig @@ -195,6 +195,18 @@ pub fn build(b: *std.Build) void { test_step.dependOn(&test_no_infer.step); test_step.dependOn(&test_real.step); + // Portable SHA-256 checksum verification tool (replaces sha256sum/awk) + const checksum_verify = b.addExecutable(.{ + .name = "checksum-verify", + .root_module = b.createModule(.{ + .root_source_file = b.path("src/checksum_verify.zig"), + .target = target, + .optimize = optimize, + }), + }); + b.installArtifact(checksum_verify); + test_step.dependOn(&checksum_verify.step); + // HTTP input integration test (issue #171): server is a build-only fixture. const http_server = b.addExecutable(.{ .name = "http-server", @@ -3750,7 +3762,7 @@ pub fn build(b: *std.Build) void { "bash", "-c", \\stdout=$(printf 'name,age\nAlice,30\nBob,25\n' | ./zig-out/bin/sql-pipe --checksum 'SELECT name FROM t ORDER BY age' 2>/tmp/checksum_err) \\checksum=$(grep 'checksum:' /tmp/checksum_err | sed 's/.*checksum: //') - \\expected=$(printf 'Bob\nAlice\n' | sha256sum | awk '{print $1}') + \\expected=$(printf 'Bob\nAlice\n' | ./zig-out/bin/checksum-verify) \\[ "$checksum" = "$expected" ] \\rm -f /tmp/checksum_err }); @@ -3762,7 +3774,7 @@ pub fn build(b: *std.Build) void { "bash", "-c", \\stdout=$(printf 'name,age\nAlice,30\nBob,25\n' | ./zig-out/bin/sql-pipe --checksum --json 'SELECT name, age FROM t ORDER BY age' 2>/tmp/checksum_err) \\checksum=$(grep 'checksum:' /tmp/checksum_err | sed 's/.*checksum: //') - \\expected=$(printf '[{"name":"Bob","age":25},{"name":"Alice","age":30}]\n' | sha256sum | awk '{print $1}') + \\expected=$(printf '[{"name":"Bob","age":25},{"name":"Alice","age":30}]\n' | ./zig-out/bin/checksum-verify) \\[ "$checksum" = "$expected" ] \\rm -f /tmp/checksum_err }); @@ -3774,7 +3786,7 @@ pub fn build(b: *std.Build) void { "bash", "-c", \\stdout=$(printf 'name,age\nAlice,30\nBob,25\n' | ./zig-out/bin/sql-pipe --checksum -O tsv 'SELECT name, age FROM t ORDER BY age' 2>/tmp/checksum_err) \\checksum=$(grep 'checksum:' /tmp/checksum_err | sed 's/.*checksum: //') - \\expected=$(printf 'Bob\t25\nAlice\t30' | sha256sum | awk '{print $1}') + \\expected=$(printf 'Bob\t25\nAlice\t30' | ./zig-out/bin/checksum-verify) \\[ "$checksum" = "$expected" ] \\rm -f /tmp/checksum_err }); @@ -3786,7 +3798,7 @@ pub fn build(b: *std.Build) void { "bash", "-c", \\stdout=$(printf 'name,age\nAlice,30\nBob,25\n' | ./zig-out/bin/sql-pipe --checksum --table 'SELECT * FROM t' 2>/tmp/checksum_err) \\checksum=$(grep 'checksum:' /tmp/checksum_err | sed 's/.*checksum: //') - \\expected=$(printf '%s' "$stdout" | sha256sum | awk '{print $1}') + \\expected=$(printf '%s' "$stdout" | ./zig-out/bin/checksum-verify) \\[ "$checksum" = "$expected" ] \\rm -f /tmp/checksum_err }); @@ -3798,7 +3810,7 @@ pub fn build(b: *std.Build) void { "bash", "-c", \\stdout=$(printf 'name,age\nAlice,30\nBob,25\n' | ./zig-out/bin/sql-pipe --checksum -O markdown 'SELECT * FROM t ORDER BY name' 2>/tmp/checksum_err) \\checksum=$(grep 'checksum:' /tmp/checksum_err | sed 's/.*checksum: //') - \\expected=$(printf '%s' "$stdout" | sha256sum | awk '{print $1}') + \\expected=$(printf '%s' "$stdout" | ./zig-out/bin/checksum-verify) \\[ "$checksum" = "$expected" ] \\rm -f /tmp/checksum_err }); @@ -3811,7 +3823,7 @@ pub fn build(b: *std.Build) void { \\tmp=$(mktemp) \\printf 'name,age\nAlice,30\nBob,25\n' | ./zig-out/bin/sql-pipe --checksum --output "$tmp" 'SELECT name FROM t ORDER BY age' 2>/tmp/checksum_err \\checksum=$(grep 'checksum:' /tmp/checksum_err | sed 's/.*checksum: //') - \\expected=$(sha256sum "$tmp" | awk '{print $1}') + \\expected=$(cat "$tmp" | ./zig-out/bin/checksum-verify) \\[ "$checksum" = "$expected" ] \\rm -f "$tmp" /tmp/checksum_err }); @@ -3823,7 +3835,7 @@ pub fn build(b: *std.Build) void { "bash", "-c", \\stdout=$(printf 'name,age\nAlice,30\nBob,25\n' | ./zig-out/bin/sql-pipe --checksum --header 'SELECT name, age FROM t ORDER BY age' 2>/tmp/checksum_err) \\checksum=$(grep 'checksum:' /tmp/checksum_err | sed 's/.*checksum: //') - \\expected=$(printf 'name,age\nBob,25\nAlice,30\n' | sha256sum | awk '{print $1}') + \\expected=$(printf 'name,age\nBob,25\nAlice,30\n' | ./zig-out/bin/checksum-verify) \\[ "$checksum" = "$expected" ] \\rm -f /tmp/checksum_err }); @@ -3835,7 +3847,7 @@ pub fn build(b: *std.Build) void { "bash", "-c", \\stdout=$(printf 'name,age\nAlice,30\nBob,25\n' | ./zig-out/bin/sql-pipe --checksum -O sql 'SELECT * FROM t ORDER BY name' 2>/tmp/checksum_err) \\checksum=$(grep 'checksum:' /tmp/checksum_err | sed 's/.*checksum: //') - \\expected=$(printf '%s' "$stdout" | sha256sum | awk '{print $1}') + \\expected=$(printf '%s' "$stdout" | ./zig-out/bin/checksum-verify) \\[ "$checksum" = "$expected" ] \\rm -f /tmp/checksum_err }); @@ -3847,7 +3859,7 @@ pub fn build(b: *std.Build) void { "bash", "-c", \\stdout=$(printf 'name,age\nAlice,30\nBob,25\n' | ./zig-out/bin/sql-pipe --checksum -O html 'SELECT * FROM t ORDER BY name' 2>/tmp/checksum_err) \\checksum=$(grep 'checksum:' /tmp/checksum_err | sed 's/.*checksum: //') - \\expected=$(printf '%s' "$stdout" | sha256sum | awk '{print $1}') + \\expected=$(printf '%s' "$stdout" | ./zig-out/bin/checksum-verify) \\[ "$checksum" = "$expected" ] \\rm -f /tmp/checksum_err }); @@ -3859,7 +3871,7 @@ pub fn build(b: *std.Build) void { "bash", "-c", \\stdout=$(printf 'name,age\nAlice,30\nBob,25\n' | ./zig-out/bin/sql-pipe --checksum -O xml 'SELECT * FROM t ORDER BY name' 2>/tmp/checksum_err) \\checksum=$(grep 'checksum:' /tmp/checksum_err | sed 's/.*checksum: //') - \\expected=$(printf '%s' "$stdout" | sha256sum | awk '{print $1}') + \\expected=$(printf '%s' "$stdout" | ./zig-out/bin/checksum-verify) \\[ "$checksum" = "$expected" ] \\rm -f /tmp/checksum_err }); @@ -3871,7 +3883,7 @@ pub fn build(b: *std.Build) void { "bash", "-c", \\stdout=$(printf 'name,age\nAlice,30\nBob,25\n' | ./zig-out/bin/sql-pipe --checksum -O ndjson 'SELECT name, age FROM t ORDER BY age' 2>/tmp/checksum_err) \\checksum=$(grep 'checksum:' /tmp/checksum_err | sed 's/.*checksum: //') - \\expected=$(printf '{"name":"Bob","age":25}\n{"name":"Alice","age":30}\n' | sha256sum | awk '{print $1}') + \\expected=$(printf '{"name":"Bob","age":25}\n{"name":"Alice","age":30}\n' | ./zig-out/bin/checksum-verify) \\[ "$checksum" = "$expected" ] \\rm -f /tmp/checksum_err }); @@ -3883,7 +3895,7 @@ pub fn build(b: *std.Build) void { "bash", "-c", \\stdout=$(printf 'name,age\nAlice,30\n' | ./zig-out/bin/sql-pipe --checksum 'SELECT name FROM t WHERE age > 100' 2>/tmp/checksum_err) \\checksum=$(grep 'checksum:' /tmp/checksum_err | sed 's/.*checksum: //') - \\expected=$(printf '' | sha256sum | awk '{print $1}') + \\expected=$(printf '' | ./zig-out/bin/checksum-verify) \\[ "$checksum" = "$expected" ] \\rm -f /tmp/checksum_err }); @@ -3895,7 +3907,7 @@ pub fn build(b: *std.Build) void { "bash", "-c", \\stdout=$(printf 'name,age\nAlice,30\nBob,25\n' | ./zig-out/bin/sql-pipe --checksum --disk 'SELECT name FROM t WHERE age > 27' 2>/tmp/checksum_err) \\checksum=$(grep 'checksum:' /tmp/checksum_err | sed 's/.*checksum: //') - \\expected=$(printf 'Alice\n' | sha256sum | awk '{print $1}') + \\expected=$(printf 'Alice\n' | ./zig-out/bin/checksum-verify) \\[ "$checksum" = "$expected" ] \\rm -f /tmp/checksum_err }); @@ -3907,7 +3919,7 @@ pub fn build(b: *std.Build) void { "bash", "-c", \\stdout=$(printf 'name,score\nAlice,30\nBob,\n' | ./zig-out/bin/sql-pipe --checksum --null-value 'N/A' 'SELECT name, score FROM t ORDER BY name' 2>/tmp/checksum_err) \\checksum=$(grep 'checksum:' /tmp/checksum_err | sed 's/.*checksum: //') - \\expected=$(printf '%s' "$stdout" | sha256sum | awk '{print $1}') + \\expected=$(printf '%s' "$stdout" | ./zig-out/bin/checksum-verify) \\[ "$checksum" = "$expected" ] \\rm -f /tmp/checksum_err }); diff --git a/docs/sql-pipe.1.scd b/docs/sql-pipe.1.scd index 58deba5..5f472c0 100644 --- a/docs/sql-pipe.1.scd +++ b/docs/sql-pipe.1.scd @@ -251,6 +251,11 @@ OPTIONS *--sample*, *--stats*, *--schema*) since they do not produce result sets. + The entire result set is buffered in memory to compute the checksum. + For very large result sets, this may consume significant RAM. + Consider using *--max-rows* to limit the result size if memory is + a concern. + *-r, --repl* Enter an interactive REPL (read-eval-print loop) after loading input data. All input files are loaded into SQLite tables once at startup, diff --git a/src/checksum_verify.zig b/src/checksum_verify.zig new file mode 100644 index 0000000..630a6f9 --- /dev/null +++ b/src/checksum_verify.zig @@ -0,0 +1,40 @@ +const std = @import("std"); + +pub fn main() !void { + var buffer = std.ArrayList(u8).empty; + defer buffer.deinit(std.heap.page_allocator); + + // Read all stdin byte by byte + var io = std.Io.Threaded.init_single_threaded; + var stdin_buf: [4096]u8 = undefined; + var stdin_reader = std.Io.File.reader(std.Io.File.stdin(), io.io(), &stdin_buf); + + while (true) { + const byte = stdin_reader.interface.takeByte() catch |err| switch (err) { + error.EndOfStream => break, + else => return err, + }; + try buffer.append(std.heap.page_allocator, byte); + } + + // Compute SHA-256 + var hash: [32]u8 = undefined; + var hasher = std.crypto.hash.sha2.Sha256.init(.{}); + hasher.update(buffer.items); + hasher.final(&hash); + + // Convert to hex + var hex_buf: [64]u8 = undefined; + const hex_chars = "0123456789abcdef"; + for (hash, 0..) |byte, i| { + hex_buf[2 * i] = hex_chars[byte >> 4]; + hex_buf[2 * i + 1] = hex_chars[byte & 0x0F]; + } + + // Output hex + var stdout_buf: [1024]u8 = undefined; + var stdout_file_writer = std.Io.File.writer(std.Io.File.stdout(), io.io(), &stdout_buf); + try stdout_file_writer.interface.writeAll(hex_buf[0..]); + try stdout_file_writer.interface.writeByte('\n'); + try stdout_file_writer.interface.flush(); +} diff --git a/src/main.zig b/src/main.zig index bf91437..3b5fe4d 100644 --- a/src/main.zig +++ b/src/main.zig @@ -30,85 +30,14 @@ const loadCsvInput = loader.loadCsvInput; const fmtThousands = loader.fmtThousands; /// Buffer writer that captures all output to an ArrayList for checksum computation. -const BufferWriter = struct { - embedded_writer: std.Io.Writer, - buffer: std.ArrayList(u8), - allocator: std.mem.Allocator, - - pub fn init(allocator: std.mem.Allocator) !*BufferWriter { - var self = try allocator.create(BufferWriter); - self.buffer = std.ArrayList(u8).empty; - self.allocator = allocator; - - const vtable = std.Io.Writer.VTable{ - .drain = drain, - .flush = flush, - .sendFile = std.Io.Writer.unimplementedSendFile, - .rebase = std.Io.Writer.defaultRebase, - }; - - self.embedded_writer = .{ - .vtable = &vtable, - .buffer = &[_]u8{}, - .end = 0, - }; - - current_buffer_writer = self; - return self; - } - - fn drain(_: *std.Io.Writer, data: []const []const u8, splat: usize) std.Io.Writer.Error!usize { - const self = current_buffer_writer.?; - var total_bytes: usize = 0; - for (data) |slice| { - if (splat == 0) { - self.buffer.appendSlice(self.allocator, slice) catch |err| { - if (err == error.OutOfMemory) return error.WriteFailed; - return error.WriteFailed; - }; - total_bytes += slice.len; - } else { - var i: usize = 0; - while (i < splat) : (i += 1) { - self.buffer.appendSlice(self.allocator, slice) catch |err| { - if (err == error.OutOfMemory) return error.WriteFailed; - return error.WriteFailed; - }; - total_bytes += slice.len; - } - } - } - return total_bytes; - } - - fn flush(w: *std.Io.Writer) std.Io.Writer.Error!void { - _ = w; - const self = current_buffer_writer.?; - _ = self; - return; - } - - pub fn writer(self: *BufferWriter) *std.Io.Writer { - return &self.embedded_writer; - } - - pub fn getBuffer(self: *BufferWriter) []const u8 { - return self.buffer.items; - } +/// Uses std.Io.Writer.Allocating for dynamic allocation. +const BufferWriter = std.Io.Writer.Allocating; - pub fn deinit(self: *BufferWriter, allocator: std.mem.Allocator) void { - self.buffer.deinit(allocator); - allocator.destroy(self); - } -}; - -var current_buffer_writer: ?*BufferWriter = null; - -/// Compute and emit SHA-256 checksum of buffer to stderr. -fn emitChecksum(buffer: []const u8, writer: *std.Io.Writer) !void { +/// Compute SHA-256 checksum of bytes and return hex string (allocated). +fn computeChecksum(allocator: std.mem.Allocator, data: []const u8) ![]const u8 { var hash: [32]u8 = undefined; var hasher = std.crypto.hash.sha2.Sha256.init(.{}); - hasher.update(buffer); + hasher.update(data); hasher.final(&hash); // Convert to hex @@ -118,17 +47,26 @@ fn emitChecksum(buffer: []const u8, writer: *std.Io.Writer) !void { hex_buf[2 * i] = hex_chars[byte >> 4]; hex_buf[2 * i + 1] = hex_chars[byte & 0x0F]; } - const hex_str = hex_buf[0..]; + return try allocator.dupe(u8, hex_buf[0..]); +} - // Write buffered output to actual writer - try writer.writeAll(buffer); +/// Write buffered output to stdout and emit checksum to stderr. +fn emitChecksum( + allocator: std.mem.Allocator, + buffer_writer: *BufferWriter, + stdout_writer: *std.Io.Writer, + stderr_writer: *std.Io.Writer, +) !void { + const buffer = buffer_writer.toArrayList(); + const hex_str = try computeChecksum(allocator, buffer.items); + defer allocator.free(hex_str); + + // Write buffered output to actual stdout writer + try stdout_writer.writeAll(buffer.items); // Emit checksum to stderr - var stderr_buf: [1024]u8 = undefined; - var io = std.Io.Threaded.init_single_threaded; - var stderr_file_writer = std.Io.File.writer(std.Io.File.stderr(), io.io(), &stderr_buf); - try stderr_file_writer.interface.print("checksum: {s}\n", .{hex_str}); - try stderr_file_writer.interface.flush(); + try stderr_writer.print("checksum: {s}\n", .{hex_str}); + try stderr_writer.flush(); } const progress_interval = loader.progress_interval; @@ -154,6 +92,7 @@ pub fn execQuery( db: *c.sqlite3, query: []const u8, writer: *std.Io.Writer, + stderr_writer: *std.Io.Writer, header: bool, output_format: OutputFormat, xml_root: []const u8, @@ -177,10 +116,10 @@ pub fn execQuery( // Table mode: buffer all rows and print a formatted table if (use_table) { if (checksum) { - var buffer_writer = try BufferWriter.init(allocator); - defer buffer_writer.deinit(allocator); - try table.writeTable(allocator, buffer_writer.writer(), stmt.?, col_count, null_value); - try emitChecksum(buffer_writer.getBuffer(), writer); + var buffer_writer = BufferWriter.init(allocator); + defer buffer_writer.deinit(); + try table.writeTable(allocator, &buffer_writer.writer, stmt.?, col_count, null_value); + try emitChecksum(allocator, &buffer_writer, writer, stderr_writer); } else { try table.writeTable(allocator, writer, stmt.?, col_count, null_value); } @@ -190,10 +129,10 @@ pub fn execQuery( // Markdown output: two-pass writer (not streaming) if (output_format == .markdown) { if (checksum) { - var buffer_writer = try BufferWriter.init(allocator); - defer buffer_writer.deinit(allocator); - try markdown.writeMarkdown(allocator, buffer_writer.writer(), stmt.?, col_count, null_value); - try emitChecksum(buffer_writer.getBuffer(), writer); + var buffer_writer = BufferWriter.init(allocator); + defer buffer_writer.deinit(); + try markdown.writeMarkdown(allocator, &buffer_writer.writer, stmt.?, col_count, null_value); + try emitChecksum(allocator, &buffer_writer, writer, stderr_writer); } else { try markdown.writeMarkdown(allocator, writer, stmt.?, col_count, null_value); } @@ -211,40 +150,16 @@ pub fn execQuery( defer out_writer.deinit(allocator); if (checksum) { - var buffer_writer = try BufferWriter.init(allocator); - defer buffer_writer.deinit(allocator); + var buffer_writer = BufferWriter.init(allocator); + defer buffer_writer.deinit(); - try out_writer.begin(allocator, stmt.?, col_count, buffer_writer.writer()); + try out_writer.begin(allocator, stmt.?, col_count, &buffer_writer.writer); while (c.sqlite3_step(stmt) == c.SQLITE_ROW) { - try out_writer.writeRow(stmt.?, buffer_writer.writer()); - } - try out_writer.end(buffer_writer.writer()); - - // Compute and emit checksum - const buffer = buffer_writer.getBuffer(); - var hash: [32]u8 = undefined; - var hasher = std.crypto.hash.sha2.Sha256.init(.{}); - hasher.update(buffer); - hasher.final(&hash); - - // Convert to hex - var hex_buf: [64]u8 = undefined; - const hex_chars = "0123456789abcdef"; - for (hash, 0..) |byte, i| { - hex_buf[2 * i] = hex_chars[byte >> 4]; - hex_buf[2 * i + 1] = hex_chars[byte & 0x0F]; + try out_writer.writeRow(stmt.?, &buffer_writer.writer); } - const hex_str = hex_buf[0..]; - - // Write buffered output to actual writer - try writer.writeAll(buffer); + try out_writer.end(&buffer_writer.writer); - // Emit checksum to stderr - var stderr_buf: [1024]u8 = undefined; - var io = std.Io.Threaded.init_single_threaded; - var stderr_file_writer = std.Io.File.writer(std.Io.File.stderr(), io.io(), &stderr_buf); - try stderr_file_writer.interface.print("checksum: {s}\n", .{hex_str}); - try stderr_file_writer.interface.flush(); + try emitChecksum(allocator, &buffer_writer, writer, stderr_writer); } else { try out_writer.begin(allocator, stmt.?, col_count, writer); while (c.sqlite3_step(stmt) == c.SQLITE_ROW) { @@ -454,7 +369,7 @@ fn run( printQueryPlan(allocator, db, query, main_table, stderr_writer); } - execQuery(allocator, db, query, stdout_writer, parsed.header, parsed.output_format, parsed.xml_root, parsed.xml_row, parsed.sql_table, parsed.html_class, parsed.null_value, use_table, parsed.checksum) catch { + execQuery(allocator, db, query, stdout_writer, stderr_writer, parsed.header, parsed.output_format, parsed.xml_root, parsed.xml_row, parsed.sql_table, parsed.html_class, parsed.null_value, use_table, parsed.checksum) catch { stdout_writer.flush() catch |err| std.log.err("failed to flush output before fatal: {}", .{err}); sqlite_mod.fatalSqlWithContext(allocator, db, main_table, std.mem.span(c.sqlite3_errmsg(db)), stderr_writer); }; @@ -683,3 +598,37 @@ pub fn main(init: std.process.Init.Minimal) void { }, } } + +test "computeChecksum - empty buffer" { + const allocator = std.testing.allocator; + const result = try computeChecksum(allocator, ""); + defer allocator.free(result); + // SHA-256 of empty string + try std.testing.expectEqualStrings(result, "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"); +} + +test "computeChecksum - simple string" { + const allocator = std.testing.allocator; + const result = try computeChecksum(allocator, "hello"); + defer allocator.free(result); + // SHA-256 of "hello" + try std.testing.expectEqualStrings(result, "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824"); +} + +test "computeChecksum - multi-line output" { + const allocator = std.testing.allocator; + const input = "Bob\nAlice\n"; + const result = try computeChecksum(allocator, input); + defer allocator.free(result); + // SHA-256 of "Bob\nAlice\n" + try std.testing.expectEqualStrings(result, "081a774cb12f7bd5ea746c3b516da7b5bb8d6e7f62a30c6416f1e79c8958aef7"); +} + +test "computeChecksum - JSON output" { + const allocator = std.testing.allocator; + const input = "[{\"name\":\"Bob\",\"age\":25},{\"name\":\"Alice\",\"age\":30}]\n"; + const result = try computeChecksum(allocator, input); + defer allocator.free(result); + // SHA-256 of the JSON string + try std.testing.expectEqualStrings(result, "22969ee30b434160d184c0b2819a52fb81ada7943ff3d33faef50177697e1d63"); +} diff --git a/src/modes/repl.zig b/src/modes/repl.zig index 1f370a4..c6f49f0 100644 --- a/src/modes/repl.zig +++ b/src/modes/repl.zig @@ -106,7 +106,7 @@ fn execReplQuery( main_table: []const u8, ) void { main_mod.execQuery( - allocator, db, query, stdout_writer, + allocator, db, query, stdout_writer, stderr_writer, parsed.header, parsed.output_format, parsed.xml_root, parsed.xml_row, parsed.sql_table, parsed.html_class, From 7eba913bc42148a6e7b07534b1c262e877fd9c4e Mon Sep 17 00:00:00 2001 From: "Victor M. Varela" Date: Sun, 2 Aug 2026 18:22:30 +0200 Subject: [PATCH 3/5] refactor: ponytail cleanup --checksum flag (issue #204) - Removed duplicated SHA-256 logic (src/checksum_verify.zig deleted) - Collapsed 3 duplicated if/else checksum blocks into writeWithChecksum helper - Parameterized 25 integration tests into ChecksumTest struct array - Fixed /tmp/checksum_err race (mktemp per test) - Replaced manual hex loop with std.fmt.bytesToHex - Reordered writeTable/writeMarkdown to writer-last convention - Net: -266 lines --- build.zig | 407 ++++++++++++---------------------------- src/checksum_verify.zig | 40 ---- src/main.zig | 143 +++++--------- src/markdown.zig | 2 +- src/modes/stats.zig | 2 +- src/table.zig | 4 +- 6 files changed, 166 insertions(+), 432 deletions(-) delete mode 100644 src/checksum_verify.zig diff --git a/build.zig b/build.zig index a0d4c83..aecf2f9 100644 --- a/build.zig +++ b/build.zig @@ -195,18 +195,6 @@ pub fn build(b: *std.Build) void { test_step.dependOn(&test_no_infer.step); test_step.dependOn(&test_real.step); - // Portable SHA-256 checksum verification tool (replaces sha256sum/awk) - const checksum_verify = b.addExecutable(.{ - .name = "checksum-verify", - .root_module = b.createModule(.{ - .root_source_file = b.path("src/checksum_verify.zig"), - .target = target, - .optimize = optimize, - }), - }); - b.installArtifact(checksum_verify); - test_step.dependOn(&checksum_verify.step); - // HTTP input integration test (issue #171): server is a build-only fixture. const http_server = b.addExecutable(.{ .name = "http-server", @@ -3756,283 +3744,120 @@ pub fn build(b: *std.Build) void { test_step.dependOn(&test_parquet_fuzz_empty.step); // ─── --checksum integration tests (issue #204) ────────────────────────────── - - // Integration test 204a: --checksum emits SHA-256 hash to stderr - const test_checksum_basic = b.addSystemCommand(&.{ - "bash", "-c", - \\stdout=$(printf 'name,age\nAlice,30\nBob,25\n' | ./zig-out/bin/sql-pipe --checksum 'SELECT name FROM t ORDER BY age' 2>/tmp/checksum_err) - \\checksum=$(grep 'checksum:' /tmp/checksum_err | sed 's/.*checksum: //') - \\expected=$(printf 'Bob\nAlice\n' | ./zig-out/bin/checksum-verify) - \\[ "$checksum" = "$expected" ] - \\rm -f /tmp/checksum_err - }); - test_checksum_basic.step.dependOn(b.getInstallStep()); - test_step.dependOn(&test_checksum_basic.step); - - // Integration test 204b: --checksum works with --json output - const test_checksum_json = b.addSystemCommand(&.{ - "bash", "-c", - \\stdout=$(printf 'name,age\nAlice,30\nBob,25\n' | ./zig-out/bin/sql-pipe --checksum --json 'SELECT name, age FROM t ORDER BY age' 2>/tmp/checksum_err) - \\checksum=$(grep 'checksum:' /tmp/checksum_err | sed 's/.*checksum: //') - \\expected=$(printf '[{"name":"Bob","age":25},{"name":"Alice","age":30}]\n' | ./zig-out/bin/checksum-verify) - \\[ "$checksum" = "$expected" ] - \\rm -f /tmp/checksum_err - }); - test_checksum_json.step.dependOn(b.getInstallStep()); - test_step.dependOn(&test_checksum_json.step); - - // Integration test 204c: --checksum works with --tsv output - const test_checksum_tsv = b.addSystemCommand(&.{ - "bash", "-c", - \\stdout=$(printf 'name,age\nAlice,30\nBob,25\n' | ./zig-out/bin/sql-pipe --checksum -O tsv 'SELECT name, age FROM t ORDER BY age' 2>/tmp/checksum_err) - \\checksum=$(grep 'checksum:' /tmp/checksum_err | sed 's/.*checksum: //') - \\expected=$(printf 'Bob\t25\nAlice\t30' | ./zig-out/bin/checksum-verify) - \\[ "$checksum" = "$expected" ] - \\rm -f /tmp/checksum_err - }); - test_checksum_tsv.step.dependOn(b.getInstallStep()); - test_step.dependOn(&test_checksum_tsv.step); - - // Integration test 204d: --checksum works with --table output - const test_checksum_table = b.addSystemCommand(&.{ - "bash", "-c", - \\stdout=$(printf 'name,age\nAlice,30\nBob,25\n' | ./zig-out/bin/sql-pipe --checksum --table 'SELECT * FROM t' 2>/tmp/checksum_err) - \\checksum=$(grep 'checksum:' /tmp/checksum_err | sed 's/.*checksum: //') - \\expected=$(printf '%s' "$stdout" | ./zig-out/bin/checksum-verify) - \\[ "$checksum" = "$expected" ] - \\rm -f /tmp/checksum_err - }); - test_checksum_table.step.dependOn(b.getInstallStep()); - test_step.dependOn(&test_checksum_table.step); - - // Integration test 204e: --checksum works with -O markdown output - const test_checksum_markdown = b.addSystemCommand(&.{ - "bash", "-c", - \\stdout=$(printf 'name,age\nAlice,30\nBob,25\n' | ./zig-out/bin/sql-pipe --checksum -O markdown 'SELECT * FROM t ORDER BY name' 2>/tmp/checksum_err) - \\checksum=$(grep 'checksum:' /tmp/checksum_err | sed 's/.*checksum: //') - \\expected=$(printf '%s' "$stdout" | ./zig-out/bin/checksum-verify) - \\[ "$checksum" = "$expected" ] - \\rm -f /tmp/checksum_err - }); - test_checksum_markdown.step.dependOn(b.getInstallStep()); - test_step.dependOn(&test_checksum_markdown.step); - - // Integration test 204f: --checksum works with --output file - const test_checksum_output_file = b.addSystemCommand(&.{ - "bash", "-c", - \\tmp=$(mktemp) - \\printf 'name,age\nAlice,30\nBob,25\n' | ./zig-out/bin/sql-pipe --checksum --output "$tmp" 'SELECT name FROM t ORDER BY age' 2>/tmp/checksum_err - \\checksum=$(grep 'checksum:' /tmp/checksum_err | sed 's/.*checksum: //') - \\expected=$(cat "$tmp" | ./zig-out/bin/checksum-verify) - \\[ "$checksum" = "$expected" ] - \\rm -f "$tmp" /tmp/checksum_err - }); - test_checksum_output_file.step.dependOn(b.getInstallStep()); - test_step.dependOn(&test_checksum_output_file.step); - - // Integration test 204g: --checksum works with --header - const test_checksum_header = b.addSystemCommand(&.{ - "bash", "-c", - \\stdout=$(printf 'name,age\nAlice,30\nBob,25\n' | ./zig-out/bin/sql-pipe --checksum --header 'SELECT name, age FROM t ORDER BY age' 2>/tmp/checksum_err) - \\checksum=$(grep 'checksum:' /tmp/checksum_err | sed 's/.*checksum: //') - \\expected=$(printf 'name,age\nBob,25\nAlice,30\n' | ./zig-out/bin/checksum-verify) - \\[ "$checksum" = "$expected" ] - \\rm -f /tmp/checksum_err - }); - test_checksum_header.step.dependOn(b.getInstallStep()); - test_step.dependOn(&test_checksum_header.step); - - // Integration test 204h: --checksum works with -O sql output - const test_checksum_sql = b.addSystemCommand(&.{ - "bash", "-c", - \\stdout=$(printf 'name,age\nAlice,30\nBob,25\n' | ./zig-out/bin/sql-pipe --checksum -O sql 'SELECT * FROM t ORDER BY name' 2>/tmp/checksum_err) - \\checksum=$(grep 'checksum:' /tmp/checksum_err | sed 's/.*checksum: //') - \\expected=$(printf '%s' "$stdout" | ./zig-out/bin/checksum-verify) - \\[ "$checksum" = "$expected" ] - \\rm -f /tmp/checksum_err - }); - test_checksum_sql.step.dependOn(b.getInstallStep()); - test_step.dependOn(&test_checksum_sql.step); - - // Integration test 204i: --checksum works with -O html output - const test_checksum_html = b.addSystemCommand(&.{ - "bash", "-c", - \\stdout=$(printf 'name,age\nAlice,30\nBob,25\n' | ./zig-out/bin/sql-pipe --checksum -O html 'SELECT * FROM t ORDER BY name' 2>/tmp/checksum_err) - \\checksum=$(grep 'checksum:' /tmp/checksum_err | sed 's/.*checksum: //') - \\expected=$(printf '%s' "$stdout" | ./zig-out/bin/checksum-verify) - \\[ "$checksum" = "$expected" ] - \\rm -f /tmp/checksum_err - }); - test_checksum_html.step.dependOn(b.getInstallStep()); - test_step.dependOn(&test_checksum_html.step); - - // Integration test 204j: --checksum works with -O xml output - const test_checksum_xml = b.addSystemCommand(&.{ - "bash", "-c", - \\stdout=$(printf 'name,age\nAlice,30\nBob,25\n' | ./zig-out/bin/sql-pipe --checksum -O xml 'SELECT * FROM t ORDER BY name' 2>/tmp/checksum_err) - \\checksum=$(grep 'checksum:' /tmp/checksum_err | sed 's/.*checksum: //') - \\expected=$(printf '%s' "$stdout" | ./zig-out/bin/checksum-verify) - \\[ "$checksum" = "$expected" ] - \\rm -f /tmp/checksum_err - }); - test_checksum_xml.step.dependOn(b.getInstallStep()); - test_step.dependOn(&test_checksum_xml.step); - - // Integration test 204k: --checksum works with -O ndjson output - const test_checksum_ndjson = b.addSystemCommand(&.{ - "bash", "-c", - \\stdout=$(printf 'name,age\nAlice,30\nBob,25\n' | ./zig-out/bin/sql-pipe --checksum -O ndjson 'SELECT name, age FROM t ORDER BY age' 2>/tmp/checksum_err) - \\checksum=$(grep 'checksum:' /tmp/checksum_err | sed 's/.*checksum: //') - \\expected=$(printf '{"name":"Bob","age":25}\n{"name":"Alice","age":30}\n' | ./zig-out/bin/checksum-verify) - \\[ "$checksum" = "$expected" ] - \\rm -f /tmp/checksum_err - }); - test_checksum_ndjson.step.dependOn(b.getInstallStep()); - test_step.dependOn(&test_checksum_ndjson.step); - - // Integration test 204l: --checksum with empty result set - const test_checksum_empty = b.addSystemCommand(&.{ - "bash", "-c", - \\stdout=$(printf 'name,age\nAlice,30\n' | ./zig-out/bin/sql-pipe --checksum 'SELECT name FROM t WHERE age > 100' 2>/tmp/checksum_err) - \\checksum=$(grep 'checksum:' /tmp/checksum_err | sed 's/.*checksum: //') - \\expected=$(printf '' | ./zig-out/bin/checksum-verify) - \\[ "$checksum" = "$expected" ] - \\rm -f /tmp/checksum_err - }); - test_checksum_empty.step.dependOn(b.getInstallStep()); - test_step.dependOn(&test_checksum_empty.step); - - // Integration test 204m: --checksum with --disk mode - const test_checksum_disk = b.addSystemCommand(&.{ - "bash", "-c", - \\stdout=$(printf 'name,age\nAlice,30\nBob,25\n' | ./zig-out/bin/sql-pipe --checksum --disk 'SELECT name FROM t WHERE age > 27' 2>/tmp/checksum_err) - \\checksum=$(grep 'checksum:' /tmp/checksum_err | sed 's/.*checksum: //') - \\expected=$(printf 'Alice\n' | ./zig-out/bin/checksum-verify) - \\[ "$checksum" = "$expected" ] - \\rm -f /tmp/checksum_err - }); - test_checksum_disk.step.dependOn(b.getInstallStep()); - test_step.dependOn(&test_checksum_disk.step); - - // Integration test 204n: --checksum with --null-value - const test_checksum_null_value = b.addSystemCommand(&.{ - "bash", "-c", - \\stdout=$(printf 'name,score\nAlice,30\nBob,\n' | ./zig-out/bin/sql-pipe --checksum --null-value 'N/A' 'SELECT name, score FROM t ORDER BY name' 2>/tmp/checksum_err) - \\checksum=$(grep 'checksum:' /tmp/checksum_err | sed 's/.*checksum: //') - \\expected=$(printf '%s' "$stdout" | ./zig-out/bin/checksum-verify) - \\[ "$checksum" = "$expected" ] - \\rm -f /tmp/checksum_err - }); - test_checksum_null_value.step.dependOn(b.getInstallStep()); - test_step.dependOn(&test_checksum_null_value.step); - - // Integration test 204o: --checksum with --verbose (both stderr outputs coexist) - const test_checksum_verbose = b.addSystemCommand(&.{ - "bash", "-c", - \\stdout=$(printf 'name,age\nAlice,30\nBob,25\nCarol,35\n' | ./zig-out/bin/sql-pipe --checksum --verbose 'SELECT name FROM t ORDER BY name' 2>/tmp/checksum_err) - \\grep -q 'checksum:' /tmp/checksum_err - \\grep -q 'Loaded 3 rows' /tmp/checksum_err - \\rm -f /tmp/checksum_err - }); - test_checksum_verbose.step.dependOn(b.getInstallStep()); - test_step.dependOn(&test_checksum_verbose.step); - - // Integration test 204p: --checksum with --explain (checksum on stdout, plan on stderr) - const test_checksum_explain = b.addSystemCommand(&.{ - "bash", "-c", - \\stdout=$(printf 'name,age\nAlice,30\nBob,25\n' | ./zig-out/bin/sql-pipe --checksum --explain 'SELECT name FROM t ORDER BY name' 2>/tmp/checksum_err) - \\grep -q 'checksum:' /tmp/checksum_err - \\grep -q 'QUERY PLAN:' /tmp/checksum_err - \\rm -f /tmp/checksum_err - }); - test_checksum_explain.step.dependOn(b.getInstallStep()); - test_step.dependOn(&test_checksum_explain.step); - - // Integration test 204q: --checksum with --save (checksum on stdout, save to file) - const test_checksum_save = b.addSystemCommand(&.{ - "bash", "-c", - \\tmp=$(mktemp) - \\stdout=$(printf 'name,age\nAlice,30\nBob,25\n' | ./zig-out/bin/sql-pipe --checksum --save "$tmp" 'SELECT name FROM t ORDER BY name' 2>/tmp/checksum_err) - \\grep -q 'checksum:' /tmp/checksum_err - \\[ "$(head -c 15 "$tmp")" = "SQLite format 3" ] - \\rm -f "$tmp" /tmp/checksum_err - }); - test_checksum_save.step.dependOn(b.getInstallStep()); - test_step.dependOn(&test_checksum_save.step); - - // Integration test 204r: --checksum with --repl (checksum per query result) - const test_checksum_repl = b.addSystemCommand(&.{ - "bash", "-c", - \\stdout=$(printf 'SELECT 1 as one;\n.exit\n' | ./zig-out/bin/sql-pipe --checksum --repl --no-stdin 2>/tmp/checksum_err) - \\grep -q 'checksum:' /tmp/checksum_err - \\rm -f /tmp/checksum_err - }); - test_checksum_repl.step.dependOn(b.getInstallStep()); - test_step.dependOn(&test_checksum_repl.step); - - // Integration test 204s: --checksum with --columns (inspect mode, no checksum) - const test_checksum_columns = b.addSystemCommand(&.{ - "bash", "-c", - \\stdout=$(printf 'name,age\nAlice,30\n' | ./zig-out/bin/sql-pipe --checksum --columns 2>/tmp/checksum_err) - \\! grep -q 'checksum:' /tmp/checksum_err - \\rm -f /tmp/checksum_err - }); - test_checksum_columns.step.dependOn(b.getInstallStep()); - test_step.dependOn(&test_checksum_columns.step); - - // Integration test 204t: --checksum with --validate (inspect mode, no checksum) - const test_checksum_validate = b.addSystemCommand(&.{ - "bash", "-c", - \\stdout=$(printf 'name,age\nAlice,30\n' | ./zig-out/bin/sql-pipe --checksum --validate 2>/tmp/checksum_err) - \\! grep -q 'checksum:' /tmp/checksum_err - \\rm -f /tmp/checksum_err - }); - test_checksum_validate.step.dependOn(b.getInstallStep()); - test_step.dependOn(&test_checksum_validate.step); - - // Integration test 204u: --checksum with --sample (inspect mode, no checksum) - const test_checksum_sample = b.addSystemCommand(&.{ - "bash", "-c", - \\stdout=$(printf 'name,age\nAlice,30\n' | ./zig-out/bin/sql-pipe --checksum --sample 1 2>/tmp/checksum_err) - \\! grep -q 'checksum:' /tmp/checksum_err - \\rm -f /tmp/checksum_err - }); - test_checksum_sample.step.dependOn(b.getInstallStep()); - test_step.dependOn(&test_checksum_sample.step); - - // Integration test 204v: --checksum with --stats (inspect mode, no checksum) - const test_checksum_stats = b.addSystemCommand(&.{ - "bash", "-c", - \\stdout=$(printf 'name,age\nAlice,30\n' | ./zig-out/bin/sql-pipe --checksum --stats 2>/tmp/checksum_err) - \\! grep -q 'checksum:' /tmp/checksum_err - \\rm -f /tmp/checksum_err - }); - test_checksum_stats.step.dependOn(b.getInstallStep()); - test_step.dependOn(&test_checksum_stats.step); - - // Integration test 204w: --checksum with --schema (inspect mode, no checksum) - const test_checksum_schema = b.addSystemCommand(&.{ - "bash", "-c", - \\stdout=$(printf 'name,age\nAlice,30\n' | ./zig-out/bin/sql-pipe --checksum --schema 2>/tmp/checksum_err) - \\! grep -q 'checksum:' /tmp/checksum_err - \\rm -f /tmp/checksum_err - }); - test_checksum_schema.step.dependOn(b.getInstallStep()); - test_step.dependOn(&test_checksum_schema.step); - - // Integration test 204x: --checksum with --help shows flag in usage - const test_checksum_help = b.addSystemCommand(&.{ - "bash", "-c", - \\./zig-out/bin/sql-pipe --help 2>&1 >/dev/null | grep -q -- '--checksum' - }); - test_checksum_help.step.dependOn(b.getInstallStep()); - test_step.dependOn(&test_checksum_help.step); - - // Integration test 204y: --checksum with --completions includes flag - const test_checksum_completions = b.addSystemCommand(&.{ - "bash", "-c", - \\./zig-out/bin/sql-pipe --completions bash | grep -q -- '--checksum' - }); - test_checksum_completions.step.dependOn(b.getInstallStep()); - test_step.dependOn(&test_checksum_completions.step); + // 25 data-driven cases (204a-204y). Scripts run with `set -euo pipefail` so a + // failed assertion actually fails the step (a bare `[` + trailing `rm -f` + // would silently mask failures). + const ChecksumTestType = enum { + checksum_match, + checksum_present, + checksum_absent, + help_flag, + completions_flag, + }; + const ChecksumTest = struct { + name: []const u8, // test identifier, e.g. "basic", "json" + args: []const u8, // sql-pipe CLI args (may contain "$tmp" for --output/--save) + input: ?[]const u8, // stdin data; null = no stdin pipe + expected_output: ?[]const u8, // checksum_match: literal expected stdout; null = hash captured stdout + check_type: ChecksumTestType, + extra_check: ?[]const u8, // extra bash assertion line (checksum_present only) + use_temp_file: bool, // wrap "$tmp" in mktemp + cleanup (--output/--save cases) + }; + const checksum_tests = [_]ChecksumTest{ + .{ .name = "basic", .args = "--checksum 'SELECT name FROM t ORDER BY age'", .input = "name,age\nAlice,30\nBob,25\n", .expected_output = "Bob\nAlice\n", .check_type = .checksum_match, .extra_check = null, .use_temp_file = false }, // 204a + .{ .name = "json", .args = "--checksum --json 'SELECT name, age FROM t ORDER BY age'", .input = "name,age\nAlice,30\nBob,25\n", .expected_output = "[{\"name\":\"Bob\",\"age\":25},{\"name\":\"Alice\",\"age\":30}]\n", .check_type = .checksum_match, .extra_check = null, .use_temp_file = false }, // 204b + .{ .name = "tsv", .args = "--checksum -O tsv 'SELECT name, age FROM t ORDER BY age'", .input = "name,age\nAlice,30\nBob,25\n", .expected_output = "Bob\t25\nAlice\t30\n", .check_type = .checksum_match, .extra_check = null, .use_temp_file = false }, // 204c + .{ .name = "table", .args = "--checksum --table 'SELECT * FROM t'", .input = "name,age\nAlice,30\nBob,25\n", .expected_output = null, .check_type = .checksum_match, .extra_check = null, .use_temp_file = false }, // 204d + .{ .name = "markdown", .args = "--checksum -O markdown 'SELECT * FROM t ORDER BY name'", .input = "name,age\nAlice,30\nBob,25\n", .expected_output = null, .check_type = .checksum_match, .extra_check = null, .use_temp_file = false }, // 204e + .{ .name = "output_file", .args = "--checksum --output \"$tmp\" 'SELECT name FROM t ORDER BY age'", .input = "name,age\nAlice,30\nBob,25\n", .expected_output = null, .check_type = .checksum_match, .extra_check = null, .use_temp_file = true }, // 204f + .{ .name = "header", .args = "--checksum --header 'SELECT name, age FROM t ORDER BY age'", .input = "name,age\nAlice,30\nBob,25\n", .expected_output = "name,age\nBob,25\nAlice,30\n", .check_type = .checksum_match, .extra_check = null, .use_temp_file = false }, // 204g + .{ .name = "sql", .args = "--checksum -O sql 'SELECT * FROM t ORDER BY name'", .input = "name,age\nAlice,30\nBob,25\n", .expected_output = null, .check_type = .checksum_match, .extra_check = null, .use_temp_file = false }, // 204h + .{ .name = "html", .args = "--checksum -O html 'SELECT * FROM t ORDER BY name'", .input = "name,age\nAlice,30\nBob,25\n", .expected_output = null, .check_type = .checksum_match, .extra_check = null, .use_temp_file = false }, // 204i + .{ .name = "xml", .args = "--checksum -O xml 'SELECT * FROM t ORDER BY name'", .input = "name,age\nAlice,30\nBob,25\n", .expected_output = null, .check_type = .checksum_match, .extra_check = null, .use_temp_file = false }, // 204j + .{ .name = "ndjson", .args = "--checksum -O ndjson 'SELECT name, age FROM t ORDER BY age'", .input = "name,age\nAlice,30\nBob,25\n", .expected_output = "{\"name\":\"Bob\",\"age\":25}\n{\"name\":\"Alice\",\"age\":30}\n", .check_type = .checksum_match, .extra_check = null, .use_temp_file = false }, // 204k + .{ .name = "empty", .args = "--checksum 'SELECT name FROM t WHERE age > 100'", .input = "name,age\nAlice,30\n", .expected_output = "", .check_type = .checksum_match, .extra_check = null, .use_temp_file = false }, // 204l + .{ .name = "disk", .args = "--checksum --disk 'SELECT name FROM t WHERE age > 27'", .input = "name,age\nAlice,30\nBob,25\n", .expected_output = "Alice\n", .check_type = .checksum_match, .extra_check = null, .use_temp_file = false }, // 204m + .{ .name = "null_value", .args = "--checksum --null-value 'N/A' 'SELECT name, score FROM t ORDER BY name'", .input = "name,score\nAlice,30\nBob,\n", .expected_output = null, .check_type = .checksum_match, .extra_check = null, .use_temp_file = false }, // 204n + .{ .name = "verbose", .args = "--checksum --verbose 'SELECT name FROM t ORDER BY name'", .input = "name,age\nAlice,30\nBob,25\nCarol,35\n", .expected_output = null, .check_type = .checksum_present, .extra_check = "grep -q 'Loaded 3 rows' \"$err_file\"", .use_temp_file = false }, // 204o + .{ .name = "explain", .args = "--checksum --explain 'SELECT name FROM t ORDER BY name'", .input = "name,age\nAlice,30\nBob,25\n", .expected_output = null, .check_type = .checksum_present, .extra_check = "grep -q 'QUERY PLAN:' \"$err_file\"", .use_temp_file = false }, // 204p + .{ .name = "save", .args = "--checksum --save \"$tmp\" 'SELECT name FROM t ORDER BY name'", .input = "name,age\nAlice,30\nBob,25\n", .expected_output = null, .check_type = .checksum_present, .extra_check = "[ \"$(head -c 15 \"$tmp\")\" = \"SQLite format 3\" ]", .use_temp_file = true }, // 204q + .{ .name = "repl", .args = "--checksum --repl --no-stdin", .input = "SELECT 1 as one;\n.exit\n", .expected_output = null, .check_type = .checksum_present, .extra_check = null, .use_temp_file = false }, // 204r + .{ .name = "columns", .args = "--checksum --columns", .input = "name,age\nAlice,30\n", .expected_output = null, .check_type = .checksum_absent, .extra_check = null, .use_temp_file = false }, // 204s + .{ .name = "validate", .args = "--checksum --validate", .input = "name,age\nAlice,30\n", .expected_output = null, .check_type = .checksum_absent, .extra_check = null, .use_temp_file = false }, // 204t + .{ .name = "sample", .args = "--checksum --sample 1", .input = "name,age\nAlice,30\n", .expected_output = null, .check_type = .checksum_absent, .extra_check = null, .use_temp_file = false }, // 204u + .{ .name = "stats", .args = "--checksum --stats", .input = "name,age\nAlice,30\n", .expected_output = null, .check_type = .checksum_absent, .extra_check = null, .use_temp_file = false }, // 204v + .{ .name = "schema", .args = "--checksum --schema", .input = "name,age\nAlice,30\n", .expected_output = null, .check_type = .checksum_absent, .extra_check = null, .use_temp_file = false }, // 204w + .{ .name = "help", .args = "--help", .input = null, .expected_output = null, .check_type = .help_flag, .extra_check = null, .use_temp_file = false }, // 204x + .{ .name = "completions", .args = "--completions bash", .input = null, .expected_output = null, .check_type = .completions_flag, .extra_check = null, .use_temp_file = false }, // 204y + }; + + for (checksum_tests) |t| { + const script = switch (t.check_type) { + .help_flag => b.allocator.dupe(u8, "./zig-out/bin/sql-pipe --help 2>&1 >/dev/null | grep -q -- '--checksum'") catch unreachable, + .completions_flag => b.allocator.dupe(u8, "./zig-out/bin/sql-pipe --completions bash | grep -q -- '--checksum'") catch unreachable, + .checksum_absent => std.fmt.allocPrint(b.allocator, + \\set -euo pipefail + \\err_file=$(mktemp) + \\stdout=$(printf '{s}' | ./zig-out/bin/sql-pipe {s} 2>"$err_file") + \\! grep -q 'checksum:' "$err_file" + \\rm -f "$err_file" + , .{ t.input.?, t.args }) catch unreachable, + .checksum_present => if (t.use_temp_file) + std.fmt.allocPrint(b.allocator, + \\set -euo pipefail + \\tmp=$(mktemp) + \\err_file=$(mktemp) + \\stdout=$(printf '{s}' | ./zig-out/bin/sql-pipe {s} 2>"$err_file") + \\grep -q 'checksum:' "$err_file" + \\{s} + \\rm -f "$tmp" "$err_file" + , .{ t.input.?, t.args, t.extra_check.? }) catch unreachable + else + std.fmt.allocPrint(b.allocator, + \\set -euo pipefail + \\err_file=$(mktemp) + \\stdout=$(printf '{s}' | ./zig-out/bin/sql-pipe {s} 2>"$err_file") + \\grep -q 'checksum:' "$err_file" + \\{s} + \\rm -f "$err_file" + , .{ t.input.?, t.args, t.extra_check orelse "" }) catch unreachable, + .checksum_match => if (t.use_temp_file) + std.fmt.allocPrint(b.allocator, + \\set -euo pipefail + \\tmp=$(mktemp) + \\err_file=$(mktemp) + \\stdout=$(printf '{s}' | ./zig-out/bin/sql-pipe {s} 2>"$err_file") + \\checksum=$(grep 'checksum:' "$err_file" | sed 's/.*checksum: //') + \\expected=$(cat "$tmp" | sha256sum | awk '{{print $1}}') + \\[ "$checksum" = "$expected" ] + \\rm -f "$tmp" "$err_file" + , .{ t.input.?, t.args }) catch unreachable + else if (t.expected_output) |expected| + std.fmt.allocPrint(b.allocator, + \\set -euo pipefail + \\err_file=$(mktemp) + \\stdout=$(printf '{s}' | ./zig-out/bin/sql-pipe {s} 2>"$err_file") + \\checksum=$(grep 'checksum:' "$err_file" | sed 's/.*checksum: //') + \\expected=$(printf '{s}' | sha256sum | awk '{{print $1}}') + \\[ "$checksum" = "$expected" ] + \\rm -f "$err_file" + , .{ t.input.?, t.args, expected }) catch unreachable + else + // Hash redirected stdout: $( ) capture strips trailing newlines, + // which would break the checksum comparison. + std.fmt.allocPrint(b.allocator, + \\set -euo pipefail + \\tmp=$(mktemp) + \\err_file=$(mktemp) + \\printf '{s}' | ./zig-out/bin/sql-pipe {s} > "$tmp" 2>"$err_file" + \\checksum=$(grep 'checksum:' "$err_file" | sed 's/.*checksum: //') + \\expected=$(sha256sum "$tmp" | awk '{{print $1}}') + \\[ "$checksum" = "$expected" ] + \\rm -f "$tmp" "$err_file" + , .{ t.input.?, t.args }) catch unreachable, + }; + const test_checksum = b.addSystemCommand(&.{ "bash", "-c", script }); + test_checksum.step.dependOn(b.getInstallStep()); + test_step.dependOn(&test_checksum.step); + } } diff --git a/src/checksum_verify.zig b/src/checksum_verify.zig deleted file mode 100644 index 630a6f9..0000000 --- a/src/checksum_verify.zig +++ /dev/null @@ -1,40 +0,0 @@ -const std = @import("std"); - -pub fn main() !void { - var buffer = std.ArrayList(u8).empty; - defer buffer.deinit(std.heap.page_allocator); - - // Read all stdin byte by byte - var io = std.Io.Threaded.init_single_threaded; - var stdin_buf: [4096]u8 = undefined; - var stdin_reader = std.Io.File.reader(std.Io.File.stdin(), io.io(), &stdin_buf); - - while (true) { - const byte = stdin_reader.interface.takeByte() catch |err| switch (err) { - error.EndOfStream => break, - else => return err, - }; - try buffer.append(std.heap.page_allocator, byte); - } - - // Compute SHA-256 - var hash: [32]u8 = undefined; - var hasher = std.crypto.hash.sha2.Sha256.init(.{}); - hasher.update(buffer.items); - hasher.final(&hash); - - // Convert to hex - var hex_buf: [64]u8 = undefined; - const hex_chars = "0123456789abcdef"; - for (hash, 0..) |byte, i| { - hex_buf[2 * i] = hex_chars[byte >> 4]; - hex_buf[2 * i + 1] = hex_chars[byte & 0x0F]; - } - - // Output hex - var stdout_buf: [1024]u8 = undefined; - var stdout_file_writer = std.Io.File.writer(std.Io.File.stdout(), io.io(), &stdout_buf); - try stdout_file_writer.interface.writeAll(hex_buf[0..]); - try stdout_file_writer.interface.writeByte('\n'); - try stdout_file_writer.interface.flush(); -} diff --git a/src/main.zig b/src/main.zig index 3b5fe4d..79e8ca4 100644 --- a/src/main.zig +++ b/src/main.zig @@ -29,44 +29,58 @@ const printUsage = args_mod.printUsage; const loadCsvInput = loader.loadCsvInput; const fmtThousands = loader.fmtThousands; -/// Buffer writer that captures all output to an ArrayList for checksum computation. -/// Uses std.Io.Writer.Allocating for dynamic allocation. -const BufferWriter = std.Io.Writer.Allocating; +/// Run a write function directly against stdout, or buffer its output and emit +/// a SHA-256 checksum of that output to stderr when checksum is true. +/// `pre_args` are spliced around the writer argument in write_fn's parameter +/// list (write_fn must take a *std.Io.Writer as its last parameter). +fn writeWithChecksum( + allocator: std.mem.Allocator, + stdout_writer: *std.Io.Writer, + stderr_writer: *std.Io.Writer, + checksum: bool, + pre_args: anytype, + comptime write_fn: anytype, +) !void { + if (!checksum) { + try @call(.auto, write_fn, pre_args ++ .{stdout_writer}); + return; + } + + // ponytail: buffers full result set in memory for SHA-256; for GB-scale + // output, wrap writer in a hashing tee writer (O(1) memory). + var buffer_writer = std.Io.Writer.Allocating.init(allocator); + defer buffer_writer.deinit(); + try @call(.auto, write_fn, pre_args ++ .{&buffer_writer.writer}); + + const buffer = buffer_writer.toArrayList(); -/// Compute SHA-256 checksum of bytes and return hex string (allocated). -fn computeChecksum(allocator: std.mem.Allocator, data: []const u8) ![]const u8 { + // Compute SHA-256 of buffered output and convert to hex. var hash: [32]u8 = undefined; var hasher = std.crypto.hash.sha2.Sha256.init(.{}); - hasher.update(data); + hasher.update(buffer.items); hasher.final(&hash); + const hex = std.fmt.bytesToHex(hash, .lower); - // Convert to hex - var hex_buf: [64]u8 = undefined; - const hex_chars = "0123456789abcdef"; - for (hash, 0..) |byte, i| { - hex_buf[2 * i] = hex_chars[byte >> 4]; - hex_buf[2 * i + 1] = hex_chars[byte & 0x0F]; - } - return try allocator.dupe(u8, hex_buf[0..]); + // Write buffered output to stdout, then emit checksum to stderr. + try stdout_writer.writeAll(buffer.items); + try stderr_writer.print("checksum: {s}\n", .{&hex}); + try stderr_writer.flush(); } -/// Write buffered output to stdout and emit checksum to stderr. -fn emitChecksum( +/// Stream all result rows through the format OutputWriter (writer is last param +/// so it can be passed to writeWithChecksum). +fn writeStreaming( + out_writer: *format.OutputWriter, allocator: std.mem.Allocator, - buffer_writer: *BufferWriter, - stdout_writer: *std.Io.Writer, - stderr_writer: *std.Io.Writer, + stmt: *c.sqlite3_stmt, + col_count: c_int, + writer: *std.Io.Writer, ) !void { - const buffer = buffer_writer.toArrayList(); - const hex_str = try computeChecksum(allocator, buffer.items); - defer allocator.free(hex_str); - - // Write buffered output to actual stdout writer - try stdout_writer.writeAll(buffer.items); - - // Emit checksum to stderr - try stderr_writer.print("checksum: {s}\n", .{hex_str}); - try stderr_writer.flush(); + try out_writer.begin(allocator, stmt, col_count, writer); + while (c.sqlite3_step(stmt) == c.SQLITE_ROW) { + try out_writer.writeRow(stmt, writer); + } + try out_writer.end(writer); } const progress_interval = loader.progress_interval; @@ -115,27 +129,13 @@ pub fn execQuery( // Table mode: buffer all rows and print a formatted table if (use_table) { - if (checksum) { - var buffer_writer = BufferWriter.init(allocator); - defer buffer_writer.deinit(); - try table.writeTable(allocator, &buffer_writer.writer, stmt.?, col_count, null_value); - try emitChecksum(allocator, &buffer_writer, writer, stderr_writer); - } else { - try table.writeTable(allocator, writer, stmt.?, col_count, null_value); - } + try writeWithChecksum(allocator, writer, stderr_writer, checksum, .{allocator, stmt.?, col_count, null_value}, table.writeTable); return; } // Markdown output: two-pass writer (not streaming) if (output_format == .markdown) { - if (checksum) { - var buffer_writer = BufferWriter.init(allocator); - defer buffer_writer.deinit(); - try markdown.writeMarkdown(allocator, &buffer_writer.writer, stmt.?, col_count, null_value); - try emitChecksum(allocator, &buffer_writer, writer, stderr_writer); - } else { - try markdown.writeMarkdown(allocator, writer, stmt.?, col_count, null_value); - } + try writeWithChecksum(allocator, writer, stderr_writer, checksum, .{allocator, stmt.?, col_count, null_value}, markdown.writeMarkdown); return; } @@ -149,24 +149,7 @@ pub fn execQuery( }); defer out_writer.deinit(allocator); - if (checksum) { - var buffer_writer = BufferWriter.init(allocator); - defer buffer_writer.deinit(); - - try out_writer.begin(allocator, stmt.?, col_count, &buffer_writer.writer); - while (c.sqlite3_step(stmt) == c.SQLITE_ROW) { - try out_writer.writeRow(stmt.?, &buffer_writer.writer); - } - try out_writer.end(&buffer_writer.writer); - - try emitChecksum(allocator, &buffer_writer, writer, stderr_writer); - } else { - try out_writer.begin(allocator, stmt.?, col_count, writer); - while (c.sqlite3_step(stmt) == c.SQLITE_ROW) { - try out_writer.writeRow(stmt.?, writer); - } - try out_writer.end(writer); - } + try writeWithChecksum(allocator, writer, stderr_writer, checksum, .{ &out_writer, allocator, stmt.?, col_count }, writeStreaming); } /// loadInput(allocator, io, db, table_name, input_format, reader, parsed, stderr_writer) → usize @@ -598,37 +581,3 @@ pub fn main(init: std.process.Init.Minimal) void { }, } } - -test "computeChecksum - empty buffer" { - const allocator = std.testing.allocator; - const result = try computeChecksum(allocator, ""); - defer allocator.free(result); - // SHA-256 of empty string - try std.testing.expectEqualStrings(result, "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"); -} - -test "computeChecksum - simple string" { - const allocator = std.testing.allocator; - const result = try computeChecksum(allocator, "hello"); - defer allocator.free(result); - // SHA-256 of "hello" - try std.testing.expectEqualStrings(result, "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824"); -} - -test "computeChecksum - multi-line output" { - const allocator = std.testing.allocator; - const input = "Bob\nAlice\n"; - const result = try computeChecksum(allocator, input); - defer allocator.free(result); - // SHA-256 of "Bob\nAlice\n" - try std.testing.expectEqualStrings(result, "081a774cb12f7bd5ea746c3b516da7b5bb8d6e7f62a30c6416f1e79c8958aef7"); -} - -test "computeChecksum - JSON output" { - const allocator = std.testing.allocator; - const input = "[{\"name\":\"Bob\",\"age\":25},{\"name\":\"Alice\",\"age\":30}]\n"; - const result = try computeChecksum(allocator, input); - defer allocator.free(result); - // SHA-256 of the JSON string - try std.testing.expectEqualStrings(result, "22969ee30b434160d184c0b2819a52fb81ada7943ff3d33faef50177697e1d63"); -} diff --git a/src/markdown.zig b/src/markdown.zig index 7b99512..458d0e4 100644 --- a/src/markdown.zig +++ b/src/markdown.zig @@ -21,10 +21,10 @@ const visual = @import("visual.zig"); /// Memory: uses an arena allocator internally; all memory is freed on return. pub fn writeMarkdown( allocator: std.mem.Allocator, - writer: *std.Io.Writer, stmt: *c.sqlite3_stmt, col_count: c_int, null_value: ?[]const u8, + writer: *std.Io.Writer, ) (std.mem.Allocator.Error || error{WriteFailed, StepFailed})!void { var arena = std.heap.ArenaAllocator.init(allocator); defer arena.deinit(); diff --git a/src/modes/stats.zig b/src/modes/stats.zig index 01c4373..88ca0fa 100644 --- a/src/modes/stats.zig +++ b/src/modes/stats.zig @@ -112,7 +112,7 @@ fn printTableStats( const col_count = c.sqlite3_column_count(stmt); if (col_count == 0) return; - table.writeTable(allocator, stdout_writer, stmt.?, col_count, null) catch |err| { + table.writeTable(allocator, stmt.?, col_count, null, stdout_writer) catch |err| { std.log.err("failed to write stats table: {}", .{err}); std.process.exit(@intFromEnum(ExitCode.usage)); }; diff --git a/src/table.zig b/src/table.zig index e13b049..c2512f4 100644 --- a/src/table.zig +++ b/src/table.zig @@ -22,10 +22,10 @@ const visual = @import("visual.zig"); /// Memory: uses an arena allocator internally; all memory is freed on return. pub fn writeTable( allocator: std.mem.Allocator, - writer: *std.Io.Writer, stmt: *c.sqlite3_stmt, col_count: c_int, null_value: ?[]const u8, + writer: *std.Io.Writer, ) (std.mem.Allocator.Error || error{WriteFailed, StepFailed})!void { var arena = std.heap.ArenaAllocator.init(allocator); defer arena.deinit(); @@ -281,7 +281,7 @@ test "isNumericString" { test "writeTable parameter order" { // Verify the public API compiles with the correct parameter order: - // writeTable(allocator, writer, stmt, col_count, null_value) + // writeTable(allocator, stmt, col_count, null_value, writer) // We can't easily call writeTable in a unit test without a database, // but we can verify the type signature. try std.testing.expect(true); From 09e95d7a0594501d4c634067f3cb4a4cff235abf Mon Sep 17 00:00:00 2001 From: "Victor M. Varela" Date: Sun, 2 Aug 2026 18:35:03 +0200 Subject: [PATCH 4/5] fix: memory leak in writeWithChecksum, error handling in run (ora-3) - writeWithChecksum: buffer ArrayList never freed after toArrayList(); in --repl mode this leaked per query. Added defer buffer.deinit(allocator). - run(): execQuery catch now switches on error type like repl.zig; OutOfMemory gets specific message, not misleading SQL error. - Extended ponytail comment: --checksum defeats --disk (all in RAM). --- src/main.zig | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/src/main.zig b/src/main.zig index 79e8ca4..07eac25 100644 --- a/src/main.zig +++ b/src/main.zig @@ -47,12 +47,14 @@ fn writeWithChecksum( } // ponytail: buffers full result set in memory for SHA-256; for GB-scale - // output, wrap writer in a hashing tee writer (O(1) memory). + // output, wrap writer in a hashing tee writer (O(1) memory). Note that + // --checksum defeats --disk: all output buffers in RAM regardless. + // Upgrade path: tee writer (O(1) memory) covers both --disk and --checksum. var buffer_writer = std.Io.Writer.Allocating.init(allocator); - defer buffer_writer.deinit(); try @call(.auto, write_fn, pre_args ++ .{&buffer_writer.writer}); - const buffer = buffer_writer.toArrayList(); + var buffer = buffer_writer.toArrayList(); + defer buffer.deinit(allocator); // Compute SHA-256 of buffered output and convert to hex. var hash: [32]u8 = undefined; @@ -352,9 +354,19 @@ fn run( printQueryPlan(allocator, db, query, main_table, stderr_writer); } - execQuery(allocator, db, query, stdout_writer, stderr_writer, parsed.header, parsed.output_format, parsed.xml_root, parsed.xml_row, parsed.sql_table, parsed.html_class, parsed.null_value, use_table, parsed.checksum) catch { - stdout_writer.flush() catch |err| std.log.err("failed to flush output before fatal: {}", .{err}); - sqlite_mod.fatalSqlWithContext(allocator, db, main_table, std.mem.span(c.sqlite3_errmsg(db)), stderr_writer); + execQuery(allocator, db, query, stdout_writer, stderr_writer, parsed.header, parsed.output_format, parsed.xml_root, parsed.xml_row, parsed.sql_table, parsed.html_class, parsed.null_value, use_table, parsed.checksum) catch |err| switch (err) { + error.PrepareQueryFailed => { + stdout_writer.flush() catch |flush_err| std.log.err("failed to flush output before fatal: {}", .{flush_err}); + sqlite_mod.fatalSqlWithContext(allocator, db, main_table, std.mem.span(c.sqlite3_errmsg(db)), stderr_writer); + }, + error.OutOfMemory => { + stdout_writer.flush() catch |flush_err| std.log.err("failed to flush output before fatal: {}", .{flush_err}); + fatal("out of memory", stderr_writer, .csv_error, .{}); + }, + else => { + stdout_writer.flush() catch |flush_err| std.log.err("failed to flush output before fatal: {}", .{flush_err}); + fatal("{s}", stderr_writer, .csv_error, .{@errorName(err)}); + }, }; } From a82c987755105bce9e02952caff3d0d7b938c5ae Mon Sep 17 00:00:00 2001 From: "Victor M. Varela" Date: Sun, 2 Aug 2026 18:42:52 +0200 Subject: [PATCH 5/5] docs: fix --max-rows guidance, add --checksum --silent + --repl multi tests (ora-3) - README/man page: --max-rows caps input rows, not output rows. Suggest LIMIT clause instead. Note --disk defeated by --checksum. - Tests: --checksum --silent verifies checksum still emits. --checksum --repl multi-query verifies per-query behavior. --- README.md | 2 +- build.zig | 2 ++ docs/sql-pipe.1.scd | 6 ++++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index d945933..ec52f2d 100644 --- a/README.md +++ b/README.md @@ -600,7 +600,7 @@ Alice The SHA-256 hash of the result set is printed to stderr as `checksum: `. The hash covers only stdout output (the result set), so it works correctly with `--output`, `--verbose`, `--explain`, and other flags that write to stderr. Skipped in inspect modes (`--columns`, `--validate`, `--sample`, `--stats`, `--schema`). -> **Note:** The entire result set is buffered in memory to compute the checksum. For very large result sets, this may consume significant RAM. Consider using `--max-rows` to limit the result size if memory is a concern. +> **Note:** The entire result set is buffered in memory to compute the checksum. For very large result sets, this may consume significant RAM. Add a `LIMIT` clause to your query to bound the result size. `--max-rows` caps input rows, not output rows. Note: `--checksum` defeats `--disk` — the result set always buffers in RAM regardless of database backing. ## Real-world examples diff --git a/build.zig b/build.zig index aecf2f9..8fac94e 100644 --- a/build.zig +++ b/build.zig @@ -3782,6 +3782,8 @@ pub fn build(b: *std.Build) void { .{ .name = "explain", .args = "--checksum --explain 'SELECT name FROM t ORDER BY name'", .input = "name,age\nAlice,30\nBob,25\n", .expected_output = null, .check_type = .checksum_present, .extra_check = "grep -q 'QUERY PLAN:' \"$err_file\"", .use_temp_file = false }, // 204p .{ .name = "save", .args = "--checksum --save \"$tmp\" 'SELECT name FROM t ORDER BY name'", .input = "name,age\nAlice,30\nBob,25\n", .expected_output = null, .check_type = .checksum_present, .extra_check = "[ \"$(head -c 15 \"$tmp\")\" = \"SQLite format 3\" ]", .use_temp_file = true }, // 204q .{ .name = "repl", .args = "--checksum --repl --no-stdin", .input = "SELECT 1 as one;\n.exit\n", .expected_output = null, .check_type = .checksum_present, .extra_check = null, .use_temp_file = false }, // 204r + .{ .name = "silent", .args = "--checksum --silent 'SELECT name FROM t'", .input = "name,age\nAlice,30\nBob,25\n", .expected_output = "Alice\nBob\n", .check_type = .checksum_match, .extra_check = null, .use_temp_file = false }, // 204z + .{ .name = "repl_multi", .args = "--checksum --repl --no-stdin", .input = "SELECT 1 as one;\nSELECT 2 as two;\nSELECT 3 as three;\n.exit\n", .expected_output = null, .check_type = .checksum_present, .extra_check = "[ \"$(grep -c 'checksum:' \"$err_file\")\" = \"3\" ]", .use_temp_file = false }, // 204aa .{ .name = "columns", .args = "--checksum --columns", .input = "name,age\nAlice,30\n", .expected_output = null, .check_type = .checksum_absent, .extra_check = null, .use_temp_file = false }, // 204s .{ .name = "validate", .args = "--checksum --validate", .input = "name,age\nAlice,30\n", .expected_output = null, .check_type = .checksum_absent, .extra_check = null, .use_temp_file = false }, // 204t .{ .name = "sample", .args = "--checksum --sample 1", .input = "name,age\nAlice,30\n", .expected_output = null, .check_type = .checksum_absent, .extra_check = null, .use_temp_file = false }, // 204u diff --git a/docs/sql-pipe.1.scd b/docs/sql-pipe.1.scd index 5f472c0..75e72cd 100644 --- a/docs/sql-pipe.1.scd +++ b/docs/sql-pipe.1.scd @@ -253,8 +253,10 @@ OPTIONS The entire result set is buffered in memory to compute the checksum. For very large result sets, this may consume significant RAM. - Consider using *--max-rows* to limit the result size if memory is - a concern. + Add a *LIMIT* clause to your query to bound the result size. + *--max-rows* caps input rows, not output rows. Note: + *--checksum* defeats *--disk* — the result set always buffers in + RAM regardless of database backing. *-r, --repl* Enter an interactive REPL (read-eval-print loop) after loading