From 719c71814f87f25f288de463df0c5130dcc22d2b Mon Sep 17 00:00:00 2001 From: jinzhongjia Date: Mon, 14 Sep 2026 14:11:05 +0800 Subject: [PATCH 1/4] fix: validate string lengths before allocation and body reads --- README.md | 4 ++-- README_CN.md | 4 ++-- src/msgpack.zig | 14 ++++++----- src/test.zig | 62 +++++++++++++++++++++++-------------------------- 4 files changed, 41 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index 677aa4f..dbacb9d 100644 --- a/README.md +++ b/README.md @@ -521,8 +521,8 @@ This library uses an **iterative parser** (not recursive) to provide strong secu **Safety Limits:** -- All limits are enforced **before** memory allocation -- Invalid input is rejected immediately without resource consumption +- For fixstr, str8, str16, and str32, string lengths are checked **before** allocating or reading string contents. +- Oversized strings return `StringTooLong` after consuming the marker and length prefix; string contents remain unread. - Configurable limits allow tuning for specific environments (embedded, server, etc.) **Memory Safety:** diff --git a/README_CN.md b/README_CN.md index 7d788f6..13419f8 100644 --- a/README_CN.md +++ b/README_CN.md @@ -429,8 +429,8 @@ msgpack.MsgPackError.ExtDataTooLarge // 扩展类型数据过大 **安全限制**： -- 所有限制在内存分配**之前**强制执行 -- 无效输入被立即拒绝，不消耗资源 +- fixstr、str8、str16、str32 均在分配字符串内存、读取正文**之前**检查长度。 +- 超限字符串返回 `StringTooLong`；此时仅消费了类型标记和长度前缀，正文尚未读取。 - 可配置限制允许针对特定环境调整（嵌入式、服务器等） **内存安全**： diff --git a/src/msgpack.zig b/src/msgpack.zig index 51ac255..48979a1 100644 --- a/src/msgpack.zig +++ b/src/msgpack.zig @@ -2456,8 +2456,15 @@ pub fn PackWithLimits( } } + inline fn validateStrLength(len: usize) !void { + if (len > parse_limits.max_string_length) { + return MsgPackError.StringTooLong; + } + } + fn readFixStrValue(self: Self, allocator: Allocator, marker_u8: u8) ![]const u8 { const len: u8 = marker_u8 - @intFromEnum(Markers.FIXSTR); + try validateStrLength(len); const str = try self.readData(allocator, len); return str; @@ -2467,6 +2474,7 @@ pub fn PackWithLimits( /// Reduces code duplication for STR8/16/32 inline fn readStrValueGeneric(self: Self, comptime LenType: type, allocator: Allocator) ![]const u8 { const len = try self.readTypedInt(LenType); + try validateStrLength(len); return try self.readData(allocator, len); } @@ -2912,12 +2920,6 @@ pub fn PackWithLimits( .FIXSTR, .STR8, .STR16, .STR32 => { const val = try self.readStrValue(marker_u8, allocator); - // Validate string length - if (val.len > parse_limits.max_string_length) { - allocator.free(val); - return MsgPackError.StringTooLong; - } - current_payload = Payload{ .str = Str.init(val) }; }, .BIN8, .BIN16, .BIN32 => { diff --git a/src/test.zig b/src/test.zig index e5c0c83..179bc17 100644 --- a/src/test.zig +++ b/src/test.zig @@ -4171,29 +4171,15 @@ test "corrupted: nested arrays with mismatched counts" { } test "malicious: str32 with excessive length claim" { - var buffer: [1000]u8 = undefined; - var input_buf: [10]u8 = undefined; - - // str32 claiming 100MB (will be rejected by limit) - input_buf[0] = 0xdb; // str32 - input_buf[1] = 0x06; // 100MB = 0x06400000 - input_buf[2] = 0x40; - input_buf[3] = 0x00; - input_buf[4] = 0x00; - - var write_buffer = fixedBufferStream(&buffer); - var read_buffer = fixedBufferStream(&input_buf); + var input = [_]u8{ 0xdb, 0xff, 0xff, 0xff, 0xff }; + var output: [0]u8 = .{}; + var write_buffer = fixedBufferStream(&output); + var read_buffer = fixedBufferStream(&input); var p = pack.init(&write_buffer, &read_buffer); + var failing = std.testing.FailingAllocator.init(allocator, .{ .fail_index = 0 }); - const result = p.read(allocator); - if (result) |payload| { - payload.free(allocator); - try expect(false); // Should not succeed - } else |err| { - // Should be LengthReading (can't read 100MB) or StringTooLong - try expect(err == msgpack.MsgPackError.LengthReading or - err == msgpack.MsgPackError.StringTooLong); - } + try std.testing.expectError(msgpack.MsgPackError.StringTooLong, p.read(failing.allocator())); + try expect(!failing.has_induced_failure); } // ========== Tests for Generic Map Keys (Non-String Keys) ========== @@ -5525,7 +5511,7 @@ test "PackerIO: packIO convenience function" { // ParseLimits: error path coverage for limits not exercised elsewhere // ============================================================================ -test "iterative parser: string too long" { +test "string limit: reject before allocation or body read" { const custom_pack = msgpack.PackWithLimits( *bufferType, *bufferType, @@ -5536,17 +5522,27 @@ test "iterative parser: string too long" { .{ .max_string_length = 10 }, ); - // str8 marker (0xd9) + length=20 + 20 bytes of zeros - var arr: [256]u8 = std.mem.zeroes([256]u8); - arr[0] = 0xd9; - arr[1] = 20; - - var write_buffer = fixedBufferStream(&arr); - var read_buffer = fixedBufferStream(&arr); - var p = custom_pack.init(&write_buffer, &read_buffer); - - const result = p.read(allocator); - try std.testing.expectError(msgpack.MsgPackError.StringTooLong, result); + // Each string format has its own length-decoding path. + const headers = [_][]const u8{ + &.{0xab}, + &.{ 0xd9, 11 }, + &.{ 0xda, 0, 11 }, + &.{ 0xdb, 0, 0, 0, 11 }, + }; + for (headers) |header| { + var input: [16]u8 = undefined; + @memcpy(input[0..header.len], header); + @memset(input[header.len..], 'x'); + var output: [0]u8 = .{}; + var write_buffer = fixedBufferStream(&output); + var read_buffer = fixedBufferStream(&input); + var p = custom_pack.init(&write_buffer, &read_buffer); + var failing = std.testing.FailingAllocator.init(allocator, .{ .fail_index = 0 }); + + try std.testing.expectError(msgpack.MsgPackError.StringTooLong, p.read(failing.allocator())); + try expect(!failing.has_induced_failure); + try std.testing.expectEqual(header.len, read_buffer.pos); + } } test "iterative parser: bin too long" { From 913afd1cd1d2da98a1f706d1d1f7aa03b7433e49 Mon Sep 17 00:00:00 2001 From: jinzhongjia Date: Mon, 14 Sep 2026 14:17:47 +0800 Subject: [PATCH 2/4] docs: recommend v0.0.17 for Zig 0.15 and older --- README.md | 5 ++--- README_CN.md | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index dbacb9d..57999f2 100644 --- a/README.md +++ b/README.md @@ -44,12 +44,11 @@ This library is tested and optimized for all major platforms and architectures: | Zig Version | Library Version | Status | | -------------------- | --------------- | ------------------------------------- | -| 0.13 and older | 0.0.6 | Legacy support | -| 0.14.x / 0.15.x | Earlier releases | Not supported by the current version | +| 0.15.x and older | 0.0.17 | Legacy support | | 0.16.0 | Current | Supported with compatibility layer | | 0.17.0-dev | Current | Initial support; CI tracks `master` | -> **Note:** For Zig 0.13 and older versions, please use version `0.0.6` of this library. +> **Note:** For Zig 0.15.x and older versions, please use version `0.0.17` of this library. > **Note:** The current library requires Zig `0.16.0` or later. Zig `0.17.0-dev` is unreleased; compatibility may change as development continues. > **Note:** Zig 0.16+ removes `std.io.FixedBufferStream`, but this library provides a compatibility layer to maintain the same API across all supported versions. diff --git a/README_CN.md b/README_CN.md index 13419f8..62ea028 100644 --- a/README_CN.md +++ b/README_CN.md @@ -44,12 +44,11 @@ Zig 编程语言的 MessagePack 实现。此库提供了一种简单高效的方 | Zig 版本 | 库版本 | 状态 | | -------------------- | -------- | ----------------- | -| 0.13 及更早版本 | 0.0.6 | 旧版支持 | -| 0.14.x / 0.15.x | 历史版本 | 当前版本不再支持 | +| 0.15.x 及更早版本 | 0.0.17 | 旧版支持 | | 0.16.0 | 当前版本 | 通过兼容层支持 | | 0.17.0-dev | 当前版本 | 初步支持；CI 跟踪 `master` | -> **注意**: 对于 Zig 0.13 及更早版本，请使用本库的 `0.0.6` 版本。 +> **注意**: 如需支持 Zig 0.15.x 及更早版本，请使用本库的 `0.0.17` 版本。 > **注意**: 当前库要求 Zig `0.16.0` 或更高版本。Zig `0.17.0-dev` 尚未发布，兼容性可能随开发进展而变化。 > **注意**: Zig 0.16+ 移除了 `std.io.FixedBufferStream`，但本库提供了兼容层以在所有支持的版本中维持相同的 API。 From 3f74b5d21ff1c76534bac80e6c6e93bde19960cf Mon Sep 17 00:00:00 2001 From: jinzhongjia Date: Tue, 15 Sep 2026 13:54:58 +0800 Subject: [PATCH 3/4] fix: free payloads without allocation or recursion --- README.md | 1 + README_CN.md | 1 + src/msgpack.zig | 185 ++++++++++++++++++++---------------------------- src/test.zig | 100 +++++++++++++++++++++++--- 4 files changed, 169 insertions(+), 118 deletions(-) diff --git a/README.md b/README.md index 57999f2..8e1eae7 100644 --- a/README.md +++ b/README.md @@ -526,6 +526,7 @@ This library uses an **iterative parser** (not recursive) to provide strong secu **Memory Safety:** +- `Payload.free(allocator)` releases owned data without allocating or recursing, even when the allocator is exhausted. - All error paths include complete cleanup (`errdefer` + `cleanupParseStack`) - Zero memory leaks verified by GPA (General Purpose Allocator) in tests - Safe to parse untrusted data from network, files, or user input diff --git a/README_CN.md b/README_CN.md index 62ea028..7deded4 100644 --- a/README_CN.md +++ b/README_CN.md @@ -434,6 +434,7 @@ msgpack.MsgPackError.ExtDataTooLarge // 扩展类型数据过大 **内存安全**： +- `Payload.free(allocator)` 无需分配新内存或递归即可释放其拥有的数据，即使分配器已耗尽也能完成清理。 - 所有错误路径包含完整清理（`errdefer` + `cleanupParseStack`） - 零内存泄漏（测试中由 GPA 验证） - 可安全解析来自网络、文件或用户输入的不可信数据 diff --git a/src/msgpack.zig b/src/msgpack.zig index 48979a1..d00dc73 100644 --- a/src/msgpack.zig +++ b/src/msgpack.zig @@ -1270,125 +1270,96 @@ pub const Payload = union(enum) { }; } - /// free all memory for this payload and sub payloads - /// the allocator is payload's allocator - /// This is an iterative implementation that avoids stack overflow from deep nesting - /// Optimization: Uses stack-allocated buffer for shallow structures to avoid heap allocation during free - pub fn free(self: Payload, allocator: Allocator) void { - // Use stack-allocated buffer for shallow structures (up to 256 items) - // This avoids heap allocation during memory cleanup for most common cases - const STACK_BUFFER_SIZE = 256; - var stack_buffer: [STACK_BUFFER_SIZE]Payload = undefined; - var stack_len: usize = 0; - - // Fallback to heap if we exceed stack buffer - var heap_stack: ?std.ArrayList(Payload) = null; - defer if (heap_stack) |*hs| { - if (current_zig.minor == 14) { - hs.deinit(); - } else { - hs.deinit(allocator); - } - }; + /// Intrusive worklist stored in an owned container's first Payload slot. + /// The displaced child is copied out before that slot is reused. + const FreeNode = struct { + next: ?*FreeNode, + container: union(enum) { + arr: []Payload, + map: PayloadHashMap, + }, - // Helper to push to stack (tries stack first, falls back to heap) - const pushPayload = struct { - fn push( - buffer: []Payload, - len: *usize, - heap: *?std.ArrayList(Payload), - alloc: Allocator, - payload: Payload, - ) void { - if (heap.*) |*h| { - // Already using heap - if (current_zig.minor == 14) { - h.append(payload) catch {}; - } else { - h.append(alloc, payload) catch {}; - } - } else if (len.* < buffer.len) { - // Stack buffer has space - buffer[len.*] = payload; - len.* += 1; - } else { - // Stack buffer full, migrate to heap - var new_heap = if (current_zig.minor == 14) - std.ArrayList(Payload).init(alloc) - else - std.ArrayList(Payload).empty; - - // Copy existing items from stack buffer to heap - for (buffer[0..len.*]) |item| { - if (current_zig.minor == 14) { - new_heap.append(item) catch return; - } else { - new_heap.append(alloc, item) catch return; + fn enqueue(payload: Payload, pending: *?*FreeNode, allocator: Allocator) void { + var current = payload; + while (true) { + switch (current) { + .str => |s| { + allocator.free(s.value()); + return; + }, + .bin => |b| { + allocator.free(b.value()); + return; + }, + .ext => |e| { + allocator.free(e.data); + return; + }, + .arr => |arr| { + if (arr.len == 0) { + allocator.free(arr); + return; } - } - // Add new item - if (current_zig.minor == 14) { - new_heap.append(payload) catch return; - } else { - new_heap.append(alloc, payload) catch return; - } - heap.* = new_heap; - len.* = 0; // Clear stack buffer - } - } - }.push; - - // Helper to pop from stack - const popPayload = struct { - fn pop( - buffer: []Payload, - len: *usize, - heap: *?std.ArrayList(Payload), - ) ?Payload { - if (heap.*) |*h| { - if (h.items.len > 0) { - return h.pop(); - } - } - if (len.* > 0) { - len.* -= 1; - return buffer[len.*]; + const child = arr[0]; + const node: *FreeNode = @ptrCast(&arr[0]); + node.* = .{ .next = pending.*, .container = .{ .arr = arr } }; + pending.* = node; + current = child; + }, + .map => |map| { + var table = map.map; + var it = table.iterator(); + const first = it.next() orelse { + table.deinit(); + return; + }; + const child = first.key_ptr.*; + const node: *FreeNode = @ptrCast(first.key_ptr); + node.* = .{ .next = pending.*, .container = .{ .map = table } }; + pending.* = node; + current = child; + }, + else => return, } - return null; } - }.pop; - - // Start with self - pushPayload(&stack_buffer, &stack_len, &heap_stack, allocator, self); - - while (popPayload(&stack_buffer, &stack_len, &heap_stack)) |payload_item| { - switch (payload_item) { - .str => |s| allocator.free(s.value()), - .bin => |b| allocator.free(b.value()), - .ext => |e| allocator.free(e.data), + } + }; + /// Free all owned memory without allocating or recursing. + /// The allocator must be the one used for this payload and its children. + pub fn free(self: Payload, allocator: Allocator) void { + comptime { + std.debug.assert(@sizeOf(FreeNode) <= @sizeOf(Payload)); + std.debug.assert(@alignOf(FreeNode) <= @alignOf(Payload)); + } + + var pending: ?*FreeNode = null; + FreeNode.enqueue(self, &pending, allocator); + while (pending) |node| { + // Copy the frame before freeing the container that stores it. + const frame = node.*; + pending = frame.next; + switch (frame.container) { .arr => |arr| { - defer allocator.free(arr); - // Push children to stack in reverse order - var i = arr.len; - while (i > 0) { - i -= 1; - pushPayload(&stack_buffer, &stack_len, &heap_stack, allocator, arr[i]); + // Element zero now holds the frame, not a Payload. + for (arr[1..]) |item| { + FreeNode.enqueue(item, &pending, allocator); } + allocator.free(arr); }, - .map => |map| { - var map_copy = map; - defer map_copy.deinit(); - // Push both keys and values to stack for recursive freeing - var it = map_copy.map.iterator(); + var table = map; + var it = table.iterator(); + // Only the first key was displaced. Its value still needs cleanup. + // Iteration uses occupancy metadata, never hashes the overwritten key. + const first = it.next().?; + FreeNode.enqueue(first.value_ptr.*, &pending, allocator); while (it.next()) |entry| { - pushPayload(&stack_buffer, &stack_len, &heap_stack, allocator, entry.key_ptr.*); - pushPayload(&stack_buffer, &stack_len, &heap_stack, allocator, entry.value_ptr.*); + FreeNode.enqueue(entry.key_ptr.*, &pending, allocator); + FreeNode.enqueue(entry.value_ptr.*, &pending, allocator); } + table.deinit(); }, - - else => {}, // nil, bool, int, uint, float, timestamp - no memory to free } } } diff --git a/src/test.zig b/src/test.zig index 179bc17..1a803d5 100644 --- a/src/test.zig +++ b/src/test.zig @@ -3667,21 +3667,99 @@ test "clonePayload map partial fail path frees partially cloned entries" { } test "iterative free: deeply nested payload" { - // Create a deeply nested structure in memory - var root = try Payload.arrPayload(1, allocator); + var failing = std.testing.FailingAllocator.init(allocator, .{}); + const alloc = failing.allocator(); + var root = Payload.nilToPayload(); + errdefer root.free(alloc); var current: *Payload = &root; - // Build 200-layer deep structure - var i: usize = 0; - while (i < 200) : (i += 1) { - const nested = try Payload.arrPayload(1, allocator); - try current.setArrElement(0, nested); - current = ¤t.arr[0]; + // Keep siblings alive at every level, alternating arrays and maps. + for (0..4096) |depth| { + if (depth % 2 == 0) { + current.* = try Payload.arrPayload(2, alloc); + current.arr[1] = try Payload.strToPayload("sibling", alloc); + current = ¤t.arr[0]; + } else { + current.* = Payload.mapPayload(alloc); + try current.mapPut("sibling", Payload.nilToPayload()); + current.map.getPtr(.{ .str = msgpack.Str.init("sibling") }).?.* = + try Payload.binToPayload("sibling", alloc); + try current.mapPut("child", Payload.nilToPayload()); + current = current.map.getPtr(.{ .str = msgpack.Str.init("child") }).?; + } + } + current.* = try Payload.extToPayload(1, "leaf", alloc); + + failing.fail_index = failing.alloc_index; + root.free(alloc); + root = Payload.nilToPayload(); + try std.testing.expectEqual(failing.allocated_bytes, failing.freed_bytes); +} + +test "iterative free: wide array under OOM" { + var failing = std.testing.FailingAllocator.init(allocator, .{}); + const alloc = failing.allocator(); + var root = try Payload.arrPayload(257, alloc); + errdefer root.free(alloc); + for (root.arr, 0..) |*item, i| { + item.* = switch (i % 3) { + 0 => try Payload.strToPayload("s", alloc), + 1 => try Payload.binToPayload("b", alloc), + else => try Payload.extToPayload(1, "e", alloc), + }; } - try current.setArrElement(0, Payload.intToPayload(42)); - // Free should not cause stack overflow - root.free(allocator); + failing.fail_index = failing.alloc_index; + root.free(alloc); + root = Payload.nilToPayload(); + try std.testing.expectEqual(failing.allocated_bytes, failing.freed_bytes); +} + +test "iterative free: wide map with container keys under OOM" { + var failing = std.testing.FailingAllocator.init(allocator, .{}); + const alloc = failing.allocator(); + var root = Payload.mapPayload(alloc); + errdefer root.free(alloc); + for (0..257) |i| { + var key_items = [_]Payload{ + .{ .uint = i }, + .{ .str = msgpack.Str.init("key") }, + }; + const key = Payload{ .arr = &key_items }; + try root.mapPutGeneric(key, Payload.nilToPayload()); + const value = root.map.getPtr(key).?; + value.* = try Payload.arrPayload(2, alloc); + value.arr[0] = try Payload.binToPayload("value", alloc); + value.arr[1] = try Payload.extToPayload(1, "extension", alloc); + } + + failing.fail_index = failing.alloc_index; + root.free(alloc); + root = Payload.nilToPayload(); + try std.testing.expectEqual(failing.allocated_bytes, failing.freed_bytes); +} + +test "iterative free: parser OOM cleans completed wide child" { + // The last string can fail after the wide child has been fully decoded. + var input: [4 + 257 * 2 + 2]u8 = undefined; + @memcpy(input[0..4], &[_]u8{ 0x92, 0xdc, 0x01, 0x01 }); + for (0..257) |i| { + input[4 + i * 2] = 0xa1; + input[5 + i * 2] = 'x'; + } + @memcpy(input[input.len - 2 ..], &[_]u8{ 0xa1, 'y' }); + + const Scenario = struct { + fn run(alloc: std.mem.Allocator, bytes: []u8) !void { + var output: [0]u8 = .{}; + var write_buffer = fixedBufferStream(&output); + var read_buffer = fixedBufferStream(bytes); + var p = pack.init(&write_buffer, &read_buffer); + const payload = try p.read(alloc); + defer payload.free(alloc); + } + }; + try std.testing.checkAllAllocationFailures(allocator, Scenario.run, .{&input}); } // ========== Large Data and Fuzz Tests ========== From e6b249212eec9891cd992ecc1b0b1a0e4c09885e Mon Sep 17 00:00:00 2001 From: jinzhongjia Date: Wed, 16 Sep 2026 15:23:32 +0800 Subject: [PATCH 4/4] fix: reject reserved MessagePack marker and add spec tests --- README.md | 4 + README_CN.md | 4 + src/msgpack.zig | 4 +- src/test.zig | 234 +++++++++++++++++++++++++++++++++++++++++++++--- 4 files changed, 233 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 8e1eae7..e7954df 100644 --- a/README.md +++ b/README.md @@ -395,6 +395,10 @@ std.debug.print("As float: {d}\n", .{ decoded_ts.timestamp.toFloat() }); ### Error Handling +Decoding rejects the reserved marker `0xc1` with `MsgPackError.TypeMarkerReading`, +including in array elements and map keys/values. Only `0xc0` encodes `nil`. +This restriction applies to type markers, not bytes inside data or length fields. + ```zig // Type conversion with error handling const int_payload = msgpack.Payload.intToPayload(-42); diff --git a/README_CN.md b/README_CN.md index 7deded4..5adf3b8 100644 --- a/README_CN.md +++ b/README_CN.md @@ -303,6 +303,10 @@ std.debug.print("浮点数形式: {d}\n", .{ decoded_ts.timestamp.toFloat() }); ### 错误处理 +解码时，保留标记 `0xc1` 会返回 `MsgPackError.TypeMarkerReading`，包括数组元素和 +map 的键、值位置。只有 `0xc0` 表示 `nil`。此限制仅针对类型标记，不影响数据内容 +或长度字段中的同值字节。 + ```zig // 类型转换与错误处理 const int_payload = msgpack.Payload.intToPayload(-42); diff --git a/src/msgpack.zig b/src/msgpack.zig index d00dc73..b3c66d4 100644 --- a/src/msgpack.zig +++ b/src/msgpack.zig @@ -2176,6 +2176,8 @@ pub fn PackWithLimits( fn readTypeMarkerU8(self: Self) !u8 { const val = try self.readByte(); + // The reserved byte is invalid only as a marker, not inside data. + if (val == 0xc1) return MsgPackError.TypeMarkerReading; return val; } @@ -2192,7 +2194,7 @@ pub fn PackWithLimits( 0x90...0x9f => .FIXARRAY, 0xa0...0xbf => .FIXSTR, 0xc0 => .NIL, - 0xc1 => .NIL, // Reserved byte, treat as NIL + 0xc1 => undefined, // Rejected by readTypeMarkerU8 before lookup. 0xc2 => .FALSE, 0xc3 => .TRUE, 0xc4 => .BIN8, diff --git a/src/test.zig b/src/test.zig index 1a803d5..0d6f747 100644 --- a/src/test.zig +++ b/src/test.zig @@ -51,26 +51,236 @@ test "PackerIO: truncated data error" { } } -test "PackerIO: invalid msgpack marker" { +test "MessagePack spec: reserved marker is rejected at every value position" { if (!has_new_io) return error.SkipZigTest; - // 0xc1 is a reserved/invalid marker byte in MessagePack - var buffer: [10]u8 = [_]u8{ 0xc1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; - var writer = std.Io.Writer.fixed(&buffer); - var reader = std.Io.Reader.fixed(&buffer); + // https://github.com/msgpack/msgpack/blob/master/spec.md#overview + // 0xc1 is "never used", not an alternative encoding of nil (0xc0). + const inputs = [_][]const u8{ + &.{0xc1}, + &.{ 0x91, 0xc1 }, // Array element. + &.{ 0x81, 0xc1, 0x01 }, // Map key. + &.{ 0x81, 0x01, 0xc1 }, // Map value. + &.{ 0x92, 0xa1, 'x', 0xc1 }, // Previously allocated array element. + &.{ 0x82, 0xa1, 'k', 0xa1, 'v', 0xc1, 0x00 }, // Completed map entry. + &.{ 0x81, 0xa1, 'k', 0xc1 }, // Allocated key waiting for its value. + &.{ 0x92, 0x91, 0xa1, 'x', 0x81, 0xa1, 'k', 0x91, 0xc1 }, // Mixed nesting. + }; + for (inputs) |input| { + var output: [0]u8 = .{}; + var writer = std.Io.Writer.fixed(&output); + var reader = std.Io.Reader.fixed(input); + var packer = msgpack.PackerIO.init(&reader, &writer); + const result = packer.read(allocator); + // Free unexpected successes too, so pre-fix failures do not leak. + if (result) |payload| payload.free(allocator) else |_| {} + try std.testing.expectError(msgpack.MsgPackError.TypeMarkerReading, result); + } +} + +test "MessagePack spec: scalar wire vectors" { + if (!has_new_io) return error.SkipZigTest; + + // Fixed wire vectors, independent of this library's encoder. + // https://github.com/msgpack/msgpack/blob/master/spec.md#formats + const Vector = struct { bytes: []const u8, value: Payload }; + const vectors = [_]Vector{ + .{ .bytes = "\xc0", .value = .{ .nil = {} } }, + .{ .bytes = "\xc2", .value = .{ .bool = false } }, + .{ .bytes = "\xc3", .value = .{ .bool = true } }, + .{ .bytes = "\x00", .value = .{ .uint = 0 } }, + .{ .bytes = "\x7f", .value = .{ .uint = 127 } }, + .{ .bytes = "\xcc\x80", .value = .{ .uint = 128 } }, + .{ .bytes = "\xcc\xc1", .value = .{ .uint = 193 } }, + .{ .bytes = "\xcd\x01\x00", .value = .{ .uint = 256 } }, + .{ .bytes = "\xce\x00\x01\x00\x00", .value = .{ .uint = 65536 } }, + .{ .bytes = "\xcf\xff\xff\xff\xff\xff\xff\xff\xff", .value = .{ .uint = std.math.maxInt(u64) } }, + .{ .bytes = "\xff", .value = .{ .int = -1 } }, + .{ .bytes = "\xe0", .value = .{ .int = -32 } }, + .{ .bytes = "\xd0\xdf", .value = .{ .int = -33 } }, + .{ .bytes = "\xd1\xff\x7f", .value = .{ .int = -129 } }, + .{ .bytes = "\xd2\xff\xff\x7f\xff", .value = .{ .int = -32769 } }, + .{ .bytes = "\xd3\x80\x00\x00\x00\x00\x00\x00\x00", .value = .{ .int = std.math.minInt(i64) } }, + .{ .bytes = "\xca\xc1\x20\x00\x00", .value = .{ .float = -10.0 } }, + .{ .bytes = "\xcb\x3f\xf0\x00\x00\x00\x00\x00\x01", .value = .{ .float = 1.0000000000000002 } }, + }; + for (vectors) |vector| { + var output: [16]u8 = undefined; + var reader = std.Io.Reader.fixed(vector.bytes); + var writer = std.Io.Writer.fixed(&output); + var packer = msgpack.PackerIO.init(&reader, &writer); + const decoded = try packer.read(allocator); + defer decoded.free(allocator); + try std.testing.expectEqualDeep(vector.value, decoded); + // Check encoding against the spec bytes too, not just a round trip. + try packer.write(vector.value); + try std.testing.expectEqualSlices(u8, vector.bytes, writer.buffered()); + } +} + +test "MessagePack spec: non-minimal integer formats remain valid" { + if (!has_new_io) return error.SkipZigTest; + + // Minimal encoding is a serializer SHOULD, not a decoder restriction. + const inputs = [_][]const u8{ + "\xcc\x01", + "\xcd\x00\x01", + "\xce\x00\x00\x00\x01", + "\xcf\x00\x00\x00\x00\x00\x00\x00\x01", + "\xd0\x01", + "\xd1\x00\x01", + "\xd2\x00\x00\x00\x01", + "\xd3\x00\x00\x00\x00\x00\x00\x00\x01", + }; + for (inputs) |input| { + var output: [0]u8 = .{}; + var reader = std.Io.Reader.fixed(input); + var writer = std.Io.Writer.fixed(&output); + var packer = msgpack.PackerIO.init(&reader, &writer); + const decoded = try packer.read(allocator); + defer decoded.free(allocator); + try std.testing.expectEqual(@as(u64, 1), try decoded.getUint()); + } +} + +test "MessagePack spec: reserved byte is allowed in raw data and extension type" { + if (!has_new_io) return error.SkipZigTest; + + // String invalid-byte handling is implementation-defined; this library + // preserves the original bytes. Binary/extension bodies are arbitrary bytes. + // https://github.com/msgpack/msgpack/blob/master/spec.md#limitation + const Vector = struct { bytes: []const u8, kind: enum { str, bin, ext } }; + const vectors = [_]Vector{ + .{ .bytes = "\xa1\xc1", .kind = .str }, + .{ .bytes = "\xd9\x01\xc1", .kind = .str }, + .{ .bytes = "\xda\x00\x01\xc1", .kind = .str }, + .{ .bytes = "\xdb\x00\x00\x00\x01\xc1", .kind = .str }, + .{ .bytes = "\xc4\x01\xc1", .kind = .bin }, + .{ .bytes = "\xc5\x00\x01\xc1", .kind = .bin }, + .{ .bytes = "\xc6\x00\x00\x00\x01\xc1", .kind = .bin }, + .{ .bytes = "\xd4\xc1\xc1", .kind = .ext }, + .{ .bytes = "\xc7\x01\xc1\xc1", .kind = .ext }, + .{ .bytes = "\xc8\x00\x01\xc1\xc1", .kind = .ext }, + .{ .bytes = "\xc9\x00\x00\x00\x01\xc1\xc1", .kind = .ext }, + }; + for (vectors) |vector| { + var output: [0]u8 = .{}; + var reader = std.Io.Reader.fixed(vector.bytes); + var writer = std.Io.Writer.fixed(&output); + var packer = msgpack.PackerIO.init(&reader, &writer); + const decoded = try packer.read(allocator); + defer decoded.free(allocator); + switch (vector.kind) { + .str => try std.testing.expectEqualSlices(u8, "\xc1", try decoded.asStr()), + .bin => try std.testing.expectEqualSlices(u8, "\xc1", try decoded.asBin()), + .ext => { + try expect(decoded == .ext); + try std.testing.expectEqual(@as(i8, -63), decoded.ext.type); + try std.testing.expectEqualSlices(u8, "\xc1", decoded.ext.data); + }, + } + } + // 0xc1 may also occur in a length field, rather than in a body. + var input: [195]u8 = undefined; + @memcpy(input[0..2], "\xc4\xc1"); + @memset(input[2..], 0x42); + var output: [195]u8 = undefined; + var reader = std.Io.Reader.fixed(&input); + var writer = std.Io.Writer.fixed(&output); var packer = msgpack.PackerIO.init(&reader, &writer); + const decoded = try packer.read(allocator); + defer decoded.free(allocator); + try std.testing.expectEqualSlices(u8, input[2..], try decoded.asBin()); + try packer.write(decoded); + try std.testing.expectEqualSlices(u8, &input, writer.buffered()); +} - // Should handle invalid marker gracefully (no crash) - const result = packer.read(allocator); - if (result) |payload| { - payload.free(allocator); - // If it succeeds, that's fine (marker might be treated as NIL or other) - } else |_| { - // Expected - invalid marker should cause error +test "MessagePack spec: array and map wire formats accept nil" { + if (!has_new_io) return error.SkipZigTest; + + const arrays = [_][]const u8{ + "\x91\xc0", + "\xdc\x00\x01\xc0", + "\xdd\x00\x00\x00\x01\xc0", + }; + for (arrays) |input| { + var output: [0]u8 = .{}; + var reader = std.Io.Reader.fixed(input); + var writer = std.Io.Writer.fixed(&output); + var packer = msgpack.PackerIO.init(&reader, &writer); + const decoded = try packer.read(allocator); + defer decoded.free(allocator); + try std.testing.expectEqual(@as(usize, 1), try decoded.getArrLen()); + try expect((try decoded.getArrElement(0)) == .nil); + } + const maps = [_][]const u8{ + "\x81\xc0\xc0", + "\xde\x00\x01\xc0\xc0", + "\xdf\x00\x00\x00\x01\xc0\xc0", + }; + for (maps) |input| { + var output: [0]u8 = .{}; + var reader = std.Io.Reader.fixed(input); + var writer = std.Io.Writer.fixed(&output); + var packer = msgpack.PackerIO.init(&reader, &writer); + const decoded = try packer.read(allocator); + defer decoded.free(allocator); + try expect(decoded == .map); + try std.testing.expectEqual(@as(usize, 1), decoded.map.count()); + var entries = decoded.map.map.iterator(); + const entry = entries.next().?; + try expect(entry.key_ptr.* == .nil); + try expect(entry.value_ptr.* == .nil); + } +} + +test "MessagePack spec: timestamp wire vectors" { + if (!has_new_io) return error.SkipZigTest; + + // https://github.com/msgpack/msgpack/blob/master/spec.md#timestamp-extension-type + const Vector = struct { bytes: []const u8, seconds: i64, nanoseconds: u32 }; + const vectors = [_]Vector{ + .{ .bytes = "\xd6\xff\xff\xff\xff\xff", .seconds = 4294967295, .nanoseconds = 0 }, + .{ .bytes = "\xd7\xff\xee\x6b\x27\xff\xff\xff\xff\xff", .seconds = 17179869183, .nanoseconds = 999999999 }, + .{ .bytes = "\xc7\x0c\xff\x3b\x9a\xc9\xff\xff\xff\xff\xff\xff\xff\xff\xff", .seconds = -1, .nanoseconds = 999999999 }, + }; + for (vectors) |vector| { + var output: [15]u8 = undefined; + var reader = std.Io.Reader.fixed(vector.bytes); + var writer = std.Io.Writer.fixed(&output); + var packer = msgpack.PackerIO.init(&reader, &writer); + const decoded = try packer.read(allocator); + defer decoded.free(allocator); + const expected = Payload.timestampToPayload(vector.seconds, vector.nanoseconds); + try std.testing.expectEqualDeep(expected, decoded); + try packer.write(expected); + try std.testing.expectEqualSlices(u8, vector.bytes, writer.buffered()); } } +test "MessagePack spec: reserved marker cleanup under allocation failure" { + // Completed children, a pending map key, and an active nested container must + // all be reclaimed whether allocation fails first or 0xc1 is reached. + var input = [_]u8{ 0x92, 0x91, 0xa1, 'x', 0x82, 0xa1, 'a', 0xa1, 'b', 0xa1, 'k', 0x91, 0xc1 }; + const Scenario = struct { + fn run(alloc: std.mem.Allocator, bytes: []u8) !void { + var output: [0]u8 = .{}; + var write_buffer = fixedBufferStream(&output); + var read_buffer = fixedBufferStream(bytes); + var p = pack.init(&write_buffer, &read_buffer); + const payload = p.read(alloc) catch |err| { + if (err == error.OutOfMemory) return err; + try std.testing.expectEqual(error.TypeMarkerReading, err); + return; + }; + defer payload.free(alloc); + return error.TestUnexpectedResult; + } + }; + try std.testing.checkAllAllocationFailures(allocator, Scenario.run, .{&input}); +} + test "PackerIO: corrupted length field" { if (!has_new_io) return error.SkipZigTest;