From e4fa102aa71c35efcbd08e5df801077d8ecd9b61 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Thu, 1 Oct 2026 19:33:54 -0700 Subject: [PATCH 1/2] Write sint32 values as varints of at most 5 bytes getSerializedSize() counts a sint32 as its zigzag value encoded as an unsigned 32-bit varint, at most 5 bytes, as protobuf does. But writeRawSignedVarInt() went through writeRawVarInt(), the int32 writer, which writes a negative int as a 10-byte varint. The zigzag value has its top bit set for every v >= 2^30 and every v < -2^30, so for each such value writeTo() wrote 5 bytes more than it had reserved, with no parsing involved: toByteArray() threw ArrayIndexOutOfBoundsException, and a pooled heap target got the extra bytes written into the neighbouring buffer of its chunk. Write the zigzag value with a new writeRawVarUInt() (byte[] and NIO sinks), which never writes more than 5 bytes. The output now matches protobuf-java for singular, repeated and packed sint32 fields. --- .../lightproto/generator/LightProtoCodec.java | 31 +++++++++++++++++-- .../lightproto/tests/NumbersTest.java | 9 ++++++ .../lightproto/tests/RepeatedNumbersTest.java | 18 +++++++++++ 3 files changed, 56 insertions(+), 2 deletions(-) diff --git a/code-generator/src/main/resources/io/streamnative/lightproto/generator/LightProtoCodec.java b/code-generator/src/main/resources/io/streamnative/lightproto/generator/LightProtoCodec.java index 2a081d5..619030a 100644 --- a/code-generator/src/main/resources/io/streamnative/lightproto/generator/LightProtoCodec.java +++ b/code-generator/src/main/resources/io/streamnative/lightproto/generator/LightProtoCodec.java @@ -425,8 +425,24 @@ static int writeRawVarInt64(byte[] a, int i, long value) { } } + /** + * Writes n as an unsigned 32-bit varint, at most 5 bytes, as protobuf encodes + * zigzag-encoded sint32 values: what computeVarUIntSize() counts. + * writeRawVarInt() is the int32 encoding, which writes a negative n as 10 bytes. + */ + static int writeRawVarUInt(byte[] a, int i, int n) { + while (true) { + if ((n & ~0x7F) == 0) { + a[i++] = (byte) n; + return i; + } + a[i++] = (byte) ((n & 0x7F) | 0x80); + n >>>= 7; + } + } + static int writeRawSignedVarInt(byte[] a, int i, int n) { - return writeRawVarInt(a, i, encodeZigZag32(n)); + return writeRawVarUInt(a, i, encodeZigZag32(n)); } static int writeRawSignedVarInt64(byte[] a, int i, long n) { @@ -513,8 +529,19 @@ static int writeRawVarInt64(java.nio.ByteBuffer nb, int i, long value) { } } + static int writeRawVarUInt(java.nio.ByteBuffer nb, int i, int n) { + while (true) { + if ((n & ~0x7F) == 0) { + nb.put(i++, (byte) n); + return i; + } + nb.put(i++, (byte) ((n & 0x7F) | 0x80)); + n >>>= 7; + } + } + static int writeRawSignedVarInt(java.nio.ByteBuffer nb, int i, int n) { - return writeRawVarInt(nb, i, encodeZigZag32(n)); + return writeRawVarUInt(nb, i, encodeZigZag32(n)); } static int writeRawSignedVarInt64(java.nio.ByteBuffer nb, int i, long n) { diff --git a/tests/src/test/java/io/streamnative/lightproto/tests/NumbersTest.java b/tests/src/test/java/io/streamnative/lightproto/tests/NumbersTest.java index b2dfb4d..e130f4d 100644 --- a/tests/src/test/java/io/streamnative/lightproto/tests/NumbersTest.java +++ b/tests/src/test/java/io/streamnative/lightproto/tests/NumbersTest.java @@ -20,6 +20,8 @@ import io.netty.buffer.Unpooled; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import java.util.Arrays; @@ -204,6 +206,13 @@ public void testNumberFields() throws Exception { verify(lpn, pbn.build()); } + @ParameterizedTest + @ValueSource(ints = {1 << 30, -(1 << 30) - 1, Integer.MAX_VALUE, Integer.MIN_VALUE}) + public void testSint32WithZigZagTopBitSet(int value) throws Exception { + // The zigzag encoding of these values is >= 2^31 unsigned: still a 5-byte varint + verify(new Numbers().setXSint32(value), NumbersOuterClass.Numbers.newBuilder().setXSint32(value).build()); + } + @Test public void testClearResetsAllFieldsToDefaults() throws Exception { Numbers lpn = new Numbers(); diff --git a/tests/src/test/java/io/streamnative/lightproto/tests/RepeatedNumbersTest.java b/tests/src/test/java/io/streamnative/lightproto/tests/RepeatedNumbersTest.java index 791d08e..08d8ff0 100644 --- a/tests/src/test/java/io/streamnative/lightproto/tests/RepeatedNumbersTest.java +++ b/tests/src/test/java/io/streamnative/lightproto/tests/RepeatedNumbersTest.java @@ -485,4 +485,22 @@ public void testNumberFieldsPacked() throws Exception { assertEquals(-12L, parsed.getXSint64At(0)); assertEquals(-13L, parsed.getXSint64At(1)); } + + @Test + public void testSint32WithZigZagTopBitSet() throws Exception { + // The zigzag encoding of these values is >= 2^31 unsigned: still a 5-byte varint + int[] values = {1 << 30, -(1 << 30) - 1, Integer.MAX_VALUE, Integer.MIN_VALUE}; + Repeated lp = new Repeated(); + RepeatedPacked lpPacked = new RepeatedPacked(); + RepeatedNumbers.Repeated.Builder pb = RepeatedNumbers.Repeated.newBuilder(); + RepeatedNumbers.RepeatedPacked.Builder pbPacked = RepeatedNumbers.RepeatedPacked.newBuilder(); + for (int value : values) { + lp.addXSint32(value); + lpPacked.addXSint32(value); + pb.addXSint32(value); + pbPacked.addXSint32(value); + } + assertArrayEquals(pb.build().toByteArray(), lp.toByteArray()); + assertArrayEquals(pbPacked.build().toByteArray(), lpPacked.toByteArray()); + } } From a484351364f506b87d905990fb02fc0580a638d2 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Thu, 1 Oct 2026 19:36:08 -0700 Subject: [PATCH 2/2] Write uint32 values as varints of at most 5 bytes A uint32 >= 2^31 was sized and written like an int32, as the 10-byte sign extension of a negative int, while protobuf writes a uint32 as an unsigned 32-bit varint of at most 5 bytes. Every reader, LightProto's included, decodes both to the same value, so the only effects were 5 extra bytes per such value, and that protobuf-java's encoding was re-serialized to a different size than it was parsed from. Size uint32 with computeVarUIntSize() and write it with writeRawVarUInt(), so the output matches protobuf-java for singular, repeated, packed and map uint32 fields. --- .../generator/LightProtoNumberField.java | 6 ++++-- .../lightproto/generator/LightProtoCodec.java | 2 +- .../lightproto/tests/NumbersTest.java | 7 +++++++ .../lightproto/tests/RepeatedNumbersTest.java | 18 ++++++++++++++++++ 4 files changed, 30 insertions(+), 3 deletions(-) diff --git a/code-generator/src/main/java/io/streamnative/lightproto/generator/LightProtoNumberField.java b/code-generator/src/main/java/io/streamnative/lightproto/generator/LightProtoNumberField.java index c297299..31ad40c 100644 --- a/code-generator/src/main/java/io/streamnative/lightproto/generator/LightProtoNumberField.java +++ b/code-generator/src/main/java/io/streamnative/lightproto/generator/LightProtoNumberField.java @@ -55,8 +55,10 @@ static void serializeNumber(PrintWriter w, ProtoFieldDescriptor field, String na } else if (field.getProtoType().equals("bool")) { writer = "writeRawByte"; value = name + " ? 1 : 0"; - } else if (field.getProtoType().equals("int32") || field.getProtoType().equals("uint32")) { + } else if (field.getProtoType().equals("int32")) { writer = "writeRawVarInt"; + } else if (field.getProtoType().equals("uint32")) { + writer = "writeRawVarUInt"; } else if (field.getProtoType().equals("sint32")) { writer = "writeRawSignedVarInt"; } else if (field.getProtoType().equals("sint64")) { @@ -140,7 +142,7 @@ static String serializedSizeOfNumber(ProtoFieldDescriptor field, String name) { } else if (field.getProtoType().equals("int32")) { return String.format("LightProtoCodec.computeVarIntSize(%s)", name); } else if (field.getProtoType().equals("uint32")) { - return String.format("LightProtoCodec.computeVarIntSize(%s)", name); + return String.format("LightProtoCodec.computeVarUIntSize(%s)", name); } else if (field.getProtoType().equals("int64")) { return String.format("LightProtoCodec.computeVarInt64Size(%s)", name); } else if (field.getProtoType().equals("uint64")) { diff --git a/code-generator/src/main/resources/io/streamnative/lightproto/generator/LightProtoCodec.java b/code-generator/src/main/resources/io/streamnative/lightproto/generator/LightProtoCodec.java index 619030a..c4ddb0e 100644 --- a/code-generator/src/main/resources/io/streamnative/lightproto/generator/LightProtoCodec.java +++ b/code-generator/src/main/resources/io/streamnative/lightproto/generator/LightProtoCodec.java @@ -427,7 +427,7 @@ static int writeRawVarInt64(byte[] a, int i, long value) { /** * Writes n as an unsigned 32-bit varint, at most 5 bytes, as protobuf encodes - * zigzag-encoded sint32 values: what computeVarUIntSize() counts. + * uint32 and zigzag-encoded sint32 values: what computeVarUIntSize() counts. * writeRawVarInt() is the int32 encoding, which writes a negative n as 10 bytes. */ static int writeRawVarUInt(byte[] a, int i, int n) { diff --git a/tests/src/test/java/io/streamnative/lightproto/tests/NumbersTest.java b/tests/src/test/java/io/streamnative/lightproto/tests/NumbersTest.java index e130f4d..c2f6522 100644 --- a/tests/src/test/java/io/streamnative/lightproto/tests/NumbersTest.java +++ b/tests/src/test/java/io/streamnative/lightproto/tests/NumbersTest.java @@ -206,6 +206,13 @@ public void testNumberFields() throws Exception { verify(lpn, pbn.build()); } + @ParameterizedTest + @ValueSource(ints = {Integer.MIN_VALUE, -1}) + public void testUint32AboveIntMax(int value) throws Exception { + // 2^31 and 2^32 - 1: protobuf writes a uint32 as an unsigned varint, 5 bytes + verify(new Numbers().setXUint32(value), NumbersOuterClass.Numbers.newBuilder().setXUint32(value).build()); + } + @ParameterizedTest @ValueSource(ints = {1 << 30, -(1 << 30) - 1, Integer.MAX_VALUE, Integer.MIN_VALUE}) public void testSint32WithZigZagTopBitSet(int value) throws Exception { diff --git a/tests/src/test/java/io/streamnative/lightproto/tests/RepeatedNumbersTest.java b/tests/src/test/java/io/streamnative/lightproto/tests/RepeatedNumbersTest.java index 08d8ff0..1495be6 100644 --- a/tests/src/test/java/io/streamnative/lightproto/tests/RepeatedNumbersTest.java +++ b/tests/src/test/java/io/streamnative/lightproto/tests/RepeatedNumbersTest.java @@ -503,4 +503,22 @@ public void testSint32WithZigZagTopBitSet() throws Exception { assertArrayEquals(pb.build().toByteArray(), lp.toByteArray()); assertArrayEquals(pbPacked.build().toByteArray(), lpPacked.toByteArray()); } + + @Test + public void testUint32AboveIntMax() throws Exception { + // 2^31 and 2^32 - 1: protobuf writes a uint32 as an unsigned varint, 5 bytes + int[] values = {Integer.MIN_VALUE, -1}; + Repeated lp = new Repeated(); + RepeatedPacked lpPacked = new RepeatedPacked(); + RepeatedNumbers.Repeated.Builder pb = RepeatedNumbers.Repeated.newBuilder(); + RepeatedNumbers.RepeatedPacked.Builder pbPacked = RepeatedNumbers.RepeatedPacked.newBuilder(); + for (int value : values) { + lp.addXUint32(value); + lpPacked.addXUint32(value); + pb.addXUint32(value); + pbPacked.addXUint32(value); + } + assertArrayEquals(pb.build().toByteArray(), lp.toByteArray()); + assertArrayEquals(pbPacked.build().toByteArray(), lpPacked.toByteArray()); + } }