From e4fa102aa71c35efcbd08e5df801077d8ecd9b61 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Thu, 1 Oct 2026 19:33:54 -0700 Subject: [PATCH] Write sint32 values as varints of at most 5 bytes getSerializedSize() counts a sint32 as its zigzag value encoded as an unsigned 32-bit varint, at most 5 bytes, as protobuf does. But writeRawSignedVarInt() went through writeRawVarInt(), the int32 writer, which writes a negative int as a 10-byte varint. The zigzag value has its top bit set for every v >= 2^30 and every v < -2^30, so for each such value writeTo() wrote 5 bytes more than it had reserved, with no parsing involved: toByteArray() threw ArrayIndexOutOfBoundsException, and a pooled heap target got the extra bytes written into the neighbouring buffer of its chunk. Write the zigzag value with a new writeRawVarUInt() (byte[] and NIO sinks), which never writes more than 5 bytes. The output now matches protobuf-java for singular, repeated and packed sint32 fields. --- .../lightproto/generator/LightProtoCodec.java | 31 +++++++++++++++++-- .../lightproto/tests/NumbersTest.java | 9 ++++++ .../lightproto/tests/RepeatedNumbersTest.java | 18 +++++++++++ 3 files changed, 56 insertions(+), 2 deletions(-) diff --git a/code-generator/src/main/resources/io/streamnative/lightproto/generator/LightProtoCodec.java b/code-generator/src/main/resources/io/streamnative/lightproto/generator/LightProtoCodec.java index 2a081d5..619030a 100644 --- a/code-generator/src/main/resources/io/streamnative/lightproto/generator/LightProtoCodec.java +++ b/code-generator/src/main/resources/io/streamnative/lightproto/generator/LightProtoCodec.java @@ -425,8 +425,24 @@ static int writeRawVarInt64(byte[] a, int i, long value) { } } + /** + * Writes n as an unsigned 32-bit varint, at most 5 bytes, as protobuf encodes + * zigzag-encoded sint32 values: what computeVarUIntSize() counts. + * writeRawVarInt() is the int32 encoding, which writes a negative n as 10 bytes. + */ + static int writeRawVarUInt(byte[] a, int i, int n) { + while (true) { + if ((n & ~0x7F) == 0) { + a[i++] = (byte) n; + return i; + } + a[i++] = (byte) ((n & 0x7F) | 0x80); + n >>>= 7; + } + } + static int writeRawSignedVarInt(byte[] a, int i, int n) { - return writeRawVarInt(a, i, encodeZigZag32(n)); + return writeRawVarUInt(a, i, encodeZigZag32(n)); } static int writeRawSignedVarInt64(byte[] a, int i, long n) { @@ -513,8 +529,19 @@ static int writeRawVarInt64(java.nio.ByteBuffer nb, int i, long value) { } } + static int writeRawVarUInt(java.nio.ByteBuffer nb, int i, int n) { + while (true) { + if ((n & ~0x7F) == 0) { + nb.put(i++, (byte) n); + return i; + } + nb.put(i++, (byte) ((n & 0x7F) | 0x80)); + n >>>= 7; + } + } + static int writeRawSignedVarInt(java.nio.ByteBuffer nb, int i, int n) { - return writeRawVarInt(nb, i, encodeZigZag32(n)); + return writeRawVarUInt(nb, i, encodeZigZag32(n)); } static int writeRawSignedVarInt64(java.nio.ByteBuffer nb, int i, long n) { diff --git a/tests/src/test/java/io/streamnative/lightproto/tests/NumbersTest.java b/tests/src/test/java/io/streamnative/lightproto/tests/NumbersTest.java index b2dfb4d..e130f4d 100644 --- a/tests/src/test/java/io/streamnative/lightproto/tests/NumbersTest.java +++ b/tests/src/test/java/io/streamnative/lightproto/tests/NumbersTest.java @@ -20,6 +20,8 @@ import io.netty.buffer.Unpooled; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import java.util.Arrays; @@ -204,6 +206,13 @@ public void testNumberFields() throws Exception { verify(lpn, pbn.build()); } + @ParameterizedTest + @ValueSource(ints = {1 << 30, -(1 << 30) - 1, Integer.MAX_VALUE, Integer.MIN_VALUE}) + public void testSint32WithZigZagTopBitSet(int value) throws Exception { + // The zigzag encoding of these values is >= 2^31 unsigned: still a 5-byte varint + verify(new Numbers().setXSint32(value), NumbersOuterClass.Numbers.newBuilder().setXSint32(value).build()); + } + @Test public void testClearResetsAllFieldsToDefaults() throws Exception { Numbers lpn = new Numbers(); diff --git a/tests/src/test/java/io/streamnative/lightproto/tests/RepeatedNumbersTest.java b/tests/src/test/java/io/streamnative/lightproto/tests/RepeatedNumbersTest.java index 791d08e..08d8ff0 100644 --- a/tests/src/test/java/io/streamnative/lightproto/tests/RepeatedNumbersTest.java +++ b/tests/src/test/java/io/streamnative/lightproto/tests/RepeatedNumbersTest.java @@ -485,4 +485,22 @@ public void testNumberFieldsPacked() throws Exception { assertEquals(-12L, parsed.getXSint64At(0)); assertEquals(-13L, parsed.getXSint64At(1)); } + + @Test + public void testSint32WithZigZagTopBitSet() throws Exception { + // The zigzag encoding of these values is >= 2^31 unsigned: still a 5-byte varint + int[] values = {1 << 30, -(1 << 30) - 1, Integer.MAX_VALUE, Integer.MIN_VALUE}; + Repeated lp = new Repeated(); + RepeatedPacked lpPacked = new RepeatedPacked(); + RepeatedNumbers.Repeated.Builder pb = RepeatedNumbers.Repeated.newBuilder(); + RepeatedNumbers.RepeatedPacked.Builder pbPacked = RepeatedNumbers.RepeatedPacked.newBuilder(); + for (int value : values) { + lp.addXSint32(value); + lpPacked.addXSint32(value); + pb.addXSint32(value); + pbPacked.addXSint32(value); + } + assertArrayEquals(pb.build().toByteArray(), lp.toByteArray()); + assertArrayEquals(pbPacked.build().toByteArray(), lpPacked.toByteArray()); + } }