From 0449562ec02c6dc69281966b45f9c598abb74826 Mon Sep 17 00:00:00 2001 From: ayush00git Date: Thu, 30 Jul 2026 12:43:20 +0530 Subject: [PATCH 01/20] feat(go): add row format null bitmap and alignment utils --- ci/tasks/go.py | 1 + go/fory/row/bitmap.go | 55 ++++++++++++++++++++++++ go/fory/row/bitmap_test.go | 87 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 143 insertions(+) create mode 100644 go/fory/row/bitmap.go create mode 100644 go/fory/row/bitmap_test.go diff --git a/ci/tasks/go.py b/ci/tasks/go.py index 3df17f81f2..109013cf42 100644 --- a/ci/tasks/go.py +++ b/ci/tasks/go.py @@ -24,4 +24,5 @@ def run(): logging.info("Executing fory go tests") common.cd_project_subdir("go/fory") common.exec_cmd("go test -v") + common.exec_cmd("go test -v ./row/...") logging.info("Executing fory go tests succeeds") diff --git a/go/fory/row/bitmap.go b/go/fory/row/bitmap.go new file mode 100644 index 0000000000..4f406283a8 --- /dev/null +++ b/go/fory/row/bitmap.go @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Package row implements the Fory standard row format defined in +// docs/specification/row_format_spec.md: a random-access binary format +// where each record is laid out as a null bitmap, fixed 8-byte field +// slots, and an 8-byte-aligned variable data region. +package row + +// The null bitmap tracks one bit per field or array element. Unlike the +// Arrow validity bitmap, a set bit means the value is NULL. Bits are +// LSB-first: bit 0 of byte 0 corresponds to index 0. + +// bitmapWidthInBytes returns the null bitmap size for n fields or +// elements, rounded up to a whole 8-byte word per the spec: +// ((n + 63) / 64) * 8. +func bitmapWidthInBytes(n int) int { + return ((n + 63) / 64) * 8 +} + +// setBit marks index i as null. +func setBit(bitmap []byte, i int) { + bitmap[i>>3] |= 1 << (uint(i) & 7) +} + +// clearBit marks index i as not null. +func clearBit(bitmap []byte, i int) { + bitmap[i>>3] &^= 1 << (uint(i) & 7) +} + +// getBit reports whether index i is null. +func getBit(bitmap []byte, i int) bool { + return bitmap[i>>3]&(1<<(uint(i)&7)) != 0 +} + +// roundToWord rounds n up to the nearest multiple of 8. The spec requires +// every variable-length value and data region to be zero-padded to an +// 8-byte boundary. +func roundToWord(n int) int { + return (n + 7) &^ 7 +} diff --git a/go/fory/row/bitmap_test.go b/go/fory/row/bitmap_test.go new file mode 100644 index 0000000000..37130b6870 --- /dev/null +++ b/go/fory/row/bitmap_test.go @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package row + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestBitmapWidthInBytes(t *testing.T) { + cases := []struct{ n, want int }{ + {0, 0}, + {1, 8}, + {10, 8}, + {63, 8}, + {64, 8}, + {65, 16}, + {128, 16}, + {129, 24}, + } + for _, c := range cases { + require.Equal(t, c.want, bitmapWidthInBytes(c.n), "n=%d", c.n) + } +} + +func TestBitOperations(t *testing.T) { + bitmap := make([]byte, 16) + + // Bit 0 of byte 0 is index 0 (LSB-first). + setBit(bitmap, 0) + require.Equal(t, byte(0b1), bitmap[0]) + require.True(t, getBit(bitmap, 0)) + + setBit(bitmap, 2) + require.Equal(t, byte(0b101), bitmap[0]) + + // Index 7 stays in byte 0; index 8 moves to byte 1. + setBit(bitmap, 7) + require.Equal(t, byte(0b10000101), bitmap[0]) + setBit(bitmap, 8) + require.Equal(t, byte(0b1), bitmap[1]) + + // Word boundary: index 63 is the last bit of the first word, + // index 64 the first bit of the second. + setBit(bitmap, 63) + require.Equal(t, byte(0b10000000), bitmap[7]) + setBit(bitmap, 64) + require.Equal(t, byte(0b1), bitmap[8]) + + // Clearing affects only the targeted bit. + clearBit(bitmap, 2) + require.False(t, getBit(bitmap, 2)) + require.True(t, getBit(bitmap, 0)) + require.True(t, getBit(bitmap, 7)) + + require.False(t, getBit(bitmap, 1)) +} + +func TestRoundToWord(t *testing.T) { + cases := []struct{ n, want int }{ + {0, 0}, + {1, 8}, + {7, 8}, + {8, 8}, + {9, 16}, + {16, 16}, + } + for _, c := range cases { + require.Equal(t, c.want, roundToWord(c.n), "n=%d", c.n) + } +} From efdb6e65eeeb08f3ec6794f076d5b9e145759c1c Mon Sep 17 00:00:00 2001 From: ayush00git Date: Thu, 30 Jul 2026 12:50:00 +0530 Subject: [PATCH 02/20] fix: refactor in-line comments --- go/fory/row/bitmap.go | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/go/fory/row/bitmap.go b/go/fory/row/bitmap.go index 4f406283a8..4e77b0cc9b 100644 --- a/go/fory/row/bitmap.go +++ b/go/fory/row/bitmap.go @@ -15,19 +15,13 @@ // specific language governing permissions and limitations // under the License. -// Package row implements the Fory standard row format defined in -// docs/specification/row_format_spec.md: a random-access binary format -// where each record is laid out as a null bitmap, fixed 8-byte field -// slots, and an 8-byte-aligned variable data region. +// Package row implements the Fory standard row format. package row -// The null bitmap tracks one bit per field or array element. Unlike the -// Arrow validity bitmap, a set bit means the value is NULL. Bits are -// LSB-first: bit 0 of byte 0 corresponds to index 0. +// The null bitmap tracks one bit per field or array element. // bitmapWidthInBytes returns the null bitmap size for n fields or -// elements, rounded up to a whole 8-byte word per the spec: -// ((n + 63) / 64) * 8. +// elements, rounded up to a whole 8-byte word. func bitmapWidthInBytes(n int) int { return ((n + 63) / 64) * 8 } @@ -47,7 +41,7 @@ func getBit(bitmap []byte, i int) bool { return bitmap[i>>3]&(1<<(uint(i)&7)) != 0 } -// roundToWord rounds n up to the nearest multiple of 8. The spec requires +// roundToWord rounds n up to the nearest multiple of 8. Row Format requires // every variable-length value and data region to be zero-padded to an // 8-byte boundary. func roundToWord(n int) int { From 86774ce4b48d5969a74510eab9b3f21a18c889b3 Mon Sep 17 00:00:00 2001 From: ayush00git Date: Thu, 30 Jul 2026 13:11:27 +0530 Subject: [PATCH 03/20] feat(go): define fixed-width and variable-width values --- go/fory/row/datatype.go | 313 +++++++++++++++++++++++++++++++++++++ go/fory/row/schema_test.go | 136 ++++++++++++++++ 2 files changed, 449 insertions(+) create mode 100644 go/fory/row/datatype.go create mode 100644 go/fory/row/schema_test.go diff --git a/go/fory/row/datatype.go b/go/fory/row/datatype.go new file mode 100644 index 0000000000..89cc7e70cc --- /dev/null +++ b/go/fory/row/datatype.go @@ -0,0 +1,313 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package row + +import ( + "fmt" + "strings" + + fory "github.com/apache/fory/go/fory" +) + +// DataType describes the row-format type of a field. +// +// Type ids are the cross-language Fory type ids shared with the Java, +// C++, and Python row-format implementations; they appear verbatim in +// the serialized schema wire format. +type DataType interface { + TypeID() fory.TypeId + // ByteWidth returns the fixed storage width in bytes, or -1 for + // variable-width types stored through an offset+size slot. + ByteWidth() int + String() string +} + +// Cross-language child field names for matching Java DataTypes. +const ( + listItemName = "item" + mapKeyName = "key" + mapValueName = "value" +) + +type BoolType struct{} + +func (BoolType) TypeID() fory.TypeId { return fory.BOOL } +func (BoolType) ByteWidth() int { return 1 } +func (BoolType) String() string { return "bool" } + +type Int8Type struct{} + +func (Int8Type) TypeID() fory.TypeId { return fory.INT8 } +func (Int8Type) ByteWidth() int { return 1 } +func (Int8Type) String() string { return "int8" } + +type Int16Type struct{} + +func (Int16Type) TypeID() fory.TypeId { return fory.INT16 } +func (Int16Type) ByteWidth() int { return 2 } +func (Int16Type) String() string { return "int16" } + +type Int32Type struct{} + +func (Int32Type) TypeID() fory.TypeId { return fory.INT32 } +func (Int32Type) ByteWidth() int { return 4 } +func (Int32Type) String() string { return "int32" } + +type Int64Type struct{} + +func (Int64Type) TypeID() fory.TypeId { return fory.INT64 } +func (Int64Type) ByteWidth() int { return 8 } +func (Int64Type) String() string { return "int64" } + +// Float16Type exists so schemas received from other languages parse, +// the Go writer, reader, and encoder do not support it yet. +type Float16Type struct{} + +func (Float16Type) TypeID() fory.TypeId { return fory.FLOAT16 } +func (Float16Type) ByteWidth() int { return 2 } +func (Float16Type) String() string { return "float16" } + +type Float32Type struct{} + +func (Float32Type) TypeID() fory.TypeId { return fory.FLOAT32 } +func (Float32Type) ByteWidth() int { return 4 } +func (Float32Type) String() string { return "float32" } + +type Float64Type struct{} + +func (Float64Type) TypeID() fory.TypeId { return fory.FLOAT64 } +func (Float64Type) ByteWidth() int { return 8 } +func (Float64Type) String() string { return "float64" } + +// StringType values are UTF-8 bytes in the variable data region, +// stored as (offset + size) in the fixed slot region. +type StringType struct{} + +func (StringType) TypeID() fory.TypeId { return fory.STRING } +func (StringType) ByteWidth() int { return -1 } +func (StringType) String() string { return "string" } + +type BinaryType struct{} + +func (BinaryType) TypeID() fory.TypeId { return fory.BINARY } +func (BinaryType) ByteWidth() int { return -1 } +func (BinaryType) String() string { return "binary" } + +// Date32Type values are days since the Unix epoch (int32). +type Date32Type struct{} + +func (Date32Type) TypeID() fory.TypeId { return fory.DATE } +func (Date32Type) ByteWidth() int { return 4 } +func (Date32Type) String() string { return "date32" } + +// TimestampType values are microseconds since the Unix epoch (int64). +type TimestampType struct{} + +func (TimestampType) TypeID() fory.TypeId { return fory.TIMESTAMP } +func (TimestampType) ByteWidth() int { return 8 } +func (TimestampType) String() string { return "timestamp" } + +// DurationType values are microseconds (int64). +type DurationType struct{} + +func (DurationType) TypeID() fory.TypeId { return fory.DURATION } +func (DurationType) ByteWidth() int { return 8 } +func (DurationType) String() string { return "duration" } + +// DecimalType exists so schemas received from other languages parse; the +// Go writer, reader, and encoder do not support it yet. Use it as a +// value, not a pointer, so schema equality compares precision and scale. +type DecimalType struct { + Precision int + Scale int +} + +func (DecimalType) TypeID() fory.TypeId { return fory.DECIMAL } +func (DecimalType) ByteWidth() int { return -1 } +func (t DecimalType) String() string { + return fmt.Sprintf("decimal(%d, %d)", t.Precision, t.Scale) +} + +// ListType is a variable-length sequence of Elem values. +type ListType struct { + Elem Field +} + +func (*ListType) TypeID() fory.TypeId { return fory.LIST } +func (*ListType) ByteWidth() int { return -1 } +func (t *ListType) String() string { return "list<" + t.Elem.Type.String() + ">" } + +// MapType stores keys and values as two adjacent arrays. +type MapType struct { + Key Field + Value Field +} + +func (*MapType) TypeID() fory.TypeId { return fory.MAP } +func (*MapType) ByteWidth() int { return -1 } +func (t *MapType) String() string { + return "map<" + t.Key.Type.String() + ", " + t.Value.Type.String() + ">" +} + +// StructType is a nested row with its own bitmap, slots, and variable +// data region. +type StructType struct { + Fields []Field +} + +func (*StructType) TypeID() fory.TypeId { return fory.STRUCT } +func (*StructType) ByteWidth() int { return -1 } +func (t *StructType) String() string { + var sb strings.Builder + sb.WriteString("struct<") + for i, f := range t.Fields { + if i > 0 { + sb.WriteString(", ") + } + sb.WriteString(f.Name) + sb.WriteString(": ") + sb.WriteString(f.Type.String()) + } + sb.WriteString(">") + return sb.String() +} + +// Field is a named, optionally nullable slot in a schema or composite type. +type Field struct { + Name string + Type DataType + Nullable bool +} + +func NewField(name string, dataType DataType, nullable bool) Field { + return Field{Name: name, Type: dataType, Nullable: nullable} +} + +func (f Field) Equal(o Field) bool { + return f.Name == o.Name && f.Nullable == o.Nullable && dataTypeEqual(f.Type, o.Type) +} + +// List returns a list type with the cross-language element field name +// "item" and nullable elements. Build ListType directly for +// non-nullable elements. +func List(elem DataType) *ListType { + return &ListType{Elem: Field{Name: listItemName, Type: elem, Nullable: true}} +} + +// Map returns a map type with the cross-language field names +// "key"/"value". Keys are always non-nullable. +func Map(key, value DataType) *MapType { + return &MapType{ + Key: Field{Name: mapKeyName, Type: key, Nullable: false}, + Value: Field{Name: mapValueName, Type: value, Nullable: true}, + } +} + +func Struct(fields []Field) *StructType { + return &StructType{Fields: fields} +} + +func dataTypeEqual(a, b DataType) bool { + switch at := a.(type) { + case *ListType: + bt, ok := b.(*ListType) + return ok && at.Elem.Equal(bt.Elem) + case *MapType: + bt, ok := b.(*MapType) + return ok && at.Key.Equal(bt.Key) && at.Value.Equal(bt.Value) + case *StructType: + bt, ok := b.(*StructType) + if !ok || len(at.Fields) != len(bt.Fields) { + return false + } + for i := range at.Fields { + if !at.Fields[i].Equal(bt.Fields[i]) { + return false + } + } + return true + default: + // Primitive types are empty structs and DecimalType is a + // comparable value, so interface equality is exact. + return a == b + } +} + +// Schema describes the fields of a top-level row. +type Schema struct { + fields []Field + byName map[string]int +} + +// NewSchema builds a schema from fields in their schema-declared order, +// which fixes the field slot layout. For duplicate names, FieldIndex +// resolves to the first occurrence. +func NewSchema(fields []Field) *Schema { + byName := make(map[string]int, len(fields)) + for i, f := range fields { + if _, ok := byName[f.Name]; !ok { + byName[f.Name] = i + } + } + return &Schema{fields: fields, byName: byName} +} + +func (s *Schema) NumFields() int { return len(s.fields) } + +func (s *Schema) Field(i int) Field { return s.fields[i] } + +// Fields returns the backing field slice; callers must not modify it. +func (s *Schema) Fields() []Field { return s.fields } + +// FieldIndex returns the ordinal of the named field, or -1 if absent. +func (s *Schema) FieldIndex(name string) int { + if i, ok := s.byName[name]; ok { + return i + } + return -1 +} + +func (s *Schema) Equal(o *Schema) bool { + if s == o { + return true + } + if o == nil || len(s.fields) != len(o.fields) { + return false + } + for i := range s.fields { + if !s.fields[i].Equal(o.fields[i]) { + return false + } + } + return true +} + +func (s *Schema) String() string { + var sb strings.Builder + sb.WriteString("schema<") + for i, f := range s.fields { + if i > 0 { + sb.WriteString(", ") + } + sb.WriteString(f.Name) + sb.WriteString(": ") + sb.WriteString(f.Type.String()) + } + sb.WriteString(">") + return sb.String() +} diff --git a/go/fory/row/schema_test.go b/go/fory/row/schema_test.go new file mode 100644 index 0000000000..2fc8691579 --- /dev/null +++ b/go/fory/row/schema_test.go @@ -0,0 +1,136 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package row + +import ( + "testing" + + fory "github.com/apache/fory/go/fory" + "github.com/stretchr/testify/require" +) + +// Type ids are serialized into the cross-language schema bytes, so they +// are pinned to their literal values here. +func TestTypeIDs(t *testing.T) { + cases := []struct { + dataType DataType + id fory.TypeId + }{ + {BoolType{}, 1}, + {Int8Type{}, 2}, + {Int16Type{}, 3}, + {Int32Type{}, 4}, + {Int64Type{}, 6}, + {Float16Type{}, 17}, + {Float32Type{}, 19}, + {Float64Type{}, 20}, + {StringType{}, 21}, + {List(Int32Type{}), 22}, + {Map(StringType{}, Int64Type{}), 24}, + {Struct(nil), 27}, + {DurationType{}, 37}, + {TimestampType{}, 38}, + {Date32Type{}, 39}, + {DecimalType{Precision: 10, Scale: 2}, 40}, + {BinaryType{}, 41}, + } + for _, c := range cases { + require.Equal(t, c.id, c.dataType.TypeID(), "%s", c.dataType) + } +} + +func TestByteWidths(t *testing.T) { + cases := []struct { + dataType DataType + width int + }{ + {BoolType{}, 1}, + {Int8Type{}, 1}, + {Int16Type{}, 2}, + {Int32Type{}, 4}, + {Int64Type{}, 8}, + {Float16Type{}, 2}, + {Float32Type{}, 4}, + {Float64Type{}, 8}, + {Date32Type{}, 4}, + {TimestampType{}, 8}, + {DurationType{}, 8}, + {StringType{}, -1}, + {BinaryType{}, -1}, + {DecimalType{}, -1}, + {List(Int32Type{}), -1}, + {Map(StringType{}, Int64Type{}), -1}, + {Struct(nil), -1}, + } + for _, c := range cases { + require.Equal(t, c.width, c.dataType.ByteWidth(), "%s", c.dataType) + } +} + +func TestCompositeFactories(t *testing.T) { + list := List(StringType{}) + require.Equal(t, "item", list.Elem.Name) + require.True(t, list.Elem.Nullable) + + m := Map(StringType{}, Int32Type{}) + require.Equal(t, "key", m.Key.Name) + require.False(t, m.Key.Nullable, "map keys must be non-nullable") + require.Equal(t, "value", m.Value.Name) + require.True(t, m.Value.Nullable) + + st := Struct([]Field{NewField("a", Int32Type{}, false)}) + require.Len(t, st.Fields, 1) + require.Equal(t, "a", st.Fields[0].Name) +} + +func TestSchemaLookup(t *testing.T) { + s := NewSchema([]Field{ + NewField("id", Int64Type{}, false), + NewField("name", StringType{}, true), + }) + require.Equal(t, 2, s.NumFields()) + require.Equal(t, "id", s.Field(0).Name) + require.Equal(t, 1, s.FieldIndex("name")) + require.Equal(t, -1, s.FieldIndex("missing")) +} + +func TestSchemaEqual(t *testing.T) { + nested := func() *Schema { + return NewSchema([]Field{ + NewField("id", Int64Type{}, false), + NewField("tags", List(StringType{}), true), + NewField("attrs", Map(StringType{}, Int32Type{}), true), + NewField("inner", Struct([]Field{NewField("x", Float64Type{}, false)}), true), + NewField("price", DecimalType{Precision: 10, Scale: 2}, true), + }) + } + require.True(t, nested().Equal(nested())) + + base := NewSchema([]Field{NewField("a", Int32Type{}, false)}) + require.False(t, base.Equal(nil)) + require.False(t, base.Equal(NewSchema([]Field{NewField("b", Int32Type{}, false)}))) + require.False(t, base.Equal(NewSchema([]Field{NewField("a", Int64Type{}, false)}))) + require.False(t, base.Equal(NewSchema([]Field{NewField("a", Int32Type{}, true)}))) + require.False(t, base.Equal(NewSchema(nil))) + + // Composite mismatches must compare structurally, not by pointer. + require.False(t, NewSchema([]Field{NewField("l", List(Int32Type{}), true)}). + Equal(NewSchema([]Field{NewField("l", List(Int64Type{}), true)}))) + require.False(t, NewSchema([]Field{NewField("d", DecimalType{Precision: 10, Scale: 2}, true)}). + Equal(NewSchema([]Field{NewField("d", DecimalType{Precision: 12, Scale: 2}, true)}))) +} From 0c700628067a287001db101043bf2401be6eac67 Mon Sep 17 00:00:00 2001 From: ayush00git Date: Thu, 30 Jul 2026 17:50:13 +0530 Subject: [PATCH 04/20] feat(go): add writer to write data byte blob to the buffer --- go/fory/row/writer.go | 370 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 370 insertions(+) create mode 100644 go/fory/row/writer.go diff --git a/go/fory/row/writer.go b/go/fory/row/writer.go new file mode 100644 index 0000000000..9b612674c9 --- /dev/null +++ b/go/fory/row/writer.go @@ -0,0 +1,370 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package row + +import ( + "encoding/binary" + "fmt" + "math" + "time" + + fory "github.com/apache/fory/go/fory" +) + +const maxArrayDataBytes = math.MaxInt32 - 15 + +// RowWriter writes one row into a ByteBuffer: null bitmap, fixed 8-byte +// field slots, then the variable data region. All slot and bitmap +// offsets are relative to the row base, so a nested RowWriter can share +// the parent's buffer and produce a self-contained row. +// +// Reset rebases the writer at the buffer's current writer index and must +// be called before writing each row. Writers are not goroutine-safe. +type RowWriter struct { + schema *Schema + buf *fory.ByteBuffer + base int + numFields int + bitmapWidth int + fixedSize int +} + +func NewRowWriter(schema *Schema) *RowWriter { + return NewRowWriterWithBuffer(schema, fory.NewByteBuffer(nil)) +} + +// NewRowWriterWithBuffer shares an existing buffer, e.g. to write a +// nested struct row into its parent's variable data region. +func NewRowWriterWithBuffer(schema *Schema, buf *fory.ByteBuffer) *RowWriter { + n := schema.NumFields() + bitmapWidth := bitmapWidthInBytes(n) + return &RowWriter{ + schema: schema, + buf: buf, + numFields: n, + bitmapWidth: bitmapWidth, + fixedSize: bitmapWidth + n*8, + } +} + +func (w *RowWriter) Schema() *Schema { return w.schema } +func (w *RowWriter) Buffer() *fory.ByteBuffer { return w.buf } + +// Reset rebases the writer at the current writer index and zeroes the +// whole fixed region. The buffer reuses dirty capacity after shrinking, +// so zeroing here is what makes null slots and padding deterministic; +// it also lets narrow fixed-width writes store just the value. +func (w *RowWriter) Reset() { + w.base = w.buf.WriterIndex() + w.buf.Reserve(w.fixedSize) + data := w.buf.GetData() + clear(data[w.base : w.base+w.fixedSize]) + w.buf.SetWriterIndex(w.base + w.fixedSize) +} + +// Size returns the bytes written for this row so far. +func (w *RowWriter) Size() int { return w.buf.WriterIndex() - w.base } + +// ToBytes returns a view of the row bytes; it stays valid only until the +// buffer is written to or reset again. +func (w *RowWriter) ToBytes() []byte { + return w.buf.GetByteSlice(w.base, w.buf.WriterIndex()) +} + +func (w *RowWriter) slot(i int) int { + if uint(i) >= uint(w.numFields) { + panic(fmt.Sprintf("row: field index %d out of range [0, %d)", i, w.numFields)) + } + return w.base + w.bitmapWidth + i*8 +} + +// SetNullAt marks field i null and zeroes its slot. +func (w *RowWriter) SetNullAt(i int) { + slot := w.slot(i) + data := w.buf.GetData() + setBit(data[w.base:w.base+w.bitmapWidth], i) + binary.LittleEndian.PutUint64(data[slot:], 0) +} + +func (w *RowWriter) SetNotNullAt(i int) { + w.slot(i) // bounds check + clearBit(w.buf.GetData()[w.base:w.base+w.bitmapWidth], i) +} + +func (w *RowWriter) WriteBool(i int, v bool) { + slot := w.slot(i) + if v { + w.buf.GetData()[slot] = 1 + } else { + w.buf.GetData()[slot] = 0 + } +} + +func (w *RowWriter) WriteInt8(i int, v int8) { + w.buf.GetData()[w.slot(i)] = byte(v) +} + +func (w *RowWriter) WriteInt16(i int, v int16) { + binary.LittleEndian.PutUint16(w.buf.GetData()[w.slot(i):], uint16(v)) +} + +func (w *RowWriter) WriteInt32(i int, v int32) { + binary.LittleEndian.PutUint32(w.buf.GetData()[w.slot(i):], uint32(v)) +} + +func (w *RowWriter) WriteInt64(i int, v int64) { + binary.LittleEndian.PutUint64(w.buf.GetData()[w.slot(i):], uint64(v)) +} + +func (w *RowWriter) WriteFloat32(i int, v float32) { + binary.LittleEndian.PutUint32(w.buf.GetData()[w.slot(i):], math.Float32bits(v)) +} + +func (w *RowWriter) WriteFloat64(i int, v float64) { + binary.LittleEndian.PutUint64(w.buf.GetData()[w.slot(i):], math.Float64bits(v)) +} + +func (w *RowWriter) WriteDate(i int, d fory.Date) error { + days, err := fory.DateToEpochDay(d) + if err != nil { + return err + } + if days < math.MinInt32 || days > math.MaxInt32 { + return fmt.Errorf("row: date %v out of date32 range", d) + } + w.WriteInt32(i, int32(days)) + return nil +} + +func (w *RowWriter) WriteTimestamp(i int, t time.Time) { + w.WriteInt64(i, t.UnixMicro()) +} + +func (w *RowWriter) WriteDuration(i int, d time.Duration) { + w.WriteInt64(i, d.Microseconds()) +} + +func (w *RowWriter) WriteString(i int, s string) { + start := appendStringRegion(w.buf, s) + w.SetOffsetAndSize(i, start, len(s)) +} + +func (w *RowWriter) WriteBytes(i int, b []byte) { + start := appendBytesRegion(w.buf, b) + w.SetOffsetAndSize(i, start, len(b)) +} + +// SetOffsetAndSize patches field i's slot with the row-relative offset +// and byte size of a value already appended to the variable data region. +// Use it after writing a nested struct, array, or map at absStart. +func (w *RowWriter) SetOffsetAndSize(i, absStart, size int) { + rel := absStart - w.base + binary.LittleEndian.PutUint64(w.buf.GetData()[w.slot(i):], uint64(rel)<<32|uint64(uint32(size))) +} + +// ArrayWriter writes one array: an 8-byte element count, a null bitmap, +// then elements at their natural width (variable-width elements use +// 8-byte offset+size slots). Offsets are relative to the array base. +type ArrayWriter struct { + elem Field + buf *fory.ByteBuffer + base int + elemSize int + numElements int + headerBytes int +} + +func NewArrayWriter(elem Field, buf *fory.ByteBuffer) *ArrayWriter { + elemSize := elem.Type.ByteWidth() + if elemSize < 0 { + elemSize = 8 + } + return &ArrayWriter{elem: elem, buf: buf, elemSize: elemSize} +} + +func (w *ArrayWriter) Buffer() *fory.ByteBuffer { return w.buf } + +// Reset rebases the writer at the current writer index and writes the +// array header plus a zeroed element region for numElements elements. +func (w *ArrayWriter) Reset(numElements int) error { + if numElements < 0 { + return fmt.Errorf("row: negative array length %d", numElements) + } + dataBytes := int64(numElements) * int64(w.elemSize) + if dataBytes > maxArrayDataBytes { + return fmt.Errorf("row: array of %d elements exceeds maximum size", numElements) + } + headerBytes := 8 + bitmapWidthInBytes(numElements) + total := headerBytes + roundToWord(int(dataBytes)) + base := w.buf.WriterIndex() + w.buf.Reserve(total) + data := w.buf.GetData() + binary.LittleEndian.PutUint64(data[base:], uint64(numElements)) + clear(data[base+8 : base+total]) + w.buf.SetWriterIndex(base + total) + w.base, w.numElements, w.headerBytes = base, numElements, headerBytes + return nil +} + +func (w *ArrayWriter) Size() int { return w.buf.WriterIndex() - w.base } + +func (w *ArrayWriter) ToBytes() []byte { + return w.buf.GetByteSlice(w.base, w.buf.WriterIndex()) +} + +func (w *ArrayWriter) slot(i int) int { + if uint(i) >= uint(w.numElements) { + panic(fmt.Sprintf("row: array index %d out of range [0, %d)", i, w.numElements)) + } + return w.base + w.headerBytes + i*w.elemSize +} + +// SetNullAt marks element i null and re-zeroes its element bytes. +func (w *ArrayWriter) SetNullAt(i int) { + slot := w.slot(i) + data := w.buf.GetData() + setBit(data[w.base+8:w.base+w.headerBytes], i) + clear(data[slot : slot+w.elemSize]) +} + +func (w *ArrayWriter) WriteBool(i int, v bool) { + slot := w.slot(i) + if v { + w.buf.GetData()[slot] = 1 + } else { + w.buf.GetData()[slot] = 0 + } +} + +func (w *ArrayWriter) WriteInt8(i int, v int8) { + w.buf.GetData()[w.slot(i)] = byte(v) +} + +func (w *ArrayWriter) WriteInt16(i int, v int16) { + binary.LittleEndian.PutUint16(w.buf.GetData()[w.slot(i):], uint16(v)) +} + +func (w *ArrayWriter) WriteInt32(i int, v int32) { + binary.LittleEndian.PutUint32(w.buf.GetData()[w.slot(i):], uint32(v)) +} + +func (w *ArrayWriter) WriteInt64(i int, v int64) { + binary.LittleEndian.PutUint64(w.buf.GetData()[w.slot(i):], uint64(v)) +} + +func (w *ArrayWriter) WriteFloat32(i int, v float32) { + binary.LittleEndian.PutUint32(w.buf.GetData()[w.slot(i):], math.Float32bits(v)) +} + +func (w *ArrayWriter) WriteFloat64(i int, v float64) { + binary.LittleEndian.PutUint64(w.buf.GetData()[w.slot(i):], math.Float64bits(v)) +} + +func (w *ArrayWriter) WriteDate(i int, d fory.Date) error { + days, err := fory.DateToEpochDay(d) + if err != nil { + return err + } + if days < math.MinInt32 || days > math.MaxInt32 { + return fmt.Errorf("row: date %v out of date32 range", d) + } + w.WriteInt32(i, int32(days)) + return nil +} + +func (w *ArrayWriter) WriteTimestamp(i int, t time.Time) { + w.WriteInt64(i, t.UnixMicro()) +} + +func (w *ArrayWriter) WriteDuration(i int, d time.Duration) { + w.WriteInt64(i, d.Microseconds()) +} + +func (w *ArrayWriter) WriteString(i int, s string) { + start := appendStringRegion(w.buf, s) + w.SetOffsetAndSize(i, start, len(s)) +} + +func (w *ArrayWriter) WriteBytes(i int, b []byte) { + start := appendBytesRegion(w.buf, b) + w.SetOffsetAndSize(i, start, len(b)) +} + +// SetOffsetAndSize patches element i's slot with the array-relative +// offset and byte size of a value already appended after the array. +func (w *ArrayWriter) SetOffsetAndSize(i, absStart, size int) { + rel := absStart - w.base + binary.LittleEndian.PutUint64(w.buf.GetData()[w.slot(i):], uint64(rel)<<32|uint64(uint32(size))) +} + +// MapWriter writes one map: an 8-byte keys-array size, the keys array, +// then the values array. Write flow: Reset, write the keys with an +// ArrayWriter, FinishKeys, write the values with another ArrayWriter. +type MapWriter struct { + buf *fory.ByteBuffer + base int +} + +func NewMapWriter(buf *fory.ByteBuffer) *MapWriter { + return &MapWriter{buf: buf} +} + +// Reset rebases the writer at the current writer index and reserves the +// keys-array size word, which FinishKeys patches later. +func (w *MapWriter) Reset() { + w.base = w.buf.WriterIndex() + w.buf.Reserve(8) + data := w.buf.GetData() + clear(data[w.base : w.base+8]) + w.buf.SetWriterIndex(w.base + 8) +} + +// FinishKeys records the keys array size; call it after the keys array +// is fully written and before starting the values array. +func (w *MapWriter) FinishKeys() { + keysSize := w.buf.WriterIndex() - w.base - 8 + binary.LittleEndian.PutUint64(w.buf.GetData()[w.base:], uint64(keysSize)) +} + +func (w *MapWriter) Size() int { return w.buf.WriterIndex() - w.base } + +// appendBytesRegion appends b to the variable data region, zero-padded +// to an 8-byte boundary, and returns its buffer offset. +func appendBytesRegion(buf *fory.ByteBuffer, b []byte) int { + n := len(b) + rounded := roundToWord(n) + start := buf.WriterIndex() + buf.Reserve(rounded) + data := buf.GetData() + clear(data[start+n : start+rounded]) + copy(data[start:], b) + buf.SetWriterIndex(start + rounded) + return start +} + +func appendStringRegion(buf *fory.ByteBuffer, s string) int { + n := len(s) + rounded := roundToWord(n) + start := buf.WriterIndex() + buf.Reserve(rounded) + data := buf.GetData() + clear(data[start+n : start+rounded]) + copy(data[start:], s) + buf.SetWriterIndex(start + rounded) + return start +} From 9680d573a9966eecedefe07ba8b0ca39ae1a94c0 Mon Sep 17 00:00:00 2001 From: ayush00git Date: Thu, 30 Jul 2026 18:12:44 +0530 Subject: [PATCH 05/20] feat(go): add row reader --- go/fory/row/row.go | 338 ++++++++++++++++++++++++++++++++++++++++ go/fory/row/row_test.go | 264 +++++++++++++++++++++++++++++++ 2 files changed, 602 insertions(+) create mode 100644 go/fory/row/row.go create mode 100644 go/fory/row/row_test.go diff --git a/go/fory/row/row.go b/go/fory/row/row.go new file mode 100644 index 0000000000..e2780b95c9 --- /dev/null +++ b/go/fory/row/row.go @@ -0,0 +1,338 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package row + +import ( + "encoding/binary" + "fmt" + "math" + "time" + + fory "github.com/apache/fory/go/fory" +) + +// Row is a zero-copy reader over one row's bytes. Getters read directly +// from the underlying data without deserializing other fields; nested +// values are sub-slice views, never copies (except String). +// +// Fixed-width getters return the zero value for null fields; use +// IsNullAt to distinguish. Variable-width getters return nil (or "") +// for null fields. An out-of-range field index panics; offsets in +// corrupt or untrusted data may panic on slice bounds, so wrap +// untrusted decoding with recover. +type Row struct { + fields []Field + data []byte + bitmapWidth int +} + +func NewRow(schema *Schema, data []byte) *Row { + return newRow(schema.Fields(), data) +} + +func newRow(fields []Field, data []byte) *Row { + return &Row{fields: fields, data: data, bitmapWidth: bitmapWidthInBytes(len(fields))} +} + +func (r *Row) NumFields() int { return len(r.fields) } +func (r *Row) SizeBytes() int { return len(r.data) } + +// Data returns the underlying row bytes; callers must not modify them. +func (r *Row) Data() []byte { return r.data } + +func (r *Row) IsNullAt(i int) bool { + if uint(i) >= uint(len(r.fields)) { + panic(fmt.Sprintf("row: field index %d out of range [0, %d)", i, len(r.fields))) + } + return getBit(r.data, i) +} + +func (r *Row) slot(i int) int { return r.bitmapWidth + i*8 } + +func (r *Row) Bool(i int) bool { + return !r.IsNullAt(i) && r.data[r.slot(i)] != 0 +} + +func (r *Row) Int8(i int) int8 { + if r.IsNullAt(i) { + return 0 + } + return int8(r.data[r.slot(i)]) +} + +func (r *Row) Int16(i int) int16 { + if r.IsNullAt(i) { + return 0 + } + return int16(binary.LittleEndian.Uint16(r.data[r.slot(i):])) +} + +func (r *Row) Int32(i int) int32 { + if r.IsNullAt(i) { + return 0 + } + return int32(binary.LittleEndian.Uint32(r.data[r.slot(i):])) +} + +func (r *Row) Int64(i int) int64 { + if r.IsNullAt(i) { + return 0 + } + return int64(binary.LittleEndian.Uint64(r.data[r.slot(i):])) +} + +func (r *Row) Float32(i int) float32 { + if r.IsNullAt(i) { + return 0 + } + return math.Float32frombits(binary.LittleEndian.Uint32(r.data[r.slot(i):])) +} + +func (r *Row) Float64(i int) float64 { + if r.IsNullAt(i) { + return 0 + } + return math.Float64frombits(binary.LittleEndian.Uint64(r.data[r.slot(i):])) +} + +func (r *Row) Date(i int) fory.Date { + if r.IsNullAt(i) { + return fory.Date{} + } + return dateFromDays(int32(binary.LittleEndian.Uint32(r.data[r.slot(i):]))) +} + +func (r *Row) Timestamp(i int) time.Time { + if r.IsNullAt(i) { + return time.Time{} + } + return time.UnixMicro(r.Int64(i)) +} + +func (r *Row) Duration(i int) time.Duration { + return time.Duration(r.Int64(i)) * time.Microsecond +} + +// varData returns the value bytes of variable-width field i, or nil if +// the field is null. An empty value returns an empty non-nil slice. +func (r *Row) varData(i int) []byte { + if r.IsNullAt(i) { + return nil + } + offset, size := decodeOffsetAndSize(binary.LittleEndian.Uint64(r.data[r.slot(i):])) + return r.data[offset : offset+size] +} + +// Binary returns a zero-copy view of field i's bytes. +func (r *Row) Binary(i int) []byte { return r.varData(i) } + +func (r *Row) String(i int) string { return string(r.varData(i)) } + +func (r *Row) Struct(i int) *Row { + data := r.varData(i) + if data == nil { + return nil + } + structType := r.fields[i].Type.(*StructType) + return newRow(structType.Fields, data) +} + +func (r *Row) Array(i int) *ArrayData { + data := r.varData(i) + if data == nil { + return nil + } + listType := r.fields[i].Type.(*ListType) + return NewArrayData(listType.Elem, data) +} + +func (r *Row) Map(i int) *MapData { + data := r.varData(i) + if data == nil { + return nil + } + return NewMapData(r.fields[i].Type.(*MapType), data) +} + +// ArrayData is a zero-copy reader over one array's bytes, with the same +// null and bounds semantics as Row. +type ArrayData struct { + elem Field + data []byte + numElements int + elemSize int + headerBytes int +} + +func NewArrayData(elem Field, data []byte) *ArrayData { + numElements := int(binary.LittleEndian.Uint64(data)) + elemSize := elem.Type.ByteWidth() + if elemSize < 0 { + elemSize = 8 + } + return &ArrayData{ + elem: elem, + data: data, + numElements: numElements, + elemSize: elemSize, + headerBytes: 8 + bitmapWidthInBytes(numElements), + } +} + +func (a *ArrayData) NumElements() int { return a.numElements } +func (a *ArrayData) SizeBytes() int { return len(a.data) } + +func (a *ArrayData) IsNullAt(i int) bool { + if uint(i) >= uint(a.numElements) { + panic(fmt.Sprintf("row: array index %d out of range [0, %d)", i, a.numElements)) + } + return getBit(a.data[8:a.headerBytes], i) +} + +func (a *ArrayData) slot(i int) int { return a.headerBytes + i*a.elemSize } + +func (a *ArrayData) Bool(i int) bool { + return !a.IsNullAt(i) && a.data[a.slot(i)] != 0 +} + +func (a *ArrayData) Int8(i int) int8 { + if a.IsNullAt(i) { + return 0 + } + return int8(a.data[a.slot(i)]) +} + +func (a *ArrayData) Int16(i int) int16 { + if a.IsNullAt(i) { + return 0 + } + return int16(binary.LittleEndian.Uint16(a.data[a.slot(i):])) +} + +func (a *ArrayData) Int32(i int) int32 { + if a.IsNullAt(i) { + return 0 + } + return int32(binary.LittleEndian.Uint32(a.data[a.slot(i):])) +} + +func (a *ArrayData) Int64(i int) int64 { + if a.IsNullAt(i) { + return 0 + } + return int64(binary.LittleEndian.Uint64(a.data[a.slot(i):])) +} + +func (a *ArrayData) Float32(i int) float32 { + if a.IsNullAt(i) { + return 0 + } + return math.Float32frombits(binary.LittleEndian.Uint32(a.data[a.slot(i):])) +} + +func (a *ArrayData) Float64(i int) float64 { + if a.IsNullAt(i) { + return 0 + } + return math.Float64frombits(binary.LittleEndian.Uint64(a.data[a.slot(i):])) +} + +func (a *ArrayData) Date(i int) fory.Date { + if a.IsNullAt(i) { + return fory.Date{} + } + return dateFromDays(int32(binary.LittleEndian.Uint32(a.data[a.slot(i):]))) +} + +func (a *ArrayData) Timestamp(i int) time.Time { + if a.IsNullAt(i) { + return time.Time{} + } + return time.UnixMicro(a.Int64(i)) +} + +func (a *ArrayData) Duration(i int) time.Duration { + return time.Duration(a.Int64(i)) * time.Microsecond +} + +func (a *ArrayData) varData(i int) []byte { + if a.IsNullAt(i) { + return nil + } + offset, size := decodeOffsetAndSize(binary.LittleEndian.Uint64(a.data[a.slot(i):])) + return a.data[offset : offset+size] +} + +// Binary returns a zero-copy view of element i's bytes. +func (a *ArrayData) Binary(i int) []byte { return a.varData(i) } + +func (a *ArrayData) String(i int) string { return string(a.varData(i)) } + +func (a *ArrayData) Struct(i int) *Row { + data := a.varData(i) + if data == nil { + return nil + } + return newRow(a.elem.Type.(*StructType).Fields, data) +} + +func (a *ArrayData) Array(i int) *ArrayData { + data := a.varData(i) + if data == nil { + return nil + } + return NewArrayData(a.elem.Type.(*ListType).Elem, data) +} + +func (a *ArrayData) Map(i int) *MapData { + data := a.varData(i) + if data == nil { + return nil + } + return NewMapData(a.elem.Type.(*MapType), data) +} + +// MapData is a zero-copy reader over one map's bytes: the keys array +// and values array views share the map's underlying data. +type MapData struct { + keys *ArrayData + values *ArrayData +} + +func NewMapData(mapType *MapType, data []byte) *MapData { + keysSize := int(binary.LittleEndian.Uint64(data)) + return &MapData{ + keys: NewArrayData(mapType.Key, data[8:8+keysSize]), + values: NewArrayData(mapType.Value, data[8+keysSize:]), + } +} + +func (m *MapData) NumElements() int { return m.keys.numElements } +func (m *MapData) Keys() *ArrayData { return m.keys } +func (m *MapData) Values() *ArrayData { return m.values } + +func decodeOffsetAndSize(slotValue uint64) (offset, size int) { + return int(slotValue >> 32), int(uint32(slotValue)) +} + +// dateFromDays converts int32 epoch days to a Date. The int32 range +// always yields an in-range year, so the conversion cannot fail. +func dateFromDays(days int32) fory.Date { + d, _ := fory.DateFromEpochDay(int64(days)) + return d +} diff --git a/go/fory/row/row_test.go b/go/fory/row/row_test.go new file mode 100644 index 0000000000..55f124a104 --- /dev/null +++ b/go/fory/row/row_test.go @@ -0,0 +1,264 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package row + +import ( + "strings" + "sync" + "testing" + "time" + + fory "github.com/apache/fory/go/fory" + "github.com/stretchr/testify/require" +) + +func int64StringSchema() *Schema { + return NewSchema([]Field{ + NewField("f1", Int64Type{}, false), + NewField("f2", StringType{}, true), + }) +} + +// The full byte image is fixed by the spec: 8-byte bitmap, one 8-byte +// slot per field (variable-width slots hold offset<<32|size relative to +// the row base), then the variable region zero-padded to 8 bytes. +func TestRowExactLayout(t *testing.T) { + w := NewRowWriter(int64StringSchema()) + w.Reset() + w.WriteInt64(0, 0x0102030405060708) + w.WriteString(1, "ab") + require.Equal(t, []byte{ + 0, 0, 0, 0, 0, 0, 0, 0, // null bitmap + 0x08, 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, // f1 slot + 0x02, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, // f2 slot: size=2, offset=24 + 'a', 'b', 0, 0, 0, 0, 0, 0, // variable region, zero-padded + }, w.ToBytes()) + + r := NewRow(w.Schema(), w.ToBytes()) + require.Equal(t, int64(0x0102030405060708), r.Int64(0)) + require.Equal(t, "ab", r.String(1)) +} + +func TestNullAndEmptyString(t *testing.T) { + w := NewRowWriter(int64StringSchema()) + w.Reset() + w.WriteInt64(0, 42) + w.SetNullAt(1) + require.Equal(t, []byte{ + 0x02, 0, 0, 0, 0, 0, 0, 0, // bit 1 set: f2 is null + 42, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, // null slot zeroed + }, w.ToBytes()) + r := NewRow(w.Schema(), w.ToBytes()) + require.True(t, r.IsNullAt(1)) + require.Nil(t, r.Binary(1)) + require.Equal(t, "", r.String(1)) + + // An empty string is not null: bit clear, size 0, no variable bytes. + w.Buffer().SetWriterIndex(0) + w.Reset() + w.WriteInt64(0, 42) + w.WriteString(1, "") + r = NewRow(w.Schema(), w.ToBytes()) + require.Equal(t, 24, r.SizeBytes()) + require.False(t, r.IsNullAt(1)) + require.Equal(t, "", r.String(1)) + require.NotNil(t, r.Binary(1)) + require.Empty(t, r.Binary(1)) +} + +func TestPrimitiveRoundTrip(t *testing.T) { + s := NewSchema([]Field{ + NewField("b", BoolType{}, false), + NewField("i8", Int8Type{}, false), + NewField("i16", Int16Type{}, false), + NewField("i32", Int32Type{}, false), + NewField("i64", Int64Type{}, false), + NewField("f32", Float32Type{}, false), + NewField("f64", Float64Type{}, false), + NewField("bin", BinaryType{}, true), + }) + w := NewRowWriter(s) + w.Reset() + w.WriteBool(0, true) + w.WriteInt8(1, -5) + w.WriteInt16(2, -1000) + w.WriteInt32(3, -100000) + w.WriteInt64(4, -5_000_000_000_000) + w.WriteFloat32(5, 3.5) + w.WriteFloat64(6, -2.25) + w.WriteBytes(7, []byte{1, 2, 3}) + + r := NewRow(s, w.ToBytes()) + require.True(t, r.Bool(0)) + require.Equal(t, int8(-5), r.Int8(1)) + require.Equal(t, int16(-1000), r.Int16(2)) + require.Equal(t, int32(-100000), r.Int32(3)) + require.Equal(t, int64(-5_000_000_000_000), r.Int64(4)) + require.Equal(t, float32(3.5), r.Float32(5)) + require.Equal(t, -2.25, r.Float64(6)) + require.Equal(t, []byte{1, 2, 3}, r.Binary(7)) +} + +func TestTemporalRoundTrip(t *testing.T) { + s := NewSchema([]Field{ + NewField("d", Date32Type{}, false), + NewField("ts", TimestampType{}, false), + NewField("dur", DurationType{}, false), + }) + w := NewRowWriter(s) + w.Reset() + date := fory.Date{Year: 2023, Month: time.March, Day: 15} + require.NoError(t, w.WriteDate(0, date)) + w.WriteTimestamp(1, time.UnixMicro(1_234_567_890_123_456)) + w.WriteDuration(2, 90*time.Second) + + r := NewRow(s, w.ToBytes()) + require.Equal(t, date, r.Date(0)) + require.Equal(t, int64(1_234_567_890_123_456), r.Timestamp(1).UnixMicro()) + require.Equal(t, 90*time.Second, r.Duration(2)) +} + +// A 65-field schema needs a 16-byte bitmap, shifting every slot by 16. +func TestMultiWordBitmap(t *testing.T) { + fields := make([]Field, 65) + for i := range fields { + fields[i] = NewField("f"+string(rune('a'+i%26))+string(rune('0'+i/26)), Int64Type{}, true) + } + s := NewSchema(fields) + w := NewRowWriter(s) + w.Reset() + w.WriteInt64(0, 100) + w.WriteInt64(64, 200) + w.SetNullAt(63) + + data := w.ToBytes() + require.Equal(t, 16+65*8, len(data)) + require.Equal(t, byte(0x80), data[7], "bit 63 is the top bit of bitmap byte 7") + + r := NewRow(s, data) + require.Equal(t, int64(100), r.Int64(0)) + require.Equal(t, int64(200), r.Int64(64)) + require.True(t, r.IsNullAt(63)) + require.Equal(t, int64(0), r.Int64(63)) +} + +// The buffer reuses dirty capacity, so a rewritten smaller row must +// still produce byte-identical output with zeroed padding and slots. +func TestWriterReuseZeroesDirtyBuffer(t *testing.T) { + w := NewRowWriter(int64StringSchema()) + w.Reset() + w.WriteInt64(0, 1) + w.WriteString(1, strings.Repeat("Z", 64)) + + w.Buffer().SetWriterIndex(0) + w.Reset() + w.WriteInt64(0, 0x0102030405060708) + w.WriteString(1, "ab") + require.Equal(t, []byte{ + 0, 0, 0, 0, 0, 0, 0, 0, + 0x08, 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, + 0x02, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, + 'a', 'b', 0, 0, 0, 0, 0, 0, + }, w.ToBytes()) + + w.Buffer().SetWriterIndex(0) + w.Reset() + w.WriteInt64(0, 42) + w.SetNullAt(1) + require.Equal(t, []byte{ + 0x02, 0, 0, 0, 0, 0, 0, 0, + 42, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + }, w.ToBytes()) +} + +func TestNestedStruct(t *testing.T) { + inner := Struct([]Field{ + NewField("x", Float64Type{}, false), + NewField("s", StringType{}, true), + }) + s := NewSchema([]Field{ + NewField("id", Int64Type{}, false), + NewField("inner", inner, true), + }) + w := NewRowWriter(s) + w.Reset() + w.WriteInt64(0, 7) + child := NewRowWriterWithBuffer(NewSchema(inner.Fields), w.Buffer()) + start := w.Buffer().WriterIndex() + child.Reset() + child.WriteFloat64(0, 3.5) + child.WriteString(1, "hi") + w.SetOffsetAndSize(1, start, w.Buffer().WriterIndex()-start) + + r := NewRow(s, w.ToBytes()) + require.Equal(t, int64(7), r.Int64(0)) + nested := r.Struct(1) + require.NotNil(t, nested) + require.Equal(t, 3.5, nested.Float64(0)) + require.Equal(t, "hi", nested.String(1)) +} + +// A value larger than the initial buffer forces mid-row growth; slots +// patched after reallocation must still land in the right place. +func TestLargeStringGrowth(t *testing.T) { + s := NewSchema([]Field{NewField("s", StringType{}, true)}) + w := NewRowWriter(s) + w.Reset() + big := strings.Repeat("x", 5000) + w.WriteString(0, big) + + r := NewRow(s, w.ToBytes()) + require.Equal(t, big, r.String(0)) + require.Equal(t, 8+8+roundToWord(5000), r.SizeBytes()) +} + +// One-past-the-end indices must be rejected, matching the other +// row format implementations. +func TestOutOfRangeIndexPanics(t *testing.T) { + w := NewRowWriter(int64StringSchema()) + w.Reset() + require.Panics(t, func() { w.WriteInt64(2, 1) }) + require.Panics(t, func() { w.SetNullAt(-1) }) + + r := NewRow(w.Schema(), w.ToBytes()) + require.Panics(t, func() { r.Int64(2) }) + require.Panics(t, func() { r.IsNullAt(-1) }) +} + +func TestConcurrentReads(t *testing.T) { + w := NewRowWriter(int64StringSchema()) + w.Reset() + w.WriteInt64(0, 99) + w.WriteString(1, "shared") + r := NewRow(w.Schema(), w.ToBytes()) + + var wg sync.WaitGroup + for g := 0; g < 4; g++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < 200; j++ { + require.Equal(t, int64(99), r.Int64(0)) + require.Equal(t, "shared", r.String(1)) + } + }() + } + wg.Wait() +} From 81ae126d85918d7d8ae553f6c58cd89670bf2a22 Mon Sep 17 00:00:00 2001 From: ayush00git Date: Thu, 30 Jul 2026 18:13:16 +0530 Subject: [PATCH 06/20] feat(test): add array and map reader tests --- go/fory/row/array_test.go | 108 ++++++++++++++++++++++++++++++++++++++ go/fory/row/map_test.go | 90 +++++++++++++++++++++++++++++++ 2 files changed, 198 insertions(+) create mode 100644 go/fory/row/array_test.go create mode 100644 go/fory/row/map_test.go diff --git a/go/fory/row/array_test.go b/go/fory/row/array_test.go new file mode 100644 index 0000000000..9ede434760 --- /dev/null +++ b/go/fory/row/array_test.go @@ -0,0 +1,108 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package row + +import ( + "testing" + + fory "github.com/apache/fory/go/fory" + "github.com/stretchr/testify/require" +) + +// Arrays store an 8-byte count, a null bitmap rounded to 8-byte words, +// then elements at their natural width, padded to an 8-byte boundary. +func TestArrayExactLayout(t *testing.T) { + buf := fory.NewByteBuffer(nil) + w := NewArrayWriter(List(Int32Type{}).Elem, buf) + require.NoError(t, w.Reset(3)) + w.WriteInt32(0, 1) + w.WriteInt32(1, 2) + w.WriteInt32(2, 3) + require.Equal(t, []byte{ + 3, 0, 0, 0, 0, 0, 0, 0, // element count + 0, 0, 0, 0, 0, 0, 0, 0, // null bitmap + 1, 0, 0, 0, 2, 0, 0, 0, // int32 elements at natural width + 3, 0, 0, 0, 0, 0, 0, 0, // last element + alignment padding + }, w.ToBytes()) + + a := NewArrayData(List(Int32Type{}).Elem, w.ToBytes()) + require.Equal(t, 3, a.NumElements()) + require.Equal(t, int32(2), a.Int32(1)) +} + +func TestStringArrayWithNullElement(t *testing.T) { + buf := fory.NewByteBuffer(nil) + elem := List(StringType{}).Elem + w := NewArrayWriter(elem, buf) + require.NoError(t, w.Reset(3)) + w.WriteString(0, "str1") + w.SetNullAt(1) + w.WriteString(2, "str2") + + a := NewArrayData(elem, w.ToBytes()) + require.Equal(t, 3, a.NumElements()) + require.Equal(t, "str1", a.String(0)) + require.True(t, a.IsNullAt(1)) + require.Equal(t, "", a.String(1)) + require.Equal(t, "str2", a.String(2)) +} + +func TestEmptyArray(t *testing.T) { + buf := fory.NewByteBuffer(nil) + elem := List(Int64Type{}).Elem + w := NewArrayWriter(elem, buf) + require.NoError(t, w.Reset(0)) + require.Equal(t, []byte{0, 0, 0, 0, 0, 0, 0, 0}, w.ToBytes()) + + a := NewArrayData(elem, w.ToBytes()) + require.Equal(t, 0, a.NumElements()) + require.Panics(t, func() { a.Int64(0) }) +} + +func TestNestedArray(t *testing.T) { + buf := fory.NewByteBuffer(nil) + outerElem := List(List(Int32Type{})).Elem // item: list + innerElem := List(Int32Type{}).Elem // item: int32 + + outer := NewArrayWriter(outerElem, buf) + require.NoError(t, outer.Reset(2)) + inner := NewArrayWriter(innerElem, buf) + for i, values := range [][]int32{{1, 2}, {3, 4, 5}} { + start := buf.WriterIndex() + require.NoError(t, inner.Reset(len(values))) + for j, v := range values { + inner.WriteInt32(j, v) + } + outer.SetOffsetAndSize(i, start, buf.WriterIndex()-start) + } + + a := NewArrayData(outerElem, outer.ToBytes()) + require.Equal(t, 2, a.NumElements()) + first := a.Array(0) + require.Equal(t, 2, first.NumElements()) + require.Equal(t, int32(2), first.Int32(1)) + second := a.Array(1) + require.Equal(t, 3, second.NumElements()) + require.Equal(t, int32(5), second.Int32(2)) +} + +func TestArrayResetRejectsInvalidLength(t *testing.T) { + w := NewArrayWriter(List(Int64Type{}).Elem, fory.NewByteBuffer(nil)) + require.Error(t, w.Reset(-1)) + require.Error(t, w.Reset(1<<40)) +} diff --git a/go/fory/row/map_test.go b/go/fory/row/map_test.go new file mode 100644 index 0000000000..e1f1fa4f93 --- /dev/null +++ b/go/fory/row/map_test.go @@ -0,0 +1,90 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package row + +import ( + "encoding/binary" + "testing" + + fory "github.com/apache/fory/go/fory" + "github.com/stretchr/testify/require" +) + +// Maps are an 8-byte keys-array size followed by two complete arrays; +// both halves must decode independently. +func TestMapLayout(t *testing.T) { + buf := fory.NewByteBuffer(nil) + mapType := Map(StringType{}, Int64Type{}) + m := NewMapWriter(buf) + m.Reset() + keys := NewArrayWriter(mapType.Key, buf) + require.NoError(t, keys.Reset(2)) + keys.WriteString(0, "k1") + keys.WriteString(1, "k2") + m.FinishKeys() + values := NewArrayWriter(mapType.Value, buf) + require.NoError(t, values.Reset(2)) + values.WriteInt64(0, 10) + values.WriteInt64(1, 20) + + data := buf.GetByteSlice(0, buf.WriterIndex()) + // keys array: 8 count + 8 bitmap + 2*8 slots + 2*8 padded strings. + require.Equal(t, uint64(48), binary.LittleEndian.Uint64(data)) + require.Equal(t, 8+48+32, m.Size()) + + md := NewMapData(mapType, data) + require.Equal(t, 2, md.NumElements()) + require.Equal(t, "k1", md.Keys().String(0)) + require.Equal(t, "k2", md.Keys().String(1)) + require.Equal(t, int64(10), md.Values().Int64(0)) + require.Equal(t, int64(20), md.Values().Int64(1)) +} + +func TestMapFieldInRow(t *testing.T) { + mapType := Map(StringType{}, Int32Type{}) + s := NewSchema([]Field{ + NewField("id", Int64Type{}, false), + NewField("attrs", mapType, true), + }) + w := NewRowWriter(s) + w.Reset() + w.WriteInt64(0, 1) + + buf := w.Buffer() + start := buf.WriterIndex() + m := NewMapWriter(buf) + m.Reset() + keys := NewArrayWriter(mapType.Key, buf) + require.NoError(t, keys.Reset(2)) + keys.WriteString(0, "a") + keys.WriteString(1, "b") + m.FinishKeys() + values := NewArrayWriter(mapType.Value, buf) + require.NoError(t, values.Reset(2)) + values.WriteInt32(0, 1) + values.SetNullAt(1) + w.SetOffsetAndSize(1, start, buf.WriterIndex()-start) + + r := NewRow(s, w.ToBytes()) + md := r.Map(1) + require.NotNil(t, md) + require.Equal(t, 2, md.NumElements()) + require.Equal(t, "a", md.Keys().String(0)) + require.Equal(t, int32(1), md.Values().Int32(0)) + require.True(t, md.Values().IsNullAt(1)) +} From 6005b886d2a767e3856f4467f7237893fda7bf20 Mon Sep 17 00:00:00 2001 From: ayush00git Date: Thu, 30 Jul 2026 18:43:18 +0530 Subject: [PATCH 07/20] feat(schema): add schema bytes methods to interop with Java --- go/fory/row/schema_bytes.go | 343 +++++++++++++++++++++++++++++++ go/fory/row/schema_bytes_test.go | 161 +++++++++++++++ 2 files changed, 504 insertions(+) create mode 100644 go/fory/row/schema_bytes.go create mode 100644 go/fory/row/schema_bytes_test.go diff --git a/go/fory/row/schema_bytes.go b/go/fory/row/schema_bytes.go new file mode 100644 index 0000000000..0150789c7f --- /dev/null +++ b/go/fory/row/schema_bytes.go @@ -0,0 +1,343 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package row + +import ( + "fmt" + "math" + + fory "github.com/apache/fory/go/fory" + "github.com/apache/fory/go/fory/meta" +) + +// Schema wire format, byte-compatible with Java SchemaEncoder: +// +// | version (1 byte) | num_fields (var uint32 small7) | fields... | +// +// Field: | header (1 byte) | name size (varint, only if large) | name +// bytes | type info |. Header bits 0-1 hold the index into +// fieldNameEncodings, bits 2-5 hold nameSize-1 (15 means a varint with +// nameSize-15 follows), bit 6 is the nullable flag. Type info is the +// type id byte, plus precision+scale for DECIMAL, a recursive element +// field for LIST, recursive key and value fields for MAP, and a field +// count plus recursive fields for STRUCT. +const ( + schemaVersion = 1 + fieldNameSizeThreshold = 15 +) + +// Header bits 0-1 store the INDEX into this table (matching Java +// SchemaEncoder.FIELD_NAME_ENCODINGS order), not the meta.Encoding +// wire values, which differ. +var fieldNameEncodings = [3]meta.Encoding{ + meta.UTF_8, + meta.ALL_TO_LOWER_SPECIAL, + meta.LOWER_UPPER_DIGIT_SPECIAL, +} + +// Stateless after construction, safe for concurrent use. +var ( + fieldNameEncoder = meta.NewEncoder('$', '_') + fieldNameDecoder = meta.NewDecoder('$', '_') +) + +func encodingIndexOf(encoding meta.Encoding) int { + for i, e := range fieldNameEncodings { + if e == encoding { + return i + } + } + return -1 +} + +// SchemaToBytes serializes a schema to the cross-language wire format. +func SchemaToBytes(s *Schema) ([]byte, error) { + buf := fory.NewByteBuffer(nil) + if err := SchemaToBuffer(s, buf); err != nil { + return nil, err + } + return buf.GetByteSlice(0, buf.WriterIndex()), nil +} + +// SchemaToBuffer serializes a schema into an existing buffer. +func SchemaToBuffer(s *Schema, buf *fory.ByteBuffer) error { + buf.WriteByte_(schemaVersion) + buf.WriteVarUint32Small7(uint32(s.NumFields())) + for _, f := range s.Fields() { + if err := writeSchemaField(buf, f); err != nil { + return err + } + } + return nil +} + +func writeSchemaField(buf *fory.ByteBuffer, f Field) error { + if f.Name == "" { + return fmt.Errorf("row: schema field with empty name") + } + encoding := fieldNameEncoder.ComputeEncodingWith(f.Name, fieldNameEncodings[:]) + metaString, err := fieldNameEncoder.EncodeWithEncoding(f.Name, encoding) + if err != nil { + return fmt.Errorf("row: encoding field name %q: %w", f.Name, err) + } + nameBytes := metaString.GetEncodedBytes() + nameSize := len(nameBytes) + encodingIndex := encodingIndexOf(metaString.GetEncoding()) + if encodingIndex < 0 { + return fmt.Errorf("row: field name %q got unsupported encoding %d", f.Name, metaString.GetEncoding()) + } + + header := encodingIndex & 0x03 + bigSize := nameSize > fieldNameSizeThreshold + if bigSize { + header |= fieldNameSizeThreshold << 2 + } else { + header |= (nameSize - 1) << 2 + } + if f.Nullable { + header |= 0x40 + } + buf.WriteByte_(byte(header)) + if bigSize { + buf.WriteVarUint32Small7(uint32(nameSize - fieldNameSizeThreshold)) + } + buf.WriteBinary(nameBytes) + return writeSchemaType(buf, f.Type) +} + +func writeSchemaType(buf *fory.ByteBuffer, dataType DataType) error { + buf.WriteByte_(byte(dataType.TypeID())) + switch t := dataType.(type) { + case DecimalType: + buf.WriteByte_(byte(t.Precision)) + buf.WriteByte_(byte(t.Scale)) + case *ListType: + return writeSchemaField(buf, t.Elem) + case *MapType: + if err := writeSchemaField(buf, t.Key); err != nil { + return err + } + return writeSchemaField(buf, t.Value) + case *StructType: + buf.WriteVarUint32Small7(uint32(len(t.Fields))) + for _, f := range t.Fields { + if err := writeSchemaField(buf, f); err != nil { + return err + } + } + } + return nil +} + +// SchemaFromBytes deserializes a schema from the cross-language wire +// format. It is safe on untrusted input: declared sizes are checked +// against remaining bytes before any allocation. +func SchemaFromBytes(data []byte) (*Schema, error) { + r := &schemaReader{buf: fory.NewByteBuffer(data), size: len(data)} + version := r.buf.ReadUint8(&r.err) + if err := r.err.CheckError(); err != nil { + return nil, err + } + if version != schemaVersion { + return nil, fmt.Errorf("row: unsupported schema version %d, expected %d", version, schemaVersion) + } + numFields := int(r.buf.ReadVarUint32Small7(&r.err)) + if err := r.err.CheckError(); err != nil { + return nil, err + } + // Every field costs at least one byte, so a declared count larger + // than the remaining input is corrupt; checking before the + // allocation below keeps attacker-declared counts harmless. + if numFields > r.remaining() { + return nil, fmt.Errorf("row: schema declares %d fields but only %d bytes remain", numFields, r.remaining()) + } + fields := make([]Field, 0, numFields) + for i := 0; i < numFields; i++ { + f, err := r.readField() + if err != nil { + return nil, err + } + fields = append(fields, f) + } + return NewSchema(fields), nil +} + +type schemaReader struct { + buf *fory.ByteBuffer + size int + err fory.Error +} + +func (r *schemaReader) remaining() int { return r.size - r.buf.ReaderIndex() } + +func (r *schemaReader) readField() (Field, error) { + header := int(r.buf.ReadUint8(&r.err)) + if err := r.err.CheckError(); err != nil { + return Field{}, err + } + encodingIndex := header & 0x03 + if encodingIndex >= len(fieldNameEncodings) { + return Field{}, fmt.Errorf("row: invalid field name encoding index %d", encodingIndex) + } + nameSizeMinus1 := (header >> 2) & 0x0F + nullable := header&0x40 != 0 + + var nameSize int + if nameSizeMinus1 == fieldNameSizeThreshold { + nameSize = int(r.buf.ReadVarUint32Small7(&r.err)) + fieldNameSizeThreshold + } else { + nameSize = nameSizeMinus1 + 1 + } + if err := r.err.CheckError(); err != nil { + return Field{}, err + } + if nameSize > r.remaining() { + return Field{}, fmt.Errorf("row: field name of %d bytes exceeds %d remaining", nameSize, r.remaining()) + } + nameBytes := r.buf.ReadBinary(nameSize, &r.err) + if err := r.err.CheckError(); err != nil { + return Field{}, err + } + name, err := fieldNameDecoder.Decode(nameBytes, fieldNameEncodings[encodingIndex]) + if err != nil { + return Field{}, fmt.Errorf("row: decoding field name: %w", err) + } + dataType, err := r.readType() + if err != nil { + return Field{}, err + } + return Field{Name: name, Type: dataType, Nullable: nullable}, nil +} + +func (r *schemaReader) readType() (DataType, error) { + typeID := fory.TypeId(r.buf.ReadUint8(&r.err)) + if err := r.err.CheckError(); err != nil { + return nil, err + } + switch typeID { + case fory.BOOL: + return BoolType{}, nil + case fory.INT8: + return Int8Type{}, nil + case fory.INT16: + return Int16Type{}, nil + case fory.INT32: + return Int32Type{}, nil + case fory.INT64: + return Int64Type{}, nil + case fory.FLOAT16: + return Float16Type{}, nil + case fory.FLOAT32: + return Float32Type{}, nil + case fory.FLOAT64: + return Float64Type{}, nil + case fory.STRING: + return StringType{}, nil + case fory.BINARY: + return BinaryType{}, nil + case fory.DURATION: + return DurationType{}, nil + case fory.TIMESTAMP: + return TimestampType{}, nil + case fory.DATE: + return Date32Type{}, nil + case fory.DECIMAL: + precision := int(r.buf.ReadUint8(&r.err)) + scale := int(r.buf.ReadUint8(&r.err)) + if err := r.err.CheckError(); err != nil { + return nil, err + } + return DecimalType{Precision: precision, Scale: scale}, nil + case fory.LIST: + elem, err := r.readField() + if err != nil { + return nil, err + } + return &ListType{Elem: elem}, nil + case fory.MAP: + keyField, err := r.readField() + if err != nil { + return nil, err + } + valueField, err := r.readField() + if err != nil { + return nil, err + } + // Java rebuilds maps from the child types only, restoring the + // canonical key/value names and nullability. + return Map(keyField.Type, valueField.Type), nil + case fory.STRUCT: + numFields := int(r.buf.ReadVarUint32Small7(&r.err)) + if err := r.err.CheckError(); err != nil { + return nil, err + } + if numFields > r.remaining() { + return nil, fmt.Errorf("row: struct declares %d fields but only %d bytes remain", numFields, r.remaining()) + } + fields := make([]Field, 0, numFields) + for i := 0; i < numFields; i++ { + f, err := r.readField() + if err != nil { + return nil, err + } + fields = append(fields, f) + } + return &StructType{Fields: fields}, nil + default: + return nil, fmt.Errorf("row: unknown type id %d in schema", typeID) + } +} + +// ComputeSchemaHash computes the cross-language schema hash, matching +// Java DataTypes.computeSchemaHash and the Python equivalent: fold +// hash*31 + typeId over fields, recursing into list elements, map keys +// and values, and struct fields; on signed-64 overflow, arithmetic- +// shift the hash right by 2 and retry. +func ComputeSchemaHash(s *Schema) int64 { + hash := int64(17) + for _, f := range s.Fields() { + hash = computeFieldHash(hash, f) + } + return hash +} + +func computeFieldHash(hash int64, f Field) int64 { + typeID := int64(f.Type.TypeID()) + for { + if hash <= math.MaxInt64/31 && hash >= math.MinInt64/31 { + product := hash * 31 + if product <= math.MaxInt64-typeID { + hash = product + typeID + break + } + } + hash >>= 2 + } + switch t := f.Type.(type) { + case *ListType: + hash = computeFieldHash(hash, t.Elem) + case *MapType: + hash = computeFieldHash(hash, t.Key) + hash = computeFieldHash(hash, t.Value) + case *StructType: + for _, child := range t.Fields { + hash = computeFieldHash(hash, child) + } + } + return hash +} diff --git a/go/fory/row/schema_bytes_test.go b/go/fory/row/schema_bytes_test.go new file mode 100644 index 0000000000..3c790f5128 --- /dev/null +++ b/go/fory/row/schema_bytes_test.go @@ -0,0 +1,161 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package row + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +// Golden bytes and hashes generated by Java +// org.apache.fory.format.type.SchemaEncoder (SCHEMA_VERSION=1) and +// DataTypes.computeSchemaHash. Regenerate with those classes if the +// Java wire format ever changes; the Java<->Go cross-language test is +// the live guard. +var schemaGoldens = []struct { + name string + schema func() *Schema + bytes []byte + hash int64 +}{ + { + name: "simple", + schema: func() *Schema { + return NewSchema([]Field{ + NewField("id", Int32Type{}, true), + NewField("name", StringType{}, true), + NewField("score", Float64Type{}, true), + NewField("active", BoolType{}, true), + }) + }, + bytes: []byte{ + 0x01, 0x04, 0x45, 0xa0, 0x60, 0x04, 0x49, 0x34, 0x0c, 0x20, 0x15, 0x4d, + 0xc8, 0x4e, 0x89, 0x00, 0x14, 0x4d, 0x00, 0x53, 0x45, 0x48, 0x01, + }, + hash: 15839823, + }, + { + name: "nested", + schema: func() *Schema { + return NewSchema([]Field{ + NewField("person", Struct([]Field{ + NewField("name", StringType{}, true), + NewField("age", Int32Type{}, false), + }), true), + NewField("tags", List(StringType{}), true), + NewField("attrs", Map(StringType{}, Int64Type{}), true), + }) + }, + bytes: []byte{ + 0x01, 0x03, 0x4d, 0x3c, 0x91, 0x93, 0x9a, 0x1b, 0x02, 0x49, 0x34, 0x0c, + 0x20, 0x15, 0x05, 0x00, 0xc4, 0x04, 0x49, 0x4c, 0x06, 0x90, 0x16, 0x49, + 0x22, 0x64, 0x60, 0x15, 0x4d, 0x82, 0x73, 0x8c, 0x80, 0x18, 0x05, 0x28, + 0x98, 0x15, 0x4d, 0xd4, 0x0b, 0xa1, 0x00, 0x06, + }, + hash: 15260761278193, + }, + { + name: "special", + schema: func() *Schema { + return NewSchema([]Field{ + NewField("a_very_long_field_name_exceeding_limit", Int64Type{}, false), + NewField("field2", StringType{}, true), + NewField("$special_name", Date32Type{}, true), + NewField("CamelCase", TimestampType{}, false), + NewField("price", DecimalType{Precision: 10, Scale: 2}, true), + }) + }, + bytes: []byte{ + 0x01, 0x05, 0x3d, 0x09, 0x03, 0x75, 0x24, 0x71, 0xb5, 0xb9, 0xa6, 0xd9, + 0x50, 0x45, 0x8f, 0x6d, 0x03, 0x09, 0xb2, 0x5c, 0x44, 0x20, 0xd0, 0xd3, + 0x6d, 0x68, 0x62, 0x26, 0x06, 0x52, 0x0a, 0x40, 0x85, 0x87, 0xb0, 0x15, + 0x61, 0xf2, 0x4f, 0x20, 0x90, 0x05, 0xed, 0xa0, 0x61, 0x00, 0x27, 0x1a, + 0x38, 0x01, 0x82, 0x16, 0xe0, 0x09, 0x08, 0x26, 0x4d, 0xbe, 0x28, 0x11, + 0x00, 0x28, 0x0a, 0x02, + }, + hash: 492901001, + }, +} + +func TestSchemaBytesGolden(t *testing.T) { + for _, g := range schemaGoldens { + schema := g.schema() + + got, err := SchemaToBytes(schema) + require.NoError(t, err, g.name) + require.Equal(t, g.bytes, got, "%s: bytes must match Java SchemaEncoder output", g.name) + + parsed, err := SchemaFromBytes(g.bytes) + require.NoError(t, err, g.name) + require.True(t, parsed.Equal(schema), "%s: parsed schema %v != %v", g.name, parsed, schema) + } +} + +func TestSchemaHashGolden(t *testing.T) { + for _, g := range schemaGoldens { + require.Equal(t, g.hash, ComputeSchemaHash(g.schema()), g.name) + } + + // 20 int64 fields overflow the int64 hash, exercising Java's + // shift-right-2-and-retry path. + fields := make([]Field, 20) + for i := range fields { + fields[i] = NewField(fmt.Sprintf("f%d", i), Int64Type{}, true) + } + require.Equal(t, int64(2627256684647412568), ComputeSchemaHash(NewSchema(fields))) +} + +func requireErrorContains(t *testing.T, err error, substr string) { + t.Helper() + require.Error(t, err) + require.Contains(t, err.Error(), substr) +} + +func TestSchemaFromBytesErrors(t *testing.T) { + // Unsupported version. + _, err := SchemaFromBytes([]byte{0x02, 0x00}) + requireErrorContains(t, err, "schema version") + + // Invalid field name encoding index (bits 0-1 == 3). + _, err = SchemaFromBytes([]byte{0x01, 0x01, 0x07}) + requireErrorContains(t, err, "encoding index") + + // Unknown type id. + corrupt := append([]byte(nil), schemaGoldens[0].bytes...) + corrupt[len(corrupt)-1] = 0x63 + _, err = SchemaFromBytes(corrupt) + requireErrorContains(t, err, "unknown type id") + + // Field count larger than the remaining input. + _, err = SchemaFromBytes([]byte{0x01, 0x7f}) + requireErrorContains(t, err, "fields") + + // Every strict prefix of a valid encoding must fail, never panic. + golden := schemaGoldens[1].bytes + for i := 0; i < len(golden); i++ { + _, err := SchemaFromBytes(golden[:i]) + require.Error(t, err, "prefix of length %d", i) + } +} + +func TestSchemaToBytesRejectsEmptyName(t *testing.T) { + _, err := SchemaToBytes(NewSchema([]Field{NewField("", Int32Type{}, true)})) + requireErrorContains(t, err, "empty name") +} From cc915029c429399ded63164687b29ae311a9251b Mon Sep 17 00:00:00 2001 From: ayush00git Date: Thu, 30 Jul 2026 19:03:50 +0530 Subject: [PATCH 08/20] fix: reject untrusted data payloads --- go/fory/row/row.go | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/go/fory/row/row.go b/go/fory/row/row.go index e2780b95c9..1fedf6ad38 100644 --- a/go/fory/row/row.go +++ b/go/fory/row/row.go @@ -135,7 +135,7 @@ func (r *Row) varData(i int) []byte { return nil } offset, size := decodeOffsetAndSize(binary.LittleEndian.Uint64(r.data[r.slot(i):])) - return r.data[offset : offset+size] + return boundedSlice(r.data, offset, size) } // Binary returns a zero-copy view of field i's bytes. @@ -197,6 +197,17 @@ func NewArrayData(elem Field, data []byte) *ArrayData { func (a *ArrayData) NumElements() int { return a.numElements } func (a *ArrayData) SizeBytes() int { return len(a.data) } +// validateBounds rejects arrays whose declared element count is not +// covered by the available bytes, so decoders can check before +// allocating from an attacker-declared count. +func (a *ArrayData) validateBounds() error { + need := int64(a.headerBytes) + int64(a.numElements)*int64(a.elemSize) + if a.numElements < 0 || need > int64(len(a.data)) { + return fmt.Errorf("row: array declares %d elements but holds only %d bytes", a.numElements, len(a.data)) + } + return nil +} + func (a *ArrayData) IsNullAt(i int) bool { if uint(i) >= uint(a.numElements) { panic(fmt.Sprintf("row: array index %d out of range [0, %d)", i, a.numElements)) @@ -275,7 +286,7 @@ func (a *ArrayData) varData(i int) []byte { return nil } offset, size := decodeOffsetAndSize(binary.LittleEndian.Uint64(a.data[a.slot(i):])) - return a.data[offset : offset+size] + return boundedSlice(a.data, offset, size) } // Binary returns a zero-copy view of element i's bytes. @@ -316,10 +327,24 @@ type MapData struct { func NewMapData(mapType *MapType, data []byte) *MapData { keysSize := int(binary.LittleEndian.Uint64(data)) + keysData := boundedSlice(data, 8, keysSize) + valuesData := boundedSlice(data, 8+keysSize, len(data)-8-keysSize) return &MapData{ - keys: NewArrayData(mapType.Key, data[8:8+keysSize]), - values: NewArrayData(mapType.Value, data[8+keysSize:]), + keys: NewArrayData(mapType.Key, keysData), + values: NewArrayData(mapType.Value, valuesData), + } +} + +// boundedSlice checks declared bounds against the data LENGTH +// and caps the view so nested +// readers cannot reach outside it either. Out-of-bounds panics are +// converted to errors by the encoder's decode entry points. +func boundedSlice(data []byte, offset, size int) []byte { + end := offset + size + if offset < 0 || size < 0 || end > len(data) { + panic(fmt.Sprintf("row: value bytes [%d:%d] exceed the enclosing %d-byte region", offset, end, len(data))) } + return data[offset:end:end] } func (m *MapData) NumElements() int { return m.keys.numElements } From c3a57d809eaed820a2b91a7177f8d41b6c329717 Mon Sep 17 00:00:00 2001 From: ayush00git Date: Thu, 30 Jul 2026 19:05:20 +0530 Subject: [PATCH 09/20] feat: add InferSchema validating struct against the schema --- go/fory/row/infer.go | 226 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 226 insertions(+) create mode 100644 go/fory/row/infer.go diff --git a/go/fory/row/infer.go b/go/fory/row/infer.go new file mode 100644 index 0000000000..a2daad1cb6 --- /dev/null +++ b/go/fory/row/infer.go @@ -0,0 +1,226 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package row + +import ( + "fmt" + "reflect" + "sort" + "strings" + "time" + "unicode" + "unicode/utf8" + + fory "github.com/apache/fory/go/fory" +) + +var ( + goDateType = reflect.TypeOf(fory.Date{}) + goTimeType = reflect.TypeOf(time.Time{}) + goDurationType = reflect.TypeOf(time.Duration(0)) +) + +// InferSchema infers the row schema for a struct type or pointer to +// struct, including every exported field not tagged `fory:"ignore"`. +// +// Fields are sorted by their lowerCamel name and named by its +// snake_case form (UserName -> user_name), matching Java's schema +// inference so both languages derive identical schemas. +// +// Type mapping: +// - bool, int8/16/32/64, float32/64: same-width row types; int maps to int64 +// - string, []byte: string and binary, nullable +// - slices, maps, nested structs: list, map, and struct, nullable +// - *T: the row type of T, nullable +// - fory.Date, time.Time, time.Duration: date32, timestamp, duration +// +// Unsigned integers, fixed-size arrays, nested pointers, and pointer +// map keys are unsupported and return an error. +func InferSchema(t reflect.Type) (*Schema, error) { + layout, err := inferStructLayout(t, nil) + if err != nil { + return nil, err + } + return layout.schema, nil +} + +// structLayout maps schema ordinals back to Go struct field indexes. +type structLayout struct { + schema *Schema + indexes []int // schema ordinal -> reflect struct field index +} + +func inferStructLayout(t reflect.Type, path []reflect.Type) (*structLayout, error) { + if t.Kind() == reflect.Ptr { + t = t.Elem() + } + if t.Kind() != reflect.Struct || t == goDateType || t == goTimeType { + return nil, fmt.Errorf("row: schema inference expects a struct type, got %v", t) + } + for _, seen := range path { + if seen == t { + return nil, fmt.Errorf("row: circular reference through type %v", t) + } + } + path = append(path, t) + + type member struct { + lowerCamel string + goIndex int + fieldType reflect.Type + } + var members []member + for i := 0; i < t.NumField(); i++ { + f := t.Field(i) + if f.PkgPath != "" || hasIgnoreTag(f.Tag.Get("fory")) { + continue + } + members = append(members, member{lowerFirst(f.Name), i, f.Type}) + } + // Java sorts by the lowerCamel member name before converting names + // to snake_case; matching that order is required for schema and + // row-layout compatibility. + sort.Slice(members, func(a, b int) bool { return members[a].lowerCamel < members[b].lowerCamel }) + + layout := &structLayout{indexes: make([]int, 0, len(members))} + fields := make([]Field, 0, len(members)) + for _, m := range members { + f, err := inferField(lowerCamelToLowerUnderscore(m.lowerCamel), m.fieldType, path) + if err != nil { + return nil, fmt.Errorf("%w (field %s of %v)", err, t.Field(m.goIndex).Name, t) + } + fields = append(fields, f) + layout.indexes = append(layout.indexes, m.goIndex) + } + layout.schema = NewSchema(fields) + return layout, nil +} + +func inferField(name string, t reflect.Type, path []reflect.Type) (Field, error) { + if t.Kind() == reflect.Ptr { + if t.Elem().Kind() == reflect.Ptr { + return Field{}, fmt.Errorf("row: nested pointer type %v is unsupported", t) + } + inner, err := inferField(name, t.Elem(), path) + if err != nil { + return Field{}, err + } + inner.Nullable = true + return inner, nil + } + switch t { + case goDateType: + return Field{Name: name, Type: Date32Type{}}, nil + case goTimeType: + return Field{Name: name, Type: TimestampType{}}, nil + case goDurationType: + return Field{Name: name, Type: DurationType{}}, nil + } + switch t.Kind() { + case reflect.Bool: + return Field{Name: name, Type: BoolType{}}, nil + case reflect.Int8: + return Field{Name: name, Type: Int8Type{}}, nil + case reflect.Int16: + return Field{Name: name, Type: Int16Type{}}, nil + case reflect.Int32: + return Field{Name: name, Type: Int32Type{}}, nil + case reflect.Int64, reflect.Int: + // `int` maps to int64 so the width never depends on the platform. + return Field{Name: name, Type: Int64Type{}}, nil + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return Field{}, fmt.Errorf("row: unsigned type %v is unsupported, use a signed type", t) + case reflect.Float32: + return Field{Name: name, Type: Float32Type{}}, nil + case reflect.Float64: + return Field{Name: name, Type: Float64Type{}}, nil + case reflect.String: + return Field{Name: name, Type: StringType{}, Nullable: true}, nil + case reflect.Slice: + if t.Elem().Kind() == reflect.Uint8 { + return Field{Name: name, Type: BinaryType{}, Nullable: true}, nil + } + elem, err := inferField(listItemName, t.Elem(), path) + if err != nil { + return Field{}, err + } + return Field{Name: name, Type: &ListType{Elem: elem}, Nullable: true}, nil + case reflect.Map: + if t.Key().Kind() == reflect.Ptr { + return Field{}, fmt.Errorf("row: pointer map key type %v is unsupported", t) + } + key, err := inferField(mapKeyName, t.Key(), path) + if err != nil { + return Field{}, err + } + key.Nullable = false + value, err := inferField(mapValueName, t.Elem(), path) + if err != nil { + return Field{}, err + } + return Field{Name: name, Type: &MapType{Key: key, Value: value}, Nullable: true}, nil + case reflect.Struct: + layout, err := inferStructLayout(t, path) + if err != nil { + return Field{}, err + } + return Field{Name: name, Type: &StructType{Fields: layout.schema.Fields()}, Nullable: true}, nil + default: + return Field{}, fmt.Errorf("row: type %v is unsupported in row format", t) + } +} + +func hasIgnoreTag(tag string) bool { + for _, part := range strings.Split(tag, ",") { + if strings.TrimSpace(part) == "ignore" { + return true + } + } + return false +} + +func lowerFirst(s string) string { + r, size := utf8.DecodeRuneInString(s) + if !unicode.IsUpper(r) { + return s + } + return string(unicode.ToLower(r)) + s[size:] +} + +// lowerCamelToLowerUnderscore ports Java StringUtils: every uppercase +// letter becomes '_' plus its lowercase, so userID becomes user_i_d. +func lowerCamelToLowerUnderscore(s string) string { + var b strings.Builder + from := 0 + for i := 0; i < len(s); i++ { + c := s[i] + if c >= 'A' && c <= 'Z' { + b.WriteString(s[from:i]) + b.WriteByte('_') + b.WriteByte(c + ('a' - 'A')) + from = i + 1 + } + } + if from == 0 { + return s + } + if from < len(s) { + b.WriteString(s[from:]) + } + return b.String() +} From f674b0bdf63657599d333519c23a5da3101f448f Mon Sep 17 00:00:00 2001 From: ayush00git Date: Thu, 30 Jul 2026 19:07:03 +0530 Subject: [PATCH 10/20] feat: encode go structs to fory row format --- go/fory/row/encoder.go | 494 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 494 insertions(+) create mode 100644 go/fory/row/encoder.go diff --git a/go/fory/row/encoder.go b/go/fory/row/encoder.go new file mode 100644 index 0000000000..fd5703ae49 --- /dev/null +++ b/go/fory/row/encoder.go @@ -0,0 +1,494 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package row + +import ( + "encoding/binary" + "fmt" + "reflect" + "time" + + fory "github.com/apache/fory/go/fory" +) + +// Encoder converts values of struct type T to and from row bytes using +// a codec tree compiled once with reflection at construction time. The +// schema is inferred per InferSchema. +// +// An Encoder owns a reusable write buffer and is NOT goroutine-safe; +// create one Encoder per goroutine. Decoding is safe on corrupt or +// untrusted input. +type Encoder[T any] struct { + codec *structCodec + hash int64 + buf *fory.ByteBuffer +} + +func NewEncoder[T any]() (*Encoder[T], error) { + t := reflect.TypeOf((*T)(nil)).Elem() + if t.Kind() != reflect.Struct { + return nil, fmt.Errorf("row: Encoder requires a struct type, got %v", t) + } + buf := fory.NewByteBuffer(nil) + codec, err := newStructCodec(t, buf) + if err != nil { + return nil, err + } + return &Encoder[T]{codec: codec, hash: ComputeSchemaHash(codec.schema), buf: buf}, nil +} + +func (e *Encoder[T]) Schema() *Schema { return e.codec.schema } + +// SchemaHash returns the cross-language schema hash used by Encode and +// Decode to detect out-of-sync struct definitions. +func (e *Encoder[T]) SchemaHash() int64 { return e.hash } + +// ToRow serializes v to row bytes (no framing, just the row). +func (e *Encoder[T]) ToRow(v *T) ([]byte, error) { + e.buf.SetWriterIndex(0) + if err := e.codec.writeStruct(reflect.ValueOf(v).Elem()); err != nil { + return nil, err + } + out := make([]byte, e.buf.WriterIndex()) + copy(out, e.buf.GetData()[:e.buf.WriterIndex()]) + return out, nil +} + +// FromRow deserializes row bytes produced with this schema. +func (e *Encoder[T]) FromRow(data []byte) (T, error) { + var out T + err := e.FromRowInto(data, &out) + return out, err +} + +func (e *Encoder[T]) FromRowInto(data []byte, out *T) (err error) { + defer func() { + if p := recover(); p != nil { + err = fmt.Errorf("row: corrupt row data: %v", p) + } + }() + return e.codec.readStruct(NewRow(e.codec.schema, data), reflect.ValueOf(out).Elem()) +} + +// Encode frames the row for cross-language exchange: an int64 +// little-endian schema hash followed by the row bytes, matching the +// Java and Python row encoders. +func (e *Encoder[T]) Encode(v *T) ([]byte, error) { + rowBytes, err := e.ToRow(v) + if err != nil { + return nil, err + } + out := make([]byte, 8+len(rowBytes)) + binary.LittleEndian.PutUint64(out, uint64(e.hash)) + copy(out[8:], rowBytes) + return out, nil +} + +// Decode verifies the schema hash and deserializes the row. +func (e *Encoder[T]) Decode(data []byte) (T, error) { + var out T + if len(data) < 8 { + return out, fmt.Errorf("row: encoded data of %d bytes is too short for the schema hash", len(data)) + } + hash := int64(binary.LittleEndian.Uint64(data)) + if hash != e.hash { + return out, fmt.Errorf("row: schema hash mismatch: data has %d, encoder expects %d; writer and reader struct definitions are out of sync", hash, e.hash) + } + err := e.FromRowInto(data[8:], &out) + return out, err +} + +// slotWriter is the shared write surface of RowWriter and ArrayWriter, +// letting one value codec serve struct fields and array elements. +type slotWriter interface { + SetNullAt(i int) + WriteBool(i int, v bool) + WriteInt8(i int, v int8) + WriteInt16(i int, v int16) + WriteInt32(i int, v int32) + WriteInt64(i int, v int64) + WriteFloat32(i int, v float32) + WriteFloat64(i int, v float64) + WriteDate(i int, d fory.Date) error + WriteTimestamp(i int, t time.Time) + WriteDuration(i int, d time.Duration) + WriteString(i int, s string) + WriteBytes(i int, b []byte) + SetOffsetAndSize(i, absStart, size int) +} + +// valueReader is the shared read surface of Row and ArrayData. +type valueReader interface { + IsNullAt(i int) bool + Bool(i int) bool + Int8(i int) int8 + Int16(i int) int16 + Int32(i int) int32 + Int64(i int) int64 + Float32(i int) float32 + Float64(i int) float64 + Date(i int) fory.Date + Timestamp(i int) time.Time + Duration(i int) time.Duration + String(i int) string + Binary(i int) []byte + Struct(i int) *Row + Array(i int) *ArrayData + Map(i int) *MapData +} + +// valueCodec reads or writes one value at slot/element i. Write +// functions handle nil for nilable Go types; read functions overwrite +// the target even when the field is null, so reused targets stay clean. +type valueCodec struct { + write func(w slotWriter, i int, v reflect.Value) error + read func(g valueReader, i int, v reflect.Value) error +} + +// structCodec writes and reads one struct level; nested structs hold +// their own child codec with a child RowWriter sharing the buffer. +type structCodec struct { + schema *Schema + writer *RowWriter + fields []fieldCodec +} + +type fieldCodec struct { + goIndex int + codec valueCodec +} + +func newStructCodec(t reflect.Type, buf *fory.ByteBuffer) (*structCodec, error) { + layout, err := inferStructLayout(t, nil) + if err != nil { + return nil, err + } + sc := &structCodec{ + schema: layout.schema, + writer: NewRowWriterWithBuffer(layout.schema, buf), + fields: make([]fieldCodec, 0, layout.schema.NumFields()), + } + for ordinal, goIndex := range layout.indexes { + codec, err := newValueCodec(t.Field(goIndex).Type, layout.schema.Field(ordinal).Type, buf) + if err != nil { + return nil, err + } + sc.fields = append(sc.fields, fieldCodec{goIndex: goIndex, codec: codec}) + } + return sc, nil +} + +func (sc *structCodec) writeStruct(v reflect.Value) error { + sc.writer.Reset() + for ordinal := range sc.fields { + fc := &sc.fields[ordinal] + if err := fc.codec.write(sc.writer, ordinal, v.Field(fc.goIndex)); err != nil { + return err + } + } + return nil +} + +func (sc *structCodec) readStruct(r *Row, v reflect.Value) error { + for ordinal := range sc.fields { + fc := &sc.fields[ordinal] + if err := fc.codec.read(r, ordinal, v.Field(fc.goIndex)); err != nil { + return err + } + } + return nil +} + +func newValueCodec(goType reflect.Type, dataType DataType, buf *fory.ByteBuffer) (valueCodec, error) { + if goType.Kind() == reflect.Ptr { + elemType := goType.Elem() + inner, err := newValueCodec(elemType, dataType, buf) + if err != nil { + return valueCodec{}, err + } + return valueCodec{ + write: func(w slotWriter, i int, v reflect.Value) error { + if v.IsNil() { + w.SetNullAt(i) + return nil + } + return inner.write(w, i, v.Elem()) + }, + read: func(g valueReader, i int, v reflect.Value) error { + if g.IsNullAt(i) { + v.Set(reflect.Zero(goType)) + return nil + } + p := reflect.New(elemType) + if err := inner.read(g, i, p.Elem()); err != nil { + return err + } + v.Set(p) + return nil + }, + }, nil + } + + switch dt := dataType.(type) { + case BoolType: + return valueCodec{ + write: func(w slotWriter, i int, v reflect.Value) error { w.WriteBool(i, v.Bool()); return nil }, + read: func(g valueReader, i int, v reflect.Value) error { v.SetBool(g.Bool(i)); return nil }, + }, nil + case Int8Type: + return valueCodec{ + write: func(w slotWriter, i int, v reflect.Value) error { w.WriteInt8(i, int8(v.Int())); return nil }, + read: func(g valueReader, i int, v reflect.Value) error { v.SetInt(int64(g.Int8(i))); return nil }, + }, nil + case Int16Type: + return valueCodec{ + write: func(w slotWriter, i int, v reflect.Value) error { w.WriteInt16(i, int16(v.Int())); return nil }, + read: func(g valueReader, i int, v reflect.Value) error { v.SetInt(int64(g.Int16(i))); return nil }, + }, nil + case Int32Type: + return valueCodec{ + write: func(w slotWriter, i int, v reflect.Value) error { w.WriteInt32(i, int32(v.Int())); return nil }, + read: func(g valueReader, i int, v reflect.Value) error { v.SetInt(int64(g.Int32(i))); return nil }, + }, nil + case Int64Type: + return valueCodec{ + write: func(w slotWriter, i int, v reflect.Value) error { w.WriteInt64(i, v.Int()); return nil }, + read: func(g valueReader, i int, v reflect.Value) error { v.SetInt(g.Int64(i)); return nil }, + }, nil + case Float32Type: + return valueCodec{ + write: func(w slotWriter, i int, v reflect.Value) error { w.WriteFloat32(i, float32(v.Float())); return nil }, + read: func(g valueReader, i int, v reflect.Value) error { v.SetFloat(float64(g.Float32(i))); return nil }, + }, nil + case Float64Type: + return valueCodec{ + write: func(w slotWriter, i int, v reflect.Value) error { w.WriteFloat64(i, v.Float()); return nil }, + read: func(g valueReader, i int, v reflect.Value) error { v.SetFloat(g.Float64(i)); return nil }, + }, nil + case Date32Type: + return valueCodec{ + write: func(w slotWriter, i int, v reflect.Value) error { return w.WriteDate(i, v.Interface().(fory.Date)) }, + read: func(g valueReader, i int, v reflect.Value) error { v.Set(reflect.ValueOf(g.Date(i))); return nil }, + }, nil + case TimestampType: + return valueCodec{ + write: func(w slotWriter, i int, v reflect.Value) error { + w.WriteTimestamp(i, v.Interface().(time.Time)) + return nil + }, + read: func(g valueReader, i int, v reflect.Value) error { v.Set(reflect.ValueOf(g.Timestamp(i))); return nil }, + }, nil + case DurationType: + return valueCodec{ + write: func(w slotWriter, i int, v reflect.Value) error { + w.WriteDuration(i, time.Duration(v.Int())) + return nil + }, + read: func(g valueReader, i int, v reflect.Value) error { v.SetInt(int64(g.Duration(i))); return nil }, + }, nil + case StringType: + return valueCodec{ + write: func(w slotWriter, i int, v reflect.Value) error { w.WriteString(i, v.String()); return nil }, + read: func(g valueReader, i int, v reflect.Value) error { v.SetString(g.String(i)); return nil }, + }, nil + case BinaryType: + return valueCodec{ + write: func(w slotWriter, i int, v reflect.Value) error { + if v.IsNil() { + w.SetNullAt(i) + return nil + } + w.WriteBytes(i, v.Bytes()) + return nil + }, + read: func(g valueReader, i int, v reflect.Value) error { + b := g.Binary(i) + if b == nil { + v.SetBytes(nil) + return nil + } + // Copy so the decoded struct never aliases the input. + owned := make([]byte, len(b)) + copy(owned, b) + v.SetBytes(owned) + return nil + }, + }, nil + case *ListType: + return newListCodec(goType, dt, buf) + case *MapType: + return newMapCodec(goType, dt, buf) + case *StructType: + child, err := newStructCodec(goType, buf) + if err != nil { + return valueCodec{}, err + } + return valueCodec{ + write: func(w slotWriter, i int, v reflect.Value) error { + start := child.writer.buf.WriterIndex() + if err := child.writeStruct(v); err != nil { + return err + } + w.SetOffsetAndSize(i, start, child.writer.buf.WriterIndex()-start) + return nil + }, + read: func(g valueReader, i int, v reflect.Value) error { + nested := g.Struct(i) + if nested == nil { + v.Set(reflect.Zero(goType)) + return nil + } + return child.readStruct(nested, v) + }, + }, nil + default: + return valueCodec{}, fmt.Errorf("row: data type %s is not supported by the encoder", dataType) + } +} + +func newListCodec(goType reflect.Type, listType *ListType, buf *fory.ByteBuffer) (valueCodec, error) { + elemCodec, err := newValueCodec(goType.Elem(), listType.Elem.Type, buf) + if err != nil { + return valueCodec{}, err + } + writer := NewArrayWriter(listType.Elem, buf) + return valueCodec{ + write: func(w slotWriter, i int, v reflect.Value) error { + if v.IsNil() { + w.SetNullAt(i) + return nil + } + n := v.Len() + start := buf.WriterIndex() + if err := writer.Reset(n); err != nil { + return err + } + for j := 0; j < n; j++ { + if err := elemCodec.write(writer, j, v.Index(j)); err != nil { + return err + } + } + w.SetOffsetAndSize(i, start, buf.WriterIndex()-start) + return nil + }, + read: func(g valueReader, i int, v reflect.Value) error { + arr := g.Array(i) + if arr == nil { + v.Set(reflect.Zero(goType)) + return nil + } + if err := arr.validateBounds(); err != nil { + return err + } + n := arr.NumElements() + s := reflect.MakeSlice(goType, n, n) + for j := 0; j < n; j++ { + if err := elemCodec.read(arr, j, s.Index(j)); err != nil { + return err + } + } + v.Set(s) + return nil + }, + }, nil +} + +func newMapCodec(goType reflect.Type, mapType *MapType, buf *fory.ByteBuffer) (valueCodec, error) { + keyCodec, err := newValueCodec(goType.Key(), mapType.Key.Type, buf) + if err != nil { + return valueCodec{}, err + } + valueCodecImpl, err := newValueCodec(goType.Elem(), mapType.Value.Type, buf) + if err != nil { + return valueCodec{}, err + } + mapWriter := NewMapWriter(buf) + keysWriter := NewArrayWriter(mapType.Key, buf) + valuesWriter := NewArrayWriter(mapType.Value, buf) + return valueCodec{ + write: func(w slotWriter, i int, v reflect.Value) error { + if v.IsNil() { + w.SetNullAt(i) + return nil + } + n := v.Len() + start := buf.WriterIndex() + mapWriter.Reset() + // Go map iteration order changes between iterations, so + // snapshot entries once to keep keys and values aligned. + keys := make([]reflect.Value, 0, n) + values := make([]reflect.Value, 0, n) + iter := v.MapRange() + for iter.Next() { + keys = append(keys, iter.Key()) + values = append(values, iter.Value()) + } + if err := keysWriter.Reset(len(keys)); err != nil { + return err + } + for j, k := range keys { + if err := keyCodec.write(keysWriter, j, k); err != nil { + return err + } + } + mapWriter.FinishKeys() + if err := valuesWriter.Reset(len(values)); err != nil { + return err + } + for j, val := range values { + if err := valueCodecImpl.write(valuesWriter, j, val); err != nil { + return err + } + } + w.SetOffsetAndSize(i, start, buf.WriterIndex()-start) + return nil + }, + read: func(g valueReader, i int, v reflect.Value) error { + md := g.Map(i) + if md == nil { + v.Set(reflect.Zero(goType)) + return nil + } + keys, values := md.Keys(), md.Values() + if err := keys.validateBounds(); err != nil { + return err + } + if err := values.validateBounds(); err != nil { + return err + } + n := keys.NumElements() + if values.NumElements() != n { + return fmt.Errorf("row: map has %d keys but %d values", n, values.NumElements()) + } + m := reflect.MakeMapWithSize(goType, n) + for j := 0; j < n; j++ { + key := reflect.New(goType.Key()).Elem() + if err := keyCodec.read(keys, j, key); err != nil { + return err + } + value := reflect.New(goType.Elem()).Elem() + if err := valueCodecImpl.read(values, j, value); err != nil { + return err + } + m.SetMapIndex(key, value) + } + v.Set(m) + return nil + }, + }, nil +} From 20b31f31ce73d321264278b7bf42c3aa4542e425 Mon Sep 17 00:00:00 2001 From: ayush00git Date: Thu, 30 Jul 2026 19:07:43 +0530 Subject: [PATCH 11/20] feat(test): add bytes <-> schema tests with Java interop --- go/fory/row/encoder_test.go | 177 ++++++++++++++++++++++++++++++++++++ go/fory/row/infer_test.go | 154 +++++++++++++++++++++++++++++++ 2 files changed, 331 insertions(+) create mode 100644 go/fory/row/encoder_test.go create mode 100644 go/fory/row/infer_test.go diff --git a/go/fory/row/encoder_test.go b/go/fory/row/encoder_test.go new file mode 100644 index 0000000000..ec000ef979 --- /dev/null +++ b/go/fory/row/encoder_test.go @@ -0,0 +1,177 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package row + +import ( + "encoding/binary" + "testing" + "time" + + fory "github.com/apache/fory/go/fory" + "github.com/stretchr/testify/require" +) + +type address struct { + City string + Zip int32 +} + +type person struct { + Id int64 + Name *string + Score float64 + Tags []string + Attrs map[string]int32 + Addr *address + Data []byte + Born fory.Date + At time.Time + Dur time.Duration +} + +func samplePerson() person { + name := "ayush" + return person{ + Id: 42, + Name: &name, + Score: 99.5, + Tags: []string{"go", "fory"}, + Attrs: map[string]int32{"a": 1, "b": 2}, + Addr: &address{City: "Delhi", Zip: 110001}, + Data: []byte{1, 2, 3}, + Born: fory.Date{Year: 2007, Month: time.June, Day: 1}, + At: time.UnixMicro(1_234_567_890_123_456), + Dur: 90 * time.Second, + } +} + +func TestEncoderRoundTrip(t *testing.T) { + enc, err := NewEncoder[person]() + require.NoError(t, err) + + original := samplePerson() + rowBytes, err := enc.ToRow(&original) + require.NoError(t, err) + + decoded, err := enc.FromRow(rowBytes) + require.NoError(t, err) + require.Equal(t, original, decoded) +} + +func TestEncoderNullsAndEmptiness(t *testing.T) { + enc, err := NewEncoder[person]() + require.NoError(t, err) + + // All nilable fields nil: they round-trip as nil, not zero values. + // Temporal fields stay valid; a zero fory.Date is rejected by + // WriteDate just like in the object-graph serializer. + original := samplePerson() + original.Name, original.Tags, original.Attrs, original.Addr, original.Data = nil, nil, nil, nil, nil + rowBytes, err := enc.ToRow(&original) + require.NoError(t, err) + decoded, err := enc.FromRow(rowBytes) + require.NoError(t, err) + require.Equal(t, original, decoded) + require.Nil(t, decoded.Name) + require.Nil(t, decoded.Tags) + require.Nil(t, decoded.Attrs) + + // Empty is distinct from nil. + original.Tags, original.Attrs, original.Data = []string{}, map[string]int32{}, []byte{} + rowBytes, err = enc.ToRow(&original) + require.NoError(t, err) + decoded, err = enc.FromRow(rowBytes) + require.NoError(t, err) + require.NotNil(t, decoded.Tags) + require.Empty(t, decoded.Tags) + require.NotNil(t, decoded.Attrs) + require.NotNil(t, decoded.Data) +} + +func TestEncoderDeepNesting(t *testing.T) { + type inner struct { + Vals []int64 + KV map[string][]int32 + } + type outer struct { + Rows []inner + Ptrs []*int32 + } + enc, err := NewEncoder[outer]() + require.NoError(t, err) + + three := int32(3) + original := outer{ + Rows: []inner{ + {Vals: []int64{1, 2}, KV: map[string][]int32{"x": {7, 8}}}, + {Vals: nil, KV: nil}, + }, + Ptrs: []*int32{&three, nil}, + } + rowBytes, err := enc.ToRow(&original) + require.NoError(t, err) + decoded, err := enc.FromRow(rowBytes) + require.NoError(t, err) + require.Equal(t, original, decoded) + require.Nil(t, decoded.Ptrs[1]) +} + +func TestEncodeDecodeFraming(t *testing.T) { + enc, err := NewEncoder[person]() + require.NoError(t, err) + original := samplePerson() + + encoded, err := enc.Encode(&original) + require.NoError(t, err) + require.Equal(t, uint64(enc.SchemaHash()), binary.LittleEndian.Uint64(encoded)) + require.Equal(t, ComputeSchemaHash(enc.Schema()), enc.SchemaHash()) + + decoded, err := enc.Decode(encoded) + require.NoError(t, err) + require.Equal(t, original, decoded) + + // Tampered hash is rejected with a descriptive error. + tampered := append([]byte(nil), encoded...) + tampered[0] ^= 0xFF + _, err = enc.Decode(tampered) + requireErrorContains(t, err, "hash mismatch") + + _, err = enc.Decode(encoded[:4]) + requireErrorContains(t, err, "too short") +} + +// Corrupt row bytes must produce errors, never panics. +func TestEncoderCorruptRowData(t *testing.T) { + enc, err := NewEncoder[person]() + require.NoError(t, err) + original := samplePerson() + rowBytes, err := enc.ToRow(&original) + require.NoError(t, err) + + for _, cut := range []int{1, 8, len(rowBytes) / 2, len(rowBytes) - 1} { + _, err := enc.FromRow(rowBytes[:cut]) + require.Error(t, err, "truncated to %d bytes", cut) + } +} + +func TestNewEncoderRejectsNonStruct(t *testing.T) { + _, err := NewEncoder[int]() + require.Error(t, err) + _, err = NewEncoder[map[string]int]() + require.Error(t, err) +} diff --git a/go/fory/row/infer_test.go b/go/fory/row/infer_test.go new file mode 100644 index 0000000000..cb7c44510f --- /dev/null +++ b/go/fory/row/infer_test.go @@ -0,0 +1,154 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package row + +import ( + "reflect" + "testing" + "time" + + fory "github.com/apache/fory/go/fory" + "github.com/stretchr/testify/require" +) + +func TestLowerCamelToLowerUnderscore(t *testing.T) { + cases := map[string]string{ + "id": "id", + "userName": "user_name", + "userID": "user_i_d", // Java quirk, kept for parity + "f2": "f2", + "aVeryLong": "a_very_long", + } + for in, want := range cases { + require.Equal(t, want, lowerCamelToLowerUnderscore(in), in) + } +} + +// Fields sort by lowerCamel name before snake_case conversion, +// matching Java's Descriptor ordering. +func TestInferFieldOrder(t *testing.T) { + type ordered struct { + ZItem int64 + AlphaBeta string + M2 int32 + } + s, err := InferSchema(reflect.TypeOf(ordered{})) + require.NoError(t, err) + require.Equal(t, 3, s.NumFields()) + require.Equal(t, "alpha_beta", s.Field(0).Name) + require.Equal(t, "m2", s.Field(1).Name) + require.Equal(t, "z_item", s.Field(2).Name) +} + +func TestInferTypeMapping(t *testing.T) { + type inner struct { + X float64 + } + type sample struct { + B bool + I8 int8 + I16 int16 + I32 int32 + I64 int64 + I int + F32 float32 + F64 float64 + S string + Bin []byte + L []int32 + M map[string]int64 + In inner + PI32 *int32 + PIn *inner + D fory.Date + T time.Time + Dur time.Duration + } + s, err := InferSchema(reflect.TypeOf(sample{})) + require.NoError(t, err) + + byName := func(name string) Field { return s.Field(s.FieldIndex(name)) } + + require.Equal(t, BoolType{}, byName("b").Type) + require.False(t, byName("b").Nullable) + require.Equal(t, Int8Type{}, byName("i8").Type) + require.Equal(t, Int16Type{}, byName("i16").Type) + require.Equal(t, Int32Type{}, byName("i32").Type) + require.Equal(t, Int64Type{}, byName("i64").Type) + require.Equal(t, Int64Type{}, byName("i").Type, "int maps to int64") + require.Equal(t, Float32Type{}, byName("f32").Type) + require.Equal(t, Float64Type{}, byName("f64").Type) + + require.Equal(t, StringType{}, byName("s").Type) + require.True(t, byName("s").Nullable) + require.Equal(t, BinaryType{}, byName("bin").Type) + + list := byName("l").Type.(*ListType) + require.Equal(t, Int32Type{}, list.Elem.Type) + require.False(t, list.Elem.Nullable, "value-type slice elements cannot be null") + require.True(t, byName("l").Nullable) + + m := byName("m").Type.(*MapType) + require.Equal(t, StringType{}, m.Key.Type) + require.False(t, m.Key.Nullable) + require.Equal(t, Int64Type{}, m.Value.Type) + + in := byName("in").Type.(*StructType) + require.Equal(t, "x", in.Fields[0].Name) + require.True(t, byName("in").Nullable) + + require.Equal(t, Int32Type{}, byName("p_i32").Type) + require.True(t, byName("p_i32").Nullable, "pointer fields are nullable") + require.True(t, byName("p_in").Nullable) + + require.Equal(t, Date32Type{}, byName("d").Type) + require.Equal(t, TimestampType{}, byName("t").Type) + require.Equal(t, DurationType{}, byName("dur").Type) +} + +func TestInferSkipsIgnoredAndUnexported(t *testing.T) { + type tagged struct { + A int64 + hidden int64 + Skipped string `fory:"ignore"` + } + s, err := InferSchema(reflect.TypeOf(tagged{})) + require.NoError(t, err) + require.Equal(t, 1, s.NumFields()) + require.Equal(t, "a", s.Field(0).Name) +} + +func TestInferRejectsUnsupportedTypes(t *testing.T) { + type node struct { + Next *node + } + cases := []any{ + struct{ U uint32 }{}, + struct{ C chan int }{}, + struct{ A [3]int32 }{}, + struct{ PP **int32 }{}, + struct{ M map[*string]int32 }{}, + node{}, + } + for _, c := range cases { + _, err := InferSchema(reflect.TypeOf(c)) + require.Error(t, err, "%T", c) + } + _, err := InferSchema(reflect.TypeOf(42)) + require.Error(t, err) +} From 56cfb7c3978de1c83f138add83ea7ac5b7b924e3 Mon Sep 17 00:00:00 2001 From: ayush00git Date: Thu, 30 Jul 2026 19:25:26 +0530 Subject: [PATCH 12/20] feat: add xlang tests to wrap with Java --- go/fory/tests/row_xlang/row_xlang_main.go | 158 ++++++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100644 go/fory/tests/row_xlang/row_xlang_main.go diff --git a/go/fory/tests/row_xlang/row_xlang_main.go b/go/fory/tests/row_xlang/row_xlang_main.go new file mode 100644 index 0000000000..f8491cdc17 --- /dev/null +++ b/go/fory/tests/row_xlang/row_xlang_main.go @@ -0,0 +1,158 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Row format cross-language peer driven by Java's +// org.apache.fory.format.GoCrossLanguageTest: each case reads bytes +// written by Java, verifies them field by field, re-encodes the same +// value, and overwrites the file for Java to check. +package main + +import ( + "bytes" + "fmt" + "os" + "reflect" + + "github.com/apache/fory/go/fory/row" +) + +// Mirrors org.apache.fory.format.CrossLanguageTest.A: pointer and +// value types are chosen so the inferred schema matches Java's +// (boxed Java types are nullable, so they map to Go pointers; string +// fields are nullable in both languages). +type a struct { + F1 *int32 + F2 map[string]string +} + +// Mirrors CrossLanguageTest.Bar. +type bar struct { + F1 *int32 + F2 string +} + +// Mirrors CrossLanguageTest.Foo. F3 uses []*string because the Java +// fixture contains a null list element. +type foo struct { + F1 *int32 + F2 string + F3 []*string + F4 map[string]*int32 + F5 *bar +} + +func main() { + if len(os.Args) < 3 { + fail("usage: row_xlang_bin ...") + } + switch caseName := os.Args[1]; caseName { + case "test_map_encoder": + testMapEncoder(os.Args[2]) + case "test_serialization_without_schema": + testSerializationWithoutSchema(os.Args[2]) + case "test_serialization_with_schema": + if len(os.Args) < 4 { + fail("test_serialization_with_schema needs ") + } + testSerializationWithSchema(os.Args[2], os.Args[3]) + default: + fail("unknown test case %q", caseName) + } +} + +func testMapEncoder(dataFile string) { + encoder, err := row.NewEncoder[a]() + must(err) + data, err := os.ReadFile(dataFile) + must(err) + + decoded, err := encoder.Decode(data) + must(err) + expected := a{F1: int32Ptr(1), F2: map[string]string{"pid": "12345", "ip": "0.0.0.0", "k1": "v1"}} + check(reflect.DeepEqual(decoded, expected), "decoded %+v, expected %+v", decoded, expected) + + encoded, err := encoder.Encode(&expected) + must(err) + must(os.WriteFile(dataFile, encoded, 0o644)) +} + +func testSerializationWithoutSchema(dataFile string) { + encoder, err := row.NewEncoder[foo]() + must(err) + data, err := os.ReadFile(dataFile) + must(err) + + decoded, err := encoder.FromRow(data) + must(err) + expected := expectedFoo() + check(reflect.DeepEqual(decoded, expected), "decoded %+v, expected %+v", decoded, expected) + + rowBytes, err := encoder.ToRow(&expected) + must(err) + must(os.WriteFile(dataFile, rowBytes, 0o644)) +} + +func testSerializationWithSchema(schemaFile, dataFile string) { + encoder, err := row.NewEncoder[foo]() + must(err) + schemaBytes, err := os.ReadFile(schemaFile) + must(err) + + parsed, err := row.SchemaFromBytes(schemaBytes) + must(err) + check(parsed.Equal(encoder.Schema()), "Java schema %v, inferred schema %v", parsed, encoder.Schema()) + check(row.ComputeSchemaHash(parsed) == encoder.SchemaHash(), + "schema hash %d, encoder hash %d", row.ComputeSchemaHash(parsed), encoder.SchemaHash()) + reencoded, err := row.SchemaToBytes(encoder.Schema()) + must(err) + check(bytes.Equal(reencoded, schemaBytes), "re-encoded schema bytes differ from Java's") + + testSerializationWithoutSchema(dataFile) +} + +func expectedFoo() foo { + return foo{ + F1: int32Ptr(1), + F2: "str", + F3: []*string{stringPtr("str1"), nil, stringPtr("str2")}, + F4: map[string]*int32{ + "k1": int32Ptr(1), "k2": int32Ptr(2), "k3": int32Ptr(3), + "k4": int32Ptr(4), "k5": int32Ptr(5), "k6": int32Ptr(6), + }, + F5: &bar{F1: int32Ptr(1), F2: "str"}, + } +} + +func int32Ptr(v int32) *int32 { return &v } +func stringPtr(s string) *string { return &s } + +func check(ok bool, format string, args ...any) { + if !ok { + fail(format, args...) + } +} + +func must(err error) { + if err != nil { + fail("%v", err) + } +} + +func fail(format string, args ...any) { + fmt.Fprintf(os.Stderr, "row_xlang: "+format+"\n", args...) + os.Exit(1) +} From f4c064adb6c4a9648517aae47eb2cdbc898acc36 Mon Sep 17 00:00:00 2001 From: ayush00git Date: Thu, 30 Jul 2026 19:26:30 +0530 Subject: [PATCH 13/20] feat: add cross-language tests with java --- .../fory/format/GoCrossLanguageTest.java | 142 ++++++++++++++++++ 1 file changed, 142 insertions(+) create mode 100644 java/fory-format/src/test/java/org/apache/fory/format/GoCrossLanguageTest.java diff --git a/java/fory-format/src/test/java/org/apache/fory/format/GoCrossLanguageTest.java b/java/fory-format/src/test/java/org/apache/fory/format/GoCrossLanguageTest.java new file mode 100644 index 0000000000..006a02a977 --- /dev/null +++ b/java/fory-format/src/test/java/org/apache/fory/format/GoCrossLanguageTest.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.format; + +import com.google.common.collect.ImmutableMap; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import org.apache.fory.format.encoder.Encoders; +import org.apache.fory.format.encoder.RowEncoder; +import org.apache.fory.format.row.binary.BinaryRow; +import org.apache.fory.format.type.DataTypes; +import org.apache.fory.memory.MemoryBuffer; +import org.apache.fory.memory.MemoryUtils; +import org.apache.fory.test.TestUtils; +import org.testng.Assert; +import org.testng.SkipException; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +/** + * Row format cross-language tests against a Go peer built from {@code go/fory/tests/row_xlang}. + * Data shapes are shared with {@link CrossLanguageTest} so the Java, Python, and Go peers exercise + * the same schemas. + */ +@Test +public class GoCrossLanguageTest { + private static final boolean IS_WINDOWS = + System.getProperty("os.name").toLowerCase().contains("windows"); + private static final String GO_BINARY = IS_WINDOWS ? "row_xlang_bin.exe" : "row_xlang_bin"; + + @BeforeClass + public void ensureGoPeerReady() { + String enabled = System.getenv("FORY_GO_JAVA_CI"); + if (!"1".equals(enabled)) { + throw new SkipException("Skipping GoCrossLanguageTest: FORY_GO_JAVA_CI not set to 1"); + } + boolean goInstalled = true; + try { + Process process = new ProcessBuilder("go", "version").start(); + if (process.waitFor() != 0) { + goInstalled = false; + } + } catch (IOException | InterruptedException e) { + goInstalled = false; + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + } + if (!goInstalled) { + throw new SkipException("Skipping GoCrossLanguageTest: go not installed"); + } + List buildCommand = + Arrays.asList("go", "build", "-o", "tests/" + GO_BINARY, "./tests/row_xlang"); + boolean buildSuccess = + TestUtils.executeCommand( + buildCommand, 120, Collections.emptyMap(), new File("../../go/fory")); + if (!buildSuccess || !new File("../../go/fory/tests/" + GO_BINARY).exists()) { + throw new SkipException("Skipping GoCrossLanguageTest: failed to build " + GO_BINARY); + } + } + + public void testMapEncoder() throws IOException { + CrossLanguageTest.A a = CrossLanguageTest.A.create(); + RowEncoder encoder = Encoders.bean(CrossLanguageTest.A.class); + Path dataFile = createTempFile("row_go_map"); + Files.write(dataFile, encoder.encode(a)); + Assert.assertTrue(runGoPeer("test_map_encoder", dataFile)); + Assert.assertEquals(encoder.decode(Files.readAllBytes(dataFile)), a); + } + + public void testSerializationWithoutSchema() throws IOException { + CrossLanguageTest.Foo foo = CrossLanguageTest.Foo.create(); + RowEncoder encoder = Encoders.bean(CrossLanguageTest.Foo.class); + Path dataFile = createTempFile("row_go_foo"); + Files.write(dataFile, encoder.toRow(foo).toBytes()); + Assert.assertTrue(runGoPeer("test_serialization_without_schema", dataFile)); + Assert.assertEquals(readFoo(encoder, dataFile), foo); + } + + public void testSerializationWithSchema() throws IOException { + CrossLanguageTest.Foo foo = CrossLanguageTest.Foo.create(); + RowEncoder encoder = Encoders.bean(CrossLanguageTest.Foo.class); + Path dataFile = createTempFile("row_go_foo"); + Path schemaFile = createTempFile("row_go_foo_schema"); + BinaryRow row = encoder.toRow(foo); + Files.write(dataFile, row.toBytes()); + Files.write(schemaFile, DataTypes.serializeSchema(row.getSchema())); + Assert.assertTrue(runGoPeer("test_serialization_with_schema", schemaFile, dataFile)); + Assert.assertEquals(readFoo(encoder, dataFile), foo); + } + + private static CrossLanguageTest.Foo readFoo( + RowEncoder encoder, Path dataFile) throws IOException { + MemoryBuffer buffer = MemoryUtils.wrap(Files.readAllBytes(dataFile)); + BinaryRow row = new BinaryRow(encoder.schema()); + row.pointTo(buffer, 0, buffer.size()); + return encoder.fromRow(row); + } + + private static Path createTempFile(String prefix) throws IOException { + Path file = Files.createTempFile(prefix, "data"); + file.toFile().deleteOnExit(); + return file; + } + + private boolean runGoPeer(String caseName, Path... files) { + List command = new ArrayList<>(); + command.add(IS_WINDOWS ? GO_BINARY : "./" + GO_BINARY); + command.add(caseName); + for (Path file : files) { + command.add(file.toAbsolutePath().toString()); + } + return TestUtils.executeCommand( + command, + 30, + ImmutableMap.of("ENABLE_CROSS_LANGUAGE_TESTS", "true"), + new File("../../go/fory/tests")); + } +} From 62321e9c8622cc4a07fb70f11aa265598cb99d4f Mon Sep 17 00:00:00 2001 From: ayush00git Date: Thu, 30 Jul 2026 19:27:28 +0530 Subject: [PATCH 14/20] feat: wire ci and update gitignore --- .github/workflows/ci.yml | 2 ++ .gitignore | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 450bd1a732..55efa183fb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1607,6 +1607,8 @@ jobs: mvn -T16 --no-transfer-progress clean install -DskipTests -Dmaven.javadoc.skip=true -Dmaven.source.skip=true cd fory-core mvn -T16 --no-transfer-progress test -Dtest=org.apache.fory.xlang.GoXlangTest + cd ../fory-format + mvn -T16 --no-transfer-progress test -Dtest=org.apache.fory.format.GoCrossLanguageTest - name: Run Go IDL Tests run: ./integration_tests/idl_tests/run_go_tests.sh diff --git a/.gitignore b/.gitignore index c543b379a8..013f108177 100644 --- a/.gitignore +++ b/.gitignore @@ -59,6 +59,8 @@ go/fory/tests/xlang_test_main go/fory/tests/xlang/xlang_test_main go/fory/xlang_test_main go/fory/xlang +go/fory/tests/row_xlang_bin +go/fory/tests/row_xlang_bin.exe # Build directories for all languages # Java From 658b794c4bd36c6eb58560ca02c254f666dc3853 Mon Sep 17 00:00:00 2001 From: ayush00git Date: Thu, 30 Jul 2026 20:02:10 +0530 Subject: [PATCH 15/20] fix: trucation of large dt and add ignore fields --- go/fory/row/array_test.go | 18 ++++++++++++++++ go/fory/row/encoder.go | 20 +++++++---------- go/fory/row/infer.go | 36 ++++++++++++++++++++++++++----- go/fory/row/infer_test.go | 14 +++++++++++- go/fory/row/row.go | 11 ++++++++-- go/fory/row/row_test.go | 12 +++++++++++ go/fory/row/schema_bytes.go | 14 +++++++++--- go/fory/row/schema_bytes_test.go | 13 +++++++++++ go/fory/row/writer.go | 37 +++++++++++++++++++++----------- 9 files changed, 139 insertions(+), 36 deletions(-) diff --git a/go/fory/row/array_test.go b/go/fory/row/array_test.go index 9ede434760..8808a1bdb8 100644 --- a/go/fory/row/array_test.go +++ b/go/fory/row/array_test.go @@ -18,6 +18,8 @@ package row import ( + "encoding/binary" + "math" "testing" fory "github.com/apache/fory/go/fory" @@ -106,3 +108,19 @@ func TestArrayResetRejectsInvalidLength(t *testing.T) { require.Error(t, w.Reset(-1)) require.Error(t, w.Reset(1<<40)) } + +// Forged element counts must fail validation even when the arithmetic +// would overflow int64 and wrap the required size negative. +func TestArrayValidateBoundsRejectsForgedCounts(t *testing.T) { + for _, count := range []uint64{ + math.MaxUint64, // negative after int conversion + math.MaxInt64, // bitmap width and product overflow + uint64(math.MaxInt64/8) + 1, // product overflow with 8-byte elements + 1 << 32, // plausible-looking but far beyond the data + } { + data := make([]byte, 16) + binary.LittleEndian.PutUint64(data, count) + a := NewArrayData(List(Int64Type{}).Elem, data) + require.Error(t, a.validateBounds(), "count %d", count) + } +} diff --git a/go/fory/row/encoder.go b/go/fory/row/encoder.go index fd5703ae49..5e0f793a17 100644 --- a/go/fory/row/encoder.go +++ b/go/fory/row/encoder.go @@ -127,9 +127,9 @@ type slotWriter interface { WriteDate(i int, d fory.Date) error WriteTimestamp(i int, t time.Time) WriteDuration(i int, d time.Duration) - WriteString(i int, s string) - WriteBytes(i int, b []byte) - SetOffsetAndSize(i, absStart, size int) + WriteString(i int, s string) error + WriteBytes(i int, b []byte) error + SetOffsetAndSize(i, absStart, size int) error } // valueReader is the shared read surface of Row and ArrayData. @@ -303,7 +303,7 @@ func newValueCodec(goType reflect.Type, dataType DataType, buf *fory.ByteBuffer) }, nil case StringType: return valueCodec{ - write: func(w slotWriter, i int, v reflect.Value) error { w.WriteString(i, v.String()); return nil }, + write: func(w slotWriter, i int, v reflect.Value) error { return w.WriteString(i, v.String()) }, read: func(g valueReader, i int, v reflect.Value) error { v.SetString(g.String(i)); return nil }, }, nil case BinaryType: @@ -313,8 +313,7 @@ func newValueCodec(goType reflect.Type, dataType DataType, buf *fory.ByteBuffer) w.SetNullAt(i) return nil } - w.WriteBytes(i, v.Bytes()) - return nil + return w.WriteBytes(i, v.Bytes()) }, read: func(g valueReader, i int, v reflect.Value) error { b := g.Binary(i) @@ -344,8 +343,7 @@ func newValueCodec(goType reflect.Type, dataType DataType, buf *fory.ByteBuffer) if err := child.writeStruct(v); err != nil { return err } - w.SetOffsetAndSize(i, start, child.writer.buf.WriterIndex()-start) - return nil + return w.SetOffsetAndSize(i, start, child.writer.buf.WriterIndex()-start) }, read: func(g valueReader, i int, v reflect.Value) error { nested := g.Struct(i) @@ -383,8 +381,7 @@ func newListCodec(goType reflect.Type, listType *ListType, buf *fory.ByteBuffer) return err } } - w.SetOffsetAndSize(i, start, buf.WriterIndex()-start) - return nil + return w.SetOffsetAndSize(i, start, buf.WriterIndex()-start) }, read: func(g valueReader, i int, v reflect.Value) error { arr := g.Array(i) @@ -455,8 +452,7 @@ func newMapCodec(goType reflect.Type, mapType *MapType, buf *fory.ByteBuffer) (v return err } } - w.SetOffsetAndSize(i, start, buf.WriterIndex()-start) - return nil + return w.SetOffsetAndSize(i, start, buf.WriterIndex()-start) }, read: func(g valueReader, i int, v reflect.Value) error { md := g.Map(i) diff --git a/go/fory/row/infer.go b/go/fory/row/infer.go index a2daad1cb6..109ae74129 100644 --- a/go/fory/row/infer.go +++ b/go/fory/row/infer.go @@ -87,7 +87,14 @@ func inferStructLayout(t reflect.Type, path []reflect.Type) (*structLayout, erro var members []member for i := 0; i < t.NumField(); i++ { f := t.Field(i) - if f.PkgPath != "" || hasIgnoreTag(f.Tag.Get("fory")) { + if f.PkgPath != "" { + continue + } + ignored, err := hasIgnoreTag(f.Tag.Get("fory")) + if err != nil { + return nil, fmt.Errorf("%w (field %s of %v)", err, f.Name, t) + } + if ignored { continue } members = append(members, member{lowerFirst(f.Name), i, f.Type}) @@ -185,13 +192,32 @@ func inferField(name string, t reflect.Type, path []reflect.Type) (Field, error) } } -func hasIgnoreTag(tag string) bool { +// hasIgnoreTag mirrors the ignore semantics of the core fory tag +// parser (parseFieldTag in field_spec.go): a whole tag of "-" or an +// ignore/ignore=true part skips the field, ignore=false keeps it, and +// any other ignore value is an error. Other tag keys are not used by +// the row format. +func hasIgnoreTag(tag string) (bool, error) { + if tag == "-" { + return true, nil + } + ignore := false for _, part := range strings.Split(tag, ",") { - if strings.TrimSpace(part) == "ignore" { - return true + part = strings.TrimSpace(part) + if part == "ignore" { + ignore = true + } else if strings.HasPrefix(part, "ignore=") { + switch strings.TrimPrefix(part, "ignore=") { + case "true": + ignore = true + case "false": + ignore = false + default: + return false, fmt.Errorf("row: invalid ignore value in fory tag %q", tag) + } } } - return false + return ignore, nil } func lowerFirst(s string) string { diff --git a/go/fory/row/infer_test.go b/go/fory/row/infer_test.go index cb7c44510f..d62513eb07 100644 --- a/go/fory/row/infer_test.go +++ b/go/fory/row/infer_test.go @@ -121,16 +121,28 @@ func TestInferTypeMapping(t *testing.T) { require.Equal(t, DurationType{}, byName("dur").Type) } +// Ignore semantics must match the core fory tag parser: "-", +// "ignore", and "ignore=true" skip; "ignore=false" keeps. func TestInferSkipsIgnoredAndUnexported(t *testing.T) { type tagged struct { A int64 hidden int64 Skipped string `fory:"ignore"` + Dash string `fory:"-"` + True string `fory:"ignore=true"` + Kept int32 `fory:"ignore=false"` } s, err := InferSchema(reflect.TypeOf(tagged{})) require.NoError(t, err) - require.Equal(t, 1, s.NumFields()) + require.Equal(t, 2, s.NumFields()) require.Equal(t, "a", s.Field(0).Name) + require.Equal(t, "kept", s.Field(1).Name) + + type badTag struct { + A int64 `fory:"ignore=yes"` + } + _, err = InferSchema(reflect.TypeOf(badTag{})) + requireErrorContains(t, err, "invalid ignore value") } func TestInferRejectsUnsupportedTypes(t *testing.T) { diff --git a/go/fory/row/row.go b/go/fory/row/row.go index 1fedf6ad38..77e03d819a 100644 --- a/go/fory/row/row.go +++ b/go/fory/row/row.go @@ -201,10 +201,17 @@ func (a *ArrayData) SizeBytes() int { return len(a.data) } // covered by the available bytes, so decoders can check before // allocating from an attacker-declared count. func (a *ArrayData) validateBounds() error { - need := int64(a.headerBytes) + int64(a.numElements)*int64(a.elemSize) - if a.numElements < 0 || need > int64(len(a.data)) { + // The coarse count check involves no arithmetic, so it cannot be + // defeated by overflow: every element occupies at least one data + // byte. It also bounds numElements low enough that headerBytes + // (computed in NewArrayData) and the product below are exact. + if a.numElements < 0 || a.numElements > len(a.data) { return fmt.Errorf("row: array declares %d elements but holds only %d bytes", a.numElements, len(a.data)) } + need := int64(a.headerBytes) + int64(a.numElements)*int64(a.elemSize) + if need > int64(len(a.data)) { + return fmt.Errorf("row: array declares %d elements needing %d bytes but holds only %d", a.numElements, need, len(a.data)) + } return nil } diff --git a/go/fory/row/row_test.go b/go/fory/row/row_test.go index 55f124a104..428b69449f 100644 --- a/go/fory/row/row_test.go +++ b/go/fory/row/row_test.go @@ -242,6 +242,18 @@ func TestOutOfRangeIndexPanics(t *testing.T) { require.Panics(t, func() { r.IsNullAt(-1) }) } +// The wire format packs offset and size into 32 bits each; larger +// values must be rejected, never silently truncated. +func TestOffsetAndSizeRejectWireLimit(t *testing.T) { + w := NewRowWriter(int64StringSchema()) + w.Reset() + require.Error(t, w.SetOffsetAndSize(1, w.Buffer().WriterIndex(), 1<<33)) + + aw := NewArrayWriter(List(StringType{}).Elem, w.Buffer()) + require.NoError(t, aw.Reset(1)) + require.Error(t, aw.SetOffsetAndSize(0, w.Buffer().WriterIndex(), 1<<33)) +} + func TestConcurrentReads(t *testing.T) { w := NewRowWriter(int64StringSchema()) w.Reset() diff --git a/go/fory/row/schema_bytes.go b/go/fory/row/schema_bytes.go index 0150789c7f..0eda14b191 100644 --- a/go/fory/row/schema_bytes.go +++ b/go/fory/row/schema_bytes.go @@ -39,6 +39,8 @@ import ( const ( schemaVersion = 1 fieldNameSizeThreshold = 15 + // Caps readField/readType recursion. + maxSchemaNestingDepth = 64 ) // Header bits 0-1 store the INDEX into this table (matching Java @@ -178,14 +180,20 @@ func SchemaFromBytes(data []byte) (*Schema, error) { } type schemaReader struct { - buf *fory.ByteBuffer - size int - err fory.Error + buf *fory.ByteBuffer + size int + depth int + err fory.Error } func (r *schemaReader) remaining() int { return r.size - r.buf.ReaderIndex() } func (r *schemaReader) readField() (Field, error) { + if r.depth >= maxSchemaNestingDepth { + return Field{}, fmt.Errorf("row: schema nesting exceeds %d levels", maxSchemaNestingDepth) + } + r.depth++ + defer func() { r.depth-- }() header := int(r.buf.ReadUint8(&r.err)) if err := r.err.CheckError(); err != nil { return Field{}, err diff --git a/go/fory/row/schema_bytes_test.go b/go/fory/row/schema_bytes_test.go index 3c790f5128..61da4a3a14 100644 --- a/go/fory/row/schema_bytes_test.go +++ b/go/fory/row/schema_bytes_test.go @@ -159,3 +159,16 @@ func TestSchemaToBytesRejectsEmptyName(t *testing.T) { _, err := SchemaToBytes(NewSchema([]Field{NewField("", Int32Type{}, true)})) requireErrorContains(t, err, "empty name") } + +// A crafted chain of nested LIST type-infos costs ~3 bytes per level; +// without a depth limit it would fatally overflow the goroutine stack. +func TestSchemaFromBytesRejectsDeepNesting(t *testing.T) { + data := []byte{0x01, 0x01} + for i := 0; i < 10*maxSchemaNestingDepth; i++ { + // field: header (UTF_8 encoding, 1-byte name, non-null), + // name "a", type LIST -> recurses into the next field. + data = append(data, 0x00, 'a', 0x16) + } + _, err := SchemaFromBytes(data) + requireErrorContains(t, err, "nesting") +} diff --git a/go/fory/row/writer.go b/go/fory/row/writer.go index 9b612674c9..dd898644de 100644 --- a/go/fory/row/writer.go +++ b/go/fory/row/writer.go @@ -159,22 +159,28 @@ func (w *RowWriter) WriteDuration(i int, d time.Duration) { w.WriteInt64(i, d.Microseconds()) } -func (w *RowWriter) WriteString(i int, s string) { +func (w *RowWriter) WriteString(i int, s string) error { start := appendStringRegion(w.buf, s) - w.SetOffsetAndSize(i, start, len(s)) + return w.SetOffsetAndSize(i, start, len(s)) } -func (w *RowWriter) WriteBytes(i int, b []byte) { +func (w *RowWriter) WriteBytes(i int, b []byte) error { start := appendBytesRegion(w.buf, b) - w.SetOffsetAndSize(i, start, len(b)) + return w.SetOffsetAndSize(i, start, len(b)) } // SetOffsetAndSize patches field i's slot with the row-relative offset // and byte size of a value already appended to the variable data region. // Use it after writing a nested struct, array, or map at absStart. -func (w *RowWriter) SetOffsetAndSize(i, absStart, size int) { +// Offsets and sizes beyond 32 bits are rejected: the wire format packs +// both into one slot, and truncating would corrupt the row silently. +func (w *RowWriter) SetOffsetAndSize(i, absStart, size int) error { rel := absStart - w.base - binary.LittleEndian.PutUint64(w.buf.GetData()[w.slot(i):], uint64(rel)<<32|uint64(uint32(size))) + if uint64(rel) > math.MaxUint32 || uint64(size) > math.MaxUint32 { + return fmt.Errorf("row: value at offset %d with %d bytes exceeds the 32-bit wire format limit", rel, size) + } + binary.LittleEndian.PutUint64(w.buf.GetData()[w.slot(i):], uint64(rel)<<32|uint64(size)) + return nil } // ArrayWriter writes one array: an 8-byte element count, a null bitmap, @@ -295,21 +301,26 @@ func (w *ArrayWriter) WriteDuration(i int, d time.Duration) { w.WriteInt64(i, d.Microseconds()) } -func (w *ArrayWriter) WriteString(i int, s string) { +func (w *ArrayWriter) WriteString(i int, s string) error { start := appendStringRegion(w.buf, s) - w.SetOffsetAndSize(i, start, len(s)) + return w.SetOffsetAndSize(i, start, len(s)) } -func (w *ArrayWriter) WriteBytes(i int, b []byte) { +func (w *ArrayWriter) WriteBytes(i int, b []byte) error { start := appendBytesRegion(w.buf, b) - w.SetOffsetAndSize(i, start, len(b)) + return w.SetOffsetAndSize(i, start, len(b)) } // SetOffsetAndSize patches element i's slot with the array-relative -// offset and byte size of a value already appended after the array. -func (w *ArrayWriter) SetOffsetAndSize(i, absStart, size int) { +// offset and byte size of a value already appended after the array, +// with the same 32-bit wire limit as RowWriter.SetOffsetAndSize. +func (w *ArrayWriter) SetOffsetAndSize(i, absStart, size int) error { rel := absStart - w.base - binary.LittleEndian.PutUint64(w.buf.GetData()[w.slot(i):], uint64(rel)<<32|uint64(uint32(size))) + if uint64(rel) > math.MaxUint32 || uint64(size) > math.MaxUint32 { + return fmt.Errorf("row: value at offset %d with %d bytes exceeds the 32-bit wire format limit", rel, size) + } + binary.LittleEndian.PutUint64(w.buf.GetData()[w.slot(i):], uint64(rel)<<32|uint64(size)) + return nil } // MapWriter writes one map: an 8-byte keys-array size, the keys array, From 4aae1717337733a88d38bc5b349501e45c96e550 Mon Sep 17 00:00:00 2001 From: ayush00git Date: Sat, 22 Aug 2026 22:47:12 +0530 Subject: [PATCH 16/20] refactor(go): copy row schema fields and resolve duplicate names like Java NewSchema and Struct now own a copy of their input slice so later mutation of the caller's fields cannot desynchronize cached name lookups. Duplicate field names resolve to the last occurrence, matching Java's Schema so identical schema bytes resolve names to the same ordinal in both runtimes. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01MwqYMrDU1kD1wcB9snj9mh --- go/fory/row/datatype.go | 21 +++++++++++---------- go/fory/row/schema_test.go | 19 +++++++++++++++++++ 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/go/fory/row/datatype.go b/go/fory/row/datatype.go index 89cc7e70cc..d53ef34fb2 100644 --- a/go/fory/row/datatype.go +++ b/go/fory/row/datatype.go @@ -218,8 +218,9 @@ func Map(key, value DataType) *MapType { } } +// Struct returns a struct type over a copy of fields. func Struct(fields []Field) *StructType { - return &StructType{Fields: fields} + return &StructType{Fields: append([]Field(nil), fields...)} } func dataTypeEqual(a, b DataType) bool { @@ -254,17 +255,17 @@ type Schema struct { byName map[string]int } -// NewSchema builds a schema from fields in their schema-declared order, -// which fixes the field slot layout. For duplicate names, FieldIndex -// resolves to the first occurrence. +// NewSchema builds a schema over a copy of fields in their declared +// order, which fixes the field slot layout. For duplicate names, +// FieldIndex resolves to the last occurrence, matching Java's Schema so +// identical schema bytes resolve names identically in both runtimes. func NewSchema(fields []Field) *Schema { - byName := make(map[string]int, len(fields)) - for i, f := range fields { - if _, ok := byName[f.Name]; !ok { - byName[f.Name] = i - } + owned := append([]Field(nil), fields...) + byName := make(map[string]int, len(owned)) + for i, f := range owned { + byName[f.Name] = i } - return &Schema{fields: fields, byName: byName} + return &Schema{fields: owned, byName: byName} } func (s *Schema) NumFields() int { return len(s.fields) } diff --git a/go/fory/row/schema_test.go b/go/fory/row/schema_test.go index 2fc8691579..32042e7e8a 100644 --- a/go/fory/row/schema_test.go +++ b/go/fory/row/schema_test.go @@ -107,6 +107,25 @@ func TestSchemaLookup(t *testing.T) { require.Equal(t, "id", s.Field(0).Name) require.Equal(t, 1, s.FieldIndex("name")) require.Equal(t, -1, s.FieldIndex("missing")) + + // Duplicate names resolve to the last occurrence, as in Java. + dup := NewSchema([]Field{ + NewField("x", Int32Type{}, false), + NewField("x", StringType{}, true), + }) + require.Equal(t, 1, dup.FieldIndex("x")) +} + +// Schemas and struct types own a copy of their fields, so later +// mutation of the caller's slice cannot desynchronize name lookups. +func TestSchemaCopiesFields(t *testing.T) { + fields := []Field{NewField("a", Int32Type{}, false)} + s := NewSchema(fields) + st := Struct(fields) + fields[0].Name = "changed" + require.Equal(t, "a", s.Field(0).Name) + require.Equal(t, 0, s.FieldIndex("a")) + require.Equal(t, "a", st.Fields[0].Name) } func TestSchemaEqual(t *testing.T) { From be4724b7d81fcb82955c77cb03842baa5f944905 Mon Sep 17 00:00:00 2001 From: ayush00git Date: Sat, 22 Aug 2026 22:47:12 +0530 Subject: [PATCH 17/20] fix(go): validate row schemas before encoding and harden schema parsing SchemaToBuffer now validates the whole schema before writing a byte: nesting within the reader's depth limit, no composite type reachable from itself, decimal parameters within the wire format's single byte, and only DataType implementations the format defines. Previously the writer could emit schemas its own reader rejects or recurse forever on a self-referential type. The parser compares wire counts and name sizes against the remaining bytes before narrowing them to int, so large values cannot wrap negative on 32-bit platforms. Schema bytes are documented as trusted input; the remaining-byte checks keep malformed schemas failing fast rather than promising untrusted-input safety. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01MwqYMrDU1kD1wcB9snj9mh --- go/fory/row/schema_bytes.go | 112 ++++++++++++++++++++++++------- go/fory/row/schema_bytes_test.go | 47 +++++++++++++ 2 files changed, 136 insertions(+), 23 deletions(-) diff --git a/go/fory/row/schema_bytes.go b/go/fory/row/schema_bytes.go index 0eda14b191..f2279cb3ec 100644 --- a/go/fory/row/schema_bytes.go +++ b/go/fory/row/schema_bytes.go @@ -76,8 +76,18 @@ func SchemaToBytes(s *Schema) ([]byte, error) { return buf.GetByteSlice(0, buf.WriterIndex()), nil } -// SchemaToBuffer serializes a schema into an existing buffer. +// SchemaToBuffer serializes a schema into an existing buffer. The +// schema is validated in full before any byte is written, so a +// rejected schema leaves the buffer untouched. func SchemaToBuffer(s *Schema, buf *fory.ByteBuffer) error { + if s == nil { + return fmt.Errorf("row: cannot serialize a nil schema") + } + for _, f := range s.Fields() { + if err := validateSchemaField(f, nil, 0); err != nil { + return err + } + } buf.WriteByte_(schemaVersion) buf.WriteVarUint32Small7(uint32(s.NumFields())) for _, f := range s.Fields() { @@ -88,10 +98,60 @@ func SchemaToBuffer(s *Schema, buf *fory.ByteBuffer) error { return nil } -func writeSchemaField(buf *fory.ByteBuffer, f Field) error { +// validateSchemaField checks what the wire format can express before +// writing: nesting within maxSchemaNestingDepth (so the reader accepts +// the output), no composite type reachable from itself, decimal +// parameters within one byte, and only the DataType implementations +// the format defines. A depth of d means the field sits d levels below +// the top-level schema, matching the reader's depth counter. +func validateSchemaField(f Field, path []DataType, depth int) error { + if depth >= maxSchemaNestingDepth { + return fmt.Errorf("row: schema nesting exceeds %d levels", maxSchemaNestingDepth) + } if f.Name == "" { return fmt.Errorf("row: schema field with empty name") } + switch t := f.Type.(type) { + case BoolType, Int8Type, Int16Type, Int32Type, Int64Type, Float16Type, Float32Type, + Float64Type, StringType, BinaryType, Date32Type, TimestampType, DurationType: + return nil + case DecimalType: + if t.Precision < 0 || t.Precision > math.MaxUint8 || t.Scale < 0 || t.Scale > math.MaxUint8 { + return fmt.Errorf("row: decimal(%d, %d) in field %q is outside the wire format's single-byte range", t.Precision, t.Scale, f.Name) + } + return nil + case *ListType, *MapType, *StructType: + for _, seen := range path { + if seen == f.Type { + return fmt.Errorf("row: schema type %T in field %q refers to itself", f.Type, f.Name) + } + } + path = append(path, f.Type) + switch t := t.(type) { + case *ListType: + return validateSchemaField(t.Elem, path, depth+1) + case *MapType: + if err := validateSchemaField(t.Key, path, depth+1); err != nil { + return err + } + return validateSchemaField(t.Value, path, depth+1) + case *StructType: + for _, child := range t.Fields { + if err := validateSchemaField(child, path, depth+1); err != nil { + return err + } + } + return nil + } + return nil + case nil: + return fmt.Errorf("row: schema field %q has no type", f.Name) + default: + return fmt.Errorf("row: schema field %q uses %T, which the wire format does not define (use DecimalType by value)", f.Name, f.Type) + } +} + +func writeSchemaField(buf *fory.ByteBuffer, f Field) error { encoding := fieldNameEncoder.ComputeEncodingWith(f.Name, fieldNameEncodings[:]) metaString, err := fieldNameEncoder.EncodeWithEncoding(f.Name, encoding) if err != nil { @@ -147,8 +207,9 @@ func writeSchemaType(buf *fory.ByteBuffer, dataType DataType) error { } // SchemaFromBytes deserializes a schema from the cross-language wire -// format. It is safe on untrusted input: declared sizes are checked -// against remaining bytes before any allocation. +// format. Schema bytes are trusted input, like the row format itself; +// declared counts are still bounded by the remaining bytes so +// malformed schemas fail fast instead of over-allocating. func SchemaFromBytes(data []byte) (*Schema, error) { r := &schemaReader{buf: fory.NewByteBuffer(data), size: len(data)} version := r.buf.ReadUint8(&r.err) @@ -158,16 +219,10 @@ func SchemaFromBytes(data []byte) (*Schema, error) { if version != schemaVersion { return nil, fmt.Errorf("row: unsupported schema version %d, expected %d", version, schemaVersion) } - numFields := int(r.buf.ReadVarUint32Small7(&r.err)) - if err := r.err.CheckError(); err != nil { + numFields, err := r.readCount("schema") + if err != nil { return nil, err } - // Every field costs at least one byte, so a declared count larger - // than the remaining input is corrupt; checking before the - // allocation below keeps attacker-declared counts harmless. - if numFields > r.remaining() { - return nil, fmt.Errorf("row: schema declares %d fields but only %d bytes remain", numFields, r.remaining()) - } fields := make([]Field, 0, numFields) for i := 0; i < numFields; i++ { f, err := r.readField() @@ -188,6 +243,21 @@ type schemaReader struct { func (r *schemaReader) remaining() int { return r.size - r.buf.ReaderIndex() } +// readCount reads a field count and bounds it by the remaining bytes +// (every field costs at least one byte) BEFORE narrowing to int, so a +// large wire value cannot wrap negative on 32-bit platforms or drive an +// oversized allocation. +func (r *schemaReader) readCount(owner string) (int, error) { + count := r.buf.ReadVarUint32Small7(&r.err) + if err := r.err.CheckError(); err != nil { + return 0, err + } + if uint64(count) > uint64(r.remaining()) { + return 0, fmt.Errorf("row: %s declares %d fields but only %d bytes remain", owner, count, r.remaining()) + } + return int(count), nil +} + func (r *schemaReader) readField() (Field, error) { if r.depth >= maxSchemaNestingDepth { return Field{}, fmt.Errorf("row: schema nesting exceeds %d levels", maxSchemaNestingDepth) @@ -205,18 +275,17 @@ func (r *schemaReader) readField() (Field, error) { nameSizeMinus1 := (header >> 2) & 0x0F nullable := header&0x40 != 0 - var nameSize int + nameSize64 := uint64(nameSizeMinus1 + 1) if nameSizeMinus1 == fieldNameSizeThreshold { - nameSize = int(r.buf.ReadVarUint32Small7(&r.err)) + fieldNameSizeThreshold - } else { - nameSize = nameSizeMinus1 + 1 + nameSize64 = uint64(r.buf.ReadVarUint32Small7(&r.err)) + fieldNameSizeThreshold } if err := r.err.CheckError(); err != nil { return Field{}, err } - if nameSize > r.remaining() { - return Field{}, fmt.Errorf("row: field name of %d bytes exceeds %d remaining", nameSize, r.remaining()) + if nameSize64 > uint64(r.remaining()) { + return Field{}, fmt.Errorf("row: field name of %d bytes exceeds %d remaining", nameSize64, r.remaining()) } + nameSize := int(nameSize64) nameBytes := r.buf.ReadBinary(nameSize, &r.err) if err := r.err.CheckError(); err != nil { return Field{}, err @@ -290,13 +359,10 @@ func (r *schemaReader) readType() (DataType, error) { // canonical key/value names and nullability. return Map(keyField.Type, valueField.Type), nil case fory.STRUCT: - numFields := int(r.buf.ReadVarUint32Small7(&r.err)) - if err := r.err.CheckError(); err != nil { + numFields, err := r.readCount("struct") + if err != nil { return nil, err } - if numFields > r.remaining() { - return nil, fmt.Errorf("row: struct declares %d fields but only %d bytes remain", numFields, r.remaining()) - } fields := make([]Field, 0, numFields) for i := 0; i < numFields; i++ { f, err := r.readField() diff --git a/go/fory/row/schema_bytes_test.go b/go/fory/row/schema_bytes_test.go index 61da4a3a14..9521412be6 100644 --- a/go/fory/row/schema_bytes_test.go +++ b/go/fory/row/schema_bytes_test.go @@ -21,6 +21,7 @@ import ( "fmt" "testing" + fory "github.com/apache/fory/go/fory" "github.com/stretchr/testify/require" ) @@ -172,3 +173,49 @@ func TestSchemaFromBytesRejectsDeepNesting(t *testing.T) { _, err := SchemaFromBytes(data) requireErrorContains(t, err, "nesting") } + +type unknownDataType struct{} + +func (unknownDataType) TypeID() fory.TypeId { return 99 } +func (unknownDataType) ByteWidth() int { return -1 } +func (unknownDataType) String() string { return "unknown" } + +// The writer validates everything the wire format cannot express +// before emitting a byte, so its output is always readable. +func TestSchemaToBytesRejectsUnwritableSchemas(t *testing.T) { + _, err := SchemaToBytes(nil) + require.Error(t, err) + + // One level deeper than the reader accepts. + var deep DataType = Int32Type{} + for i := 0; i < maxSchemaNestingDepth; i++ { + deep = List(deep) + } + _, err = SchemaToBytes(NewSchema([]Field{NewField("deep", deep, true)})) + requireErrorContains(t, err, "nesting") + // Exactly the reader's limit still round-trips. + var ok DataType = Int32Type{} + for i := 0; i < maxSchemaNestingDepth-1; i++ { + ok = List(ok) + } + okBytes, err := SchemaToBytes(NewSchema([]Field{NewField("ok", ok, true)})) + require.NoError(t, err) + _, err = SchemaFromBytes(okBytes) + require.NoError(t, err) + + // A type that refers to itself. + loop := &ListType{} + loop.Elem = NewField("item", loop, true) + _, err = SchemaToBytes(NewSchema([]Field{NewField("loop", loop, true)})) + requireErrorContains(t, err, "refers to itself") + + // Decimal parameters wider than the single wire byte. + _, err = SchemaToBytes(NewSchema([]Field{NewField("d", DecimalType{Precision: 300, Scale: 2}, true)})) + requireErrorContains(t, err, "decimal") + + // Pointer decimal and foreign DataType implementations. + _, err = SchemaToBytes(NewSchema([]Field{NewField("d", &DecimalType{Precision: 10, Scale: 2}, true)})) + require.Error(t, err) + _, err = SchemaToBytes(NewSchema([]Field{NewField("u", unknownDataType{}, true)})) + require.Error(t, err) +} From 17114bc5fb64bbd0e8b95f0ac012b2996fb8b1c0 Mon Sep 17 00:00:00 2001 From: ayush00git Date: Sat, 22 Aug 2026 22:47:49 +0530 Subject: [PATCH 18/20] fix(go): align row schema inference with Java and harden the codecs Schema inference now derives the same schema Java does for equivalent struct definitions: []byte maps to list like Java's byte[] (with a raw-copy codec), map values are canonically nullable, and fory.Date/time.Time/time.Duration carriers are nullable because their Java counterparts are objects. Decoding a null into a Go carrier that cannot hold nil (scalar, string, temporal, value struct, value-type element) is now an error instead of a silent zero value. The fory tag grammar mirrors the core parser (top-level comma split, trimmed '=', strict boolean forms, duplicate and unknown keys rejected), recursive slice and map aliases are detected, pointers to slices or maps are rejected because two nil states would share one null bit, and map keys are restricted to shapes whose encoded fields determine Go equality. Readers and writers reject what the wire format cannot represent: strings must be valid UTF-8, timestamps must fit int64 microseconds, durations that would overflow time.Duration are rejected, array counts are compared against the size limit before multiplying, and wire counts are narrowed to int only after bounds checks so 32-bit platforms cannot wrap. INT64 values are checked against the Go int width. Encode writes the schema hash into the reusable buffer so framed output needs one copy, and map codecs write keys during a single iteration with one typed value snapshot instead of per-entry reflect.Value allocations; benchmarks cover both paths. Public entry points return errors for nil arguments. Row Format is documented as a trusted in-memory format, and SchemaHash is documented as the type-shape fingerprint it is. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01MwqYMrDU1kD1wcB9snj9mh --- go/fory/row/array_test.go | 4 +- go/fory/row/encoder.go | 319 ++++++++++++++++++------------ go/fory/row/encoder_bench_test.go | 76 +++++++ go/fory/row/encoder_test.go | 122 ++++++++++++ go/fory/row/infer.go | 219 ++++++++++++++++---- go/fory/row/infer_test.go | 74 ++++++- go/fory/row/row.go | 40 +++- go/fory/row/row_test.go | 39 +++- go/fory/row/writer.go | 52 ++++- 9 files changed, 760 insertions(+), 185 deletions(-) create mode 100644 go/fory/row/encoder_bench_test.go diff --git a/go/fory/row/array_test.go b/go/fory/row/array_test.go index 8808a1bdb8..7faef1c518 100644 --- a/go/fory/row/array_test.go +++ b/go/fory/row/array_test.go @@ -106,7 +106,9 @@ func TestNestedArray(t *testing.T) { func TestArrayResetRejectsInvalidLength(t *testing.T) { w := NewArrayWriter(List(Int64Type{}).Elem, fory.NewByteBuffer(nil)) require.Error(t, w.Reset(-1)) - require.Error(t, w.Reset(1<<40)) + require.Error(t, w.Reset(maxArrayDataBytes/8+1)) + // Extreme counts must fail the limit check, not overflow past it. + require.Error(t, w.Reset(math.MaxInt)) } // Forged element counts must fail validation even when the arithmetic diff --git a/go/fory/row/encoder.go b/go/fory/row/encoder.go index 5e0f793a17..aa459d9ce9 100644 --- a/go/fory/row/encoder.go +++ b/go/fory/row/encoder.go @@ -30,9 +30,12 @@ import ( // a codec tree compiled once with reflection at construction time. The // schema is inferred per InferSchema. // -// An Encoder owns a reusable write buffer and is NOT goroutine-safe; -// create one Encoder per goroutine. Decoding is safe on corrupt or -// untrusted input. +// The row format is a trusted in-memory format: FromRow and Decode +// expect trusted, schema-matched bytes produced by a row writer for the +// same schema. Malformed input surfaces as an error rather than a +// panic, but no allocation or work limits are enforced against hostile +// bytes. An Encoder owns a reusable write buffer and is NOT +// goroutine-safe; create one Encoder per goroutine. type Encoder[T any] struct { codec *structCodec hash int64 @@ -54,13 +57,41 @@ func NewEncoder[T any]() (*Encoder[T], error) { func (e *Encoder[T]) Schema() *Schema { return e.codec.schema } -// SchemaHash returns the cross-language schema hash used by Encode and -// Decode to detect out-of-sync struct definitions. +// SchemaHash returns the cross-language type-shape fingerprint of the +// schema: a fold over the recursive type ids only. It does not cover +// field names, nullability, decimal parameters, or the order of +// same-typed fields, so Encode/Decode can detect only type-shape +// mismatches between writer and reader. func (e *Encoder[T]) SchemaHash() int64 { return e.hash } // ToRow serializes v to row bytes (no framing, just the row). func (e *Encoder[T]) ToRow(v *T) ([]byte, error) { + if v == nil { + return nil, errNilValue + } + e.buf.SetWriterIndex(0) + return e.finishRow(v) +} + +// Encode frames the row for cross-language exchange: an int64 +// little-endian schema hash followed by the row bytes, matching the +// Java and Python row encoders. +func (e *Encoder[T]) Encode(v *T) ([]byte, error) { + if v == nil { + return nil, errNilValue + } + // The hash is written into the reusable buffer ahead of the row so + // the framed output needs a single copy. e.buf.SetWriterIndex(0) + e.buf.Reserve(8) + binary.LittleEndian.PutUint64(e.buf.GetData()[:8], uint64(e.hash)) + e.buf.SetWriterIndex(8) + return e.finishRow(v) +} + +// finishRow writes v at the buffer's current writer index and returns +// an owned copy of everything written so far. +func (e *Encoder[T]) finishRow(v *T) ([]byte, error) { if err := e.codec.writeStruct(reflect.ValueOf(v).Elem()); err != nil { return nil, err } @@ -77,29 +108,18 @@ func (e *Encoder[T]) FromRow(data []byte) (T, error) { } func (e *Encoder[T]) FromRowInto(data []byte, out *T) (err error) { + if out == nil { + return fmt.Errorf("row: cannot decode into a nil target") + } defer func() { if p := recover(); p != nil { - err = fmt.Errorf("row: corrupt row data: %v", p) + err = fmt.Errorf("row: malformed row data: %v", p) } }() return e.codec.readStruct(NewRow(e.codec.schema, data), reflect.ValueOf(out).Elem()) } -// Encode frames the row for cross-language exchange: an int64 -// little-endian schema hash followed by the row bytes, matching the -// Java and Python row encoders. -func (e *Encoder[T]) Encode(v *T) ([]byte, error) { - rowBytes, err := e.ToRow(v) - if err != nil { - return nil, err - } - out := make([]byte, 8+len(rowBytes)) - binary.LittleEndian.PutUint64(out, uint64(e.hash)) - copy(out[8:], rowBytes) - return out, nil -} - -// Decode verifies the schema hash and deserializes the row. +// Decode verifies the schema type-shape hash and deserializes the row. func (e *Encoder[T]) Decode(data []byte) (T, error) { var out T if len(data) < 8 { @@ -107,12 +127,18 @@ func (e *Encoder[T]) Decode(data []byte) (T, error) { } hash := int64(binary.LittleEndian.Uint64(data)) if hash != e.hash { - return out, fmt.Errorf("row: schema hash mismatch: data has %d, encoder expects %d; writer and reader struct definitions are out of sync", hash, e.hash) + return out, fmt.Errorf("row: schema type-shape hash mismatch: data has %d, encoder expects %d", hash, e.hash) } err := e.FromRowInto(data[8:], &out) return out, err } +var errNilValue = fmt.Errorf("row: cannot encode a nil value") + +func nullIntoValueError(goType reflect.Type) error { + return fmt.Errorf("row: null value for non-nullable Go type %v; use a pointer carrier to receive nulls", goType) +} + // slotWriter is the shared write surface of RowWriter and ArrayWriter, // letting one value codec serve struct fields and array elements. type slotWriter interface { @@ -125,7 +151,7 @@ type slotWriter interface { WriteFloat32(i int, v float32) WriteFloat64(i int, v float64) WriteDate(i int, d fory.Date) error - WriteTimestamp(i int, t time.Time) + WriteTimestamp(i int, t time.Time) error WriteDuration(i int, d time.Duration) WriteString(i int, s string) error WriteBytes(i int, b []byte) error @@ -153,8 +179,9 @@ type valueReader interface { } // valueCodec reads or writes one value at slot/element i. Write -// functions handle nil for nilable Go types; read functions overwrite -// the target even when the field is null, so reused targets stay clean. +// functions handle nil for nilable Go types. Read functions overwrite +// the target even when the value is null so reused targets stay clean, +// and reject null for Go carriers that cannot represent it. type valueCodec struct { write func(w slotWriter, i int, v reflect.Value) error read func(g valueReader, i int, v reflect.Value) error @@ -246,89 +273,66 @@ func newValueCodec(goType reflect.Type, dataType DataType, buf *fory.ByteBuffer) switch dt := dataType.(type) { case BoolType: - return valueCodec{ - write: func(w slotWriter, i int, v reflect.Value) error { w.WriteBool(i, v.Bool()); return nil }, - read: func(g valueReader, i int, v reflect.Value) error { v.SetBool(g.Bool(i)); return nil }, - }, nil + return scalarCodec(goType, + func(w slotWriter, i int, v reflect.Value) error { w.WriteBool(i, v.Bool()); return nil }, + func(g valueReader, i int, v reflect.Value) error { v.SetBool(g.Bool(i)); return nil }), nil case Int8Type: - return valueCodec{ - write: func(w slotWriter, i int, v reflect.Value) error { w.WriteInt8(i, int8(v.Int())); return nil }, - read: func(g valueReader, i int, v reflect.Value) error { v.SetInt(int64(g.Int8(i))); return nil }, - }, nil + return scalarCodec(goType, + func(w slotWriter, i int, v reflect.Value) error { w.WriteInt8(i, int8(v.Int())); return nil }, + func(g valueReader, i int, v reflect.Value) error { v.SetInt(int64(g.Int8(i))); return nil }), nil case Int16Type: - return valueCodec{ - write: func(w slotWriter, i int, v reflect.Value) error { w.WriteInt16(i, int16(v.Int())); return nil }, - read: func(g valueReader, i int, v reflect.Value) error { v.SetInt(int64(g.Int16(i))); return nil }, - }, nil + return scalarCodec(goType, + func(w slotWriter, i int, v reflect.Value) error { w.WriteInt16(i, int16(v.Int())); return nil }, + func(g valueReader, i int, v reflect.Value) error { v.SetInt(int64(g.Int16(i))); return nil }), nil case Int32Type: - return valueCodec{ - write: func(w slotWriter, i int, v reflect.Value) error { w.WriteInt32(i, int32(v.Int())); return nil }, - read: func(g valueReader, i int, v reflect.Value) error { v.SetInt(int64(g.Int32(i))); return nil }, - }, nil + return scalarCodec(goType, + func(w slotWriter, i int, v reflect.Value) error { w.WriteInt32(i, int32(v.Int())); return nil }, + func(g valueReader, i int, v reflect.Value) error { v.SetInt(int64(g.Int32(i))); return nil }), nil case Int64Type: - return valueCodec{ - write: func(w slotWriter, i int, v reflect.Value) error { w.WriteInt64(i, v.Int()); return nil }, - read: func(g valueReader, i int, v reflect.Value) error { v.SetInt(g.Int64(i)); return nil }, - }, nil + return scalarCodec(goType, + func(w slotWriter, i int, v reflect.Value) error { w.WriteInt64(i, v.Int()); return nil }, + func(g valueReader, i int, v reflect.Value) error { + x := g.Int64(i) + // `int` is 32 bits wide on some platforms. + if v.OverflowInt(x) { + return fmt.Errorf("row: value %d overflows %v", x, goType) + } + v.SetInt(x) + return nil + }), nil case Float32Type: - return valueCodec{ - write: func(w slotWriter, i int, v reflect.Value) error { w.WriteFloat32(i, float32(v.Float())); return nil }, - read: func(g valueReader, i int, v reflect.Value) error { v.SetFloat(float64(g.Float32(i))); return nil }, - }, nil + return scalarCodec(goType, + func(w slotWriter, i int, v reflect.Value) error { w.WriteFloat32(i, float32(v.Float())); return nil }, + func(g valueReader, i int, v reflect.Value) error { v.SetFloat(float64(g.Float32(i))); return nil }), nil case Float64Type: - return valueCodec{ - write: func(w slotWriter, i int, v reflect.Value) error { w.WriteFloat64(i, v.Float()); return nil }, - read: func(g valueReader, i int, v reflect.Value) error { v.SetFloat(g.Float64(i)); return nil }, - }, nil + return scalarCodec(goType, + func(w slotWriter, i int, v reflect.Value) error { w.WriteFloat64(i, v.Float()); return nil }, + func(g valueReader, i int, v reflect.Value) error { v.SetFloat(g.Float64(i)); return nil }), nil case Date32Type: - return valueCodec{ - write: func(w slotWriter, i int, v reflect.Value) error { return w.WriteDate(i, v.Interface().(fory.Date)) }, - read: func(g valueReader, i int, v reflect.Value) error { v.Set(reflect.ValueOf(g.Date(i))); return nil }, - }, nil + return scalarCodec(goType, + func(w slotWriter, i int, v reflect.Value) error { return w.WriteDate(i, v.Interface().(fory.Date)) }, + func(g valueReader, i int, v reflect.Value) error { v.Set(reflect.ValueOf(g.Date(i))); return nil }), nil case TimestampType: - return valueCodec{ - write: func(w slotWriter, i int, v reflect.Value) error { - w.WriteTimestamp(i, v.Interface().(time.Time)) - return nil + return scalarCodec(goType, + func(w slotWriter, i int, v reflect.Value) error { + return w.WriteTimestamp(i, v.Interface().(time.Time)) }, - read: func(g valueReader, i int, v reflect.Value) error { v.Set(reflect.ValueOf(g.Timestamp(i))); return nil }, - }, nil + func(g valueReader, i int, v reflect.Value) error { v.Set(reflect.ValueOf(g.Timestamp(i))); return nil }), nil case DurationType: - return valueCodec{ - write: func(w slotWriter, i int, v reflect.Value) error { + return scalarCodec(goType, + func(w slotWriter, i int, v reflect.Value) error { w.WriteDuration(i, time.Duration(v.Int())) return nil }, - read: func(g valueReader, i int, v reflect.Value) error { v.SetInt(int64(g.Duration(i))); return nil }, - }, nil + func(g valueReader, i int, v reflect.Value) error { v.SetInt(int64(g.Duration(i))); return nil }), nil case StringType: - return valueCodec{ - write: func(w slotWriter, i int, v reflect.Value) error { return w.WriteString(i, v.String()) }, - read: func(g valueReader, i int, v reflect.Value) error { v.SetString(g.String(i)); return nil }, - }, nil - case BinaryType: - return valueCodec{ - write: func(w slotWriter, i int, v reflect.Value) error { - if v.IsNil() { - w.SetNullAt(i) - return nil - } - return w.WriteBytes(i, v.Bytes()) - }, - read: func(g valueReader, i int, v reflect.Value) error { - b := g.Binary(i) - if b == nil { - v.SetBytes(nil) - return nil - } - // Copy so the decoded struct never aliases the input. - owned := make([]byte, len(b)) - copy(owned, b) - v.SetBytes(owned) - return nil - }, - }, nil + return scalarCodec(goType, + func(w slotWriter, i int, v reflect.Value) error { return w.WriteString(i, v.String()) }, + func(g valueReader, i int, v reflect.Value) error { v.SetString(g.String(i)); return nil }), nil case *ListType: + if goType.Elem().Kind() == reflect.Uint8 { + return newByteListCodec(goType, dt, buf), nil + } return newListCodec(goType, dt, buf) case *MapType: return newMapCodec(goType, dt, buf) @@ -348,8 +352,7 @@ func newValueCodec(goType reflect.Type, dataType DataType, buf *fory.ByteBuffer) read: func(g valueReader, i int, v reflect.Value) error { nested := g.Struct(i) if nested == nil { - v.Set(reflect.Zero(goType)) - return nil + return nullIntoValueError(goType) } return child.readStruct(nested, v) }, @@ -359,6 +362,65 @@ func newValueCodec(goType reflect.Type, dataType DataType, buf *fory.ByteBuffer) } } +// scalarCodec wraps a codec for a Go carrier that cannot hold nil, so a +// null value on read is an error instead of a silent zero value. +func scalarCodec(goType reflect.Type, + write func(w slotWriter, i int, v reflect.Value) error, + read func(g valueReader, i int, v reflect.Value) error) valueCodec { + return valueCodec{ + write: write, + read: func(g valueReader, i int, v reflect.Value) error { + if g.IsNullAt(i) { + return nullIntoValueError(goType) + } + return read(g, i, v) + }, + } +} + +// newByteListCodec handles []byte as list, the Java byte[] model: +// the bytes are copied straight into and out of the int8 element region +// with no per-element reflection. +func newByteListCodec(goType reflect.Type, listType *ListType, buf *fory.ByteBuffer) valueCodec { + writer := NewArrayWriter(listType.Elem, buf) + return valueCodec{ + write: func(w slotWriter, i int, v reflect.Value) error { + if v.IsNil() { + w.SetNullAt(i) + return nil + } + b := v.Bytes() + start := buf.WriterIndex() + if err := writer.Reset(len(b)); err != nil { + return err + } + copy(buf.GetData()[writer.base+writer.headerBytes:], b) + return w.SetOffsetAndSize(i, start, buf.WriterIndex()-start) + }, + read: func(g valueReader, i int, v reflect.Value) error { + arr := g.Array(i) + if arr == nil { + v.Set(reflect.Zero(goType)) + return nil + } + if err := arr.validateBounds(); err != nil { + return err + } + n := arr.NumElements() + for j := 0; j < n; j++ { + if arr.IsNullAt(j) { + return nullIntoValueError(goType.Elem()) + } + } + // Copy so the decoded struct never aliases the input. + owned := make([]byte, n) + copy(owned, arr.data[arr.headerBytes:arr.headerBytes+n]) + v.SetBytes(owned) + return nil + }, + } +} + func newListCodec(goType reflect.Type, listType *ListType, buf *fory.ByteBuffer) (valueCodec, error) { elemCodec, err := newValueCodec(goType.Elem(), listType.Elem.Type, buf) if err != nil { @@ -406,17 +468,24 @@ func newListCodec(goType reflect.Type, listType *ListType, buf *fory.ByteBuffer) } func newMapCodec(goType reflect.Type, mapType *MapType, buf *fory.ByteBuffer) (valueCodec, error) { - keyCodec, err := newValueCodec(goType.Key(), mapType.Key.Type, buf) + keyType, valueType := goType.Key(), goType.Elem() + keyCodec, err := newValueCodec(keyType, mapType.Key.Type, buf) if err != nil { return valueCodec{}, err } - valueCodecImpl, err := newValueCodec(goType.Elem(), mapType.Value.Type, buf) + valueCodecImpl, err := newValueCodec(valueType, mapType.Value.Type, buf) if err != nil { return valueCodec{}, err } mapWriter := NewMapWriter(buf) keysWriter := NewArrayWriter(mapType.Key, buf) valuesWriter := NewArrayWriter(mapType.Value, buf) + // Scratch values reused across entries: the encoder is single-use + // per goroutine and a map site is never re-entered recursively. + keyScratch := reflect.New(keyType).Elem() + readKey := reflect.New(keyType).Elem() + readValue := reflect.New(valueType).Elem() + values := reflect.MakeSlice(reflect.SliceOf(valueType), 0, 0) return valueCodec{ write: func(w slotWriter, i int, v reflect.Value) error { if v.IsNil() { @@ -426,32 +495,42 @@ func newMapCodec(goType reflect.Type, mapType *MapType, buf *fory.ByteBuffer) (v n := v.Len() start := buf.WriterIndex() mapWriter.Reset() - // Go map iteration order changes between iterations, so - // snapshot entries once to keep keys and values aligned. - keys := make([]reflect.Value, 0, n) - values := make([]reflect.Value, 0, n) - iter := v.MapRange() - for iter.Next() { - keys = append(keys, iter.Key()) - values = append(values, iter.Value()) - } - if err := keysWriter.Reset(len(keys)); err != nil { + if err := keysWriter.Reset(n); err != nil { return err } - for j, k := range keys { - if err := keyCodec.write(keysWriter, j, k); err != nil { + // Keys are written during the single map iteration while + // values are captured into one typed slice, so each entry + // costs no reflect.Value allocation. Go map iteration order + // changes between iterations, which is why both halves + // cannot simply be iterated twice. + if values.Cap() < n { + values = reflect.MakeSlice(reflect.SliceOf(valueType), n, n) + } + values = values.Slice(0, n) + j := 0 + for iter := v.MapRange(); iter.Next(); j++ { + if j >= n { + return fmt.Errorf("row: map changed size during encoding") + } + keyScratch.SetIterKey(iter) + if err := keyCodec.write(keysWriter, j, keyScratch); err != nil { return err } + values.Index(j).SetIterValue(iter) + } + if j != n { + return fmt.Errorf("row: map changed size during encoding") } mapWriter.FinishKeys() - if err := valuesWriter.Reset(len(values)); err != nil { + if err := valuesWriter.Reset(n); err != nil { return err } - for j, val := range values { - if err := valueCodecImpl.write(valuesWriter, j, val); err != nil { + for j := 0; j < n; j++ { + if err := valueCodecImpl.write(valuesWriter, j, values.Index(j)); err != nil { return err } } + values.Clear() // drop references to the caller's values return w.SetOffsetAndSize(i, start, buf.WriterIndex()-start) }, read: func(g valueReader, i int, v reflect.Value) error { @@ -460,28 +539,26 @@ func newMapCodec(goType reflect.Type, mapType *MapType, buf *fory.ByteBuffer) (v v.Set(reflect.Zero(goType)) return nil } - keys, values := md.Keys(), md.Values() + keys, vals := md.Keys(), md.Values() if err := keys.validateBounds(); err != nil { return err } - if err := values.validateBounds(); err != nil { + if err := vals.validateBounds(); err != nil { return err } n := keys.NumElements() - if values.NumElements() != n { - return fmt.Errorf("row: map has %d keys but %d values", n, values.NumElements()) + if vals.NumElements() != n { + return fmt.Errorf("row: map has %d keys but %d values", n, vals.NumElements()) } m := reflect.MakeMapWithSize(goType, n) for j := 0; j < n; j++ { - key := reflect.New(goType.Key()).Elem() - if err := keyCodec.read(keys, j, key); err != nil { + if err := keyCodec.read(keys, j, readKey); err != nil { return err } - value := reflect.New(goType.Elem()).Elem() - if err := valueCodecImpl.read(values, j, value); err != nil { + if err := valueCodecImpl.read(vals, j, readValue); err != nil { return err } - m.SetMapIndex(key, value) + m.SetMapIndex(readKey, readValue) } v.Set(m) return nil diff --git a/go/fory/row/encoder_bench_test.go b/go/fory/row/encoder_bench_test.go new file mode 100644 index 0000000000..e355b57169 --- /dev/null +++ b/go/fory/row/encoder_bench_test.go @@ -0,0 +1,76 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package row + +import ( + "fmt" + "testing" +) + +// Framed encoding must cost one output allocation beyond the row work. +func BenchmarkEncodeFramed(b *testing.B) { + enc, err := NewEncoder[person]() + if err != nil { + b.Fatal(err) + } + value := samplePerson() + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err := enc.Encode(&value); err != nil { + b.Fatal(err) + } + } +} + +type mapHolder struct { + Entries map[string]int64 +} + +// Map codecs must stay linear without per-entry reflection allocations. +func BenchmarkMapRoundTrip(b *testing.B) { + enc, err := NewEncoder[mapHolder]() + if err != nil { + b.Fatal(err) + } + value := mapHolder{Entries: make(map[string]int64, 256)} + for i := 0; i < 256; i++ { + value.Entries[fmt.Sprintf("key-%d", i)] = int64(i) + } + b.Run("encode", func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + if _, err := enc.ToRow(&value); err != nil { + b.Fatal(err) + } + } + }) + rowBytes, err := enc.ToRow(&value) + if err != nil { + b.Fatal(err) + } + b.Run("decode", func(b *testing.B) { + b.ReportAllocs() + var out mapHolder + for i := 0; i < b.N; i++ { + if err := enc.FromRowInto(rowBytes, &out); err != nil { + b.Fatal(err) + } + } + }) +} diff --git a/go/fory/row/encoder_test.go b/go/fory/row/encoder_test.go index ec000ef979..15e165b08d 100644 --- a/go/fory/row/encoder_test.go +++ b/go/fory/row/encoder_test.go @@ -175,3 +175,125 @@ func TestNewEncoderRejectsNonStruct(t *testing.T) { _, err = NewEncoder[map[string]int]() require.Error(t, err) } + +// A null for a Go carrier that cannot hold nil is a decode error, not a +// silent zero value: the wire and type hash cannot distinguish the two. +func TestDecodeRejectsNullIntoValueCarrier(t *testing.T) { + type inner struct { + X int32 + } + type carriers struct { + S string + In inner + L []int32 + } + enc, err := NewEncoder[carriers]() + require.NoError(t, err) + schema := enc.Schema() + w := NewRowWriter(schema) + idx := schema.FieldIndex + + // buildRow writes a complete valid row, then nulls one target: + // the string field, the value-struct field, or one list element. + buildRow := func(nullTarget string) []byte { + buf := w.Buffer() + buf.SetWriterIndex(0) + w.Reset() + if nullTarget == "s" { + w.SetNullAt(idx("s")) + } else { + require.NoError(t, w.WriteString(idx("s"), "x")) + } + if nullTarget == "in" { + w.SetNullAt(idx("in")) + } else { + child := NewRowWriterWithBuffer(NewSchema(schema.Field(idx("in")).Type.(*StructType).Fields), buf) + start := buf.WriterIndex() + child.Reset() + child.WriteInt32(0, 1) + require.NoError(t, w.SetOffsetAndSize(idx("in"), start, buf.WriterIndex()-start)) + } + aw := NewArrayWriter(schema.Field(idx("l")).Type.(*ListType).Elem, buf) + start := buf.WriterIndex() + require.NoError(t, aw.Reset(2)) + aw.WriteInt32(0, 1) + if nullTarget == "l[1]" { + aw.SetNullAt(1) + } else { + aw.WriteInt32(1, 2) + } + require.NoError(t, w.SetOffsetAndSize(idx("l"), start, buf.WriterIndex()-start)) + return append([]byte(nil), w.ToBytes()...) + } + + decoded, err := enc.FromRow(buildRow("")) + require.NoError(t, err) + require.Equal(t, carriers{S: "x", In: inner{X: 1}, L: []int32{1, 2}}, decoded) + for _, target := range []string{"s", "in", "l[1]"} { + _, err := enc.FromRow(buildRow(target)) + requireErrorContains(t, err, "null value") + } +} + +// Inferred schemas must survive the wire format unchanged, which +// requires the canonical (nullable) map value field. +func TestInferredSchemaRoundTripsThroughBytes(t *testing.T) { + type primitiveMap struct { + Counts map[string]int32 + Flags map[int64]bool + } + for _, schema := range []*Schema{ + mustSchema(t, NewEncoder[primitiveMap]), + mustSchema(t, NewEncoder[person]), + } { + encoded, err := SchemaToBytes(schema) + require.NoError(t, err) + parsed, err := SchemaFromBytes(encoded) + require.NoError(t, err) + require.True(t, parsed.Equal(schema), "parsed %v != inferred %v", parsed, schema) + } + + enc, err := NewEncoder[primitiveMap]() + require.NoError(t, err) + original := primitiveMap{Counts: map[string]int32{"a": 1, "b": -2}, Flags: map[int64]bool{7: true}} + rowBytes, err := enc.ToRow(&original) + require.NoError(t, err) + decoded, err := enc.FromRow(rowBytes) + require.NoError(t, err) + require.Equal(t, original, decoded) +} + +func mustSchema[T any](t *testing.T, newEncoder func() (*Encoder[T], error)) *Schema { + t.Helper() + enc, err := newEncoder() + require.NoError(t, err) + return enc.Schema() +} + +func TestEncoderRejectsNilArguments(t *testing.T) { + enc, err := NewEncoder[person]() + require.NoError(t, err) + _, err = enc.ToRow(nil) + require.Error(t, err) + _, err = enc.Encode(nil) + require.Error(t, err) + require.Error(t, enc.FromRowInto([]byte{}, nil)) +} + +// Struct-typed map keys must round-trip entry for entry. +func TestEncoderStructMapKeys(t *testing.T) { + type point struct { + X, Y int32 + } + type grid struct { + Cells map[point]string + } + enc, err := NewEncoder[grid]() + require.NoError(t, err) + original := grid{Cells: map[point]string{{1, 2}: "a", {3, 4}: "b"}} + rowBytes, err := enc.ToRow(&original) + require.NoError(t, err) + decoded, err := enc.FromRow(rowBytes) + require.NoError(t, err) + require.Equal(t, original, decoded) +} diff --git a/go/fory/row/infer.go b/go/fory/row/infer.go index 109ae74129..f0e1eb1978 100644 --- a/go/fory/row/infer.go +++ b/go/fory/row/infer.go @@ -36,22 +36,38 @@ var ( ) // InferSchema infers the row schema for a struct type or pointer to -// struct, including every exported field not tagged `fory:"ignore"`. +// struct, including every exported field not ignored by its fory tag +// (`fory:"-"`, `fory:"ignore"`, or `fory:"ignore=true"`). // // Fields are sorted by their lowerCamel name and named by its // snake_case form (UserName -> user_name), matching Java's schema // inference so both languages derive identical schemas. // // Type mapping: -// - bool, int8/16/32/64, float32/64: same-width row types; int maps to int64 -// - string, []byte: string and binary, nullable -// - slices, maps, nested structs: list, map, and struct, nullable +// - bool, int8/16/32/64, float32/64: same-width row types, non-nullable; +// int maps to int64 +// - string: string, nullable +// - []byte: list, nullable, matching Java's byte[] (the row +// format's binary type is reachable only through explicit schemas) +// - fory.Date, time.Time, time.Duration: date32, timestamp, duration, +// nullable (their Java carriers are objects) +// - slices, maps, nested structs: list, map, and struct, nullable; +// map values are always nullable, map keys never // - *T: the row type of T, nullable -// - fory.Date, time.Time, time.Duration: date32, timestamp, duration // -// Unsigned integers, fixed-size arrays, nested pointers, and pointer -// map keys are unsupported and return an error. +// A nullable field whose Go carrier cannot hold nil (a scalar, string, +// temporal, or value struct) still decodes, but a null value for it is +// a decode error; use a pointer carrier when nulls must round-trip. +// +// Unsupported: unsigned integers, fixed-size arrays, nested pointers, +// pointers to slices or maps (two nil states, one null bit), map keys +// whose encoded fields do not determine Go equality (pointers, +// time.Time, structs with unexported or ignored fields), and recursive +// types. func InferSchema(t reflect.Type) (*Schema, error) { + if t == nil { + return nil, fmt.Errorf("row: cannot infer a schema from a nil type") + } layout, err := inferStructLayout(t, nil) if err != nil { return nil, err @@ -72,10 +88,8 @@ func inferStructLayout(t reflect.Type, path []reflect.Type) (*structLayout, erro if t.Kind() != reflect.Struct || t == goDateType || t == goTimeType { return nil, fmt.Errorf("row: schema inference expects a struct type, got %v", t) } - for _, seen := range path { - if seen == t { - return nil, fmt.Errorf("row: circular reference through type %v", t) - } + if err := checkCycle(t, path); err != nil { + return nil, err } path = append(path, t) @@ -118,10 +132,27 @@ func inferStructLayout(t reflect.Type, path []reflect.Type) (*structLayout, erro return layout, nil } +// checkCycle rejects a type already on the active inference path. Named +// slice and map types can recurse just like structs (type L []L), so +// every composite type is tracked, not only structs. +func checkCycle(t reflect.Type, path []reflect.Type) error { + for _, seen := range path { + if seen == t { + return fmt.Errorf("row: circular reference through type %v", t) + } + } + return nil +} + func inferField(name string, t reflect.Type, path []reflect.Type) (Field, error) { if t.Kind() == reflect.Ptr { - if t.Elem().Kind() == reflect.Ptr { + switch t.Elem().Kind() { + case reflect.Ptr: return Field{}, fmt.Errorf("row: nested pointer type %v is unsupported", t) + case reflect.Slice, reflect.Map: + // A nil pointer and a pointer to a nil container would share + // one null bit, so the value could not round-trip. + return Field{}, fmt.Errorf("row: pointer to slice or map type %v is unsupported; use %v", t, t.Elem()) } inner, err := inferField(name, t.Elem(), path) if err != nil { @@ -132,11 +163,11 @@ func inferField(name string, t reflect.Type, path []reflect.Type) (Field, error) } switch t { case goDateType: - return Field{Name: name, Type: Date32Type{}}, nil + return Field{Name: name, Type: Date32Type{}, Nullable: true}, nil case goTimeType: - return Field{Name: name, Type: TimestampType{}}, nil + return Field{Name: name, Type: TimestampType{}, Nullable: true}, nil case goDurationType: - return Field{Name: name, Type: DurationType{}}, nil + return Field{Name: name, Type: DurationType{}, Nullable: true}, nil } switch t.Kind() { case reflect.Bool: @@ -160,26 +191,39 @@ func inferField(name string, t reflect.Type, path []reflect.Type) (Field, error) return Field{Name: name, Type: StringType{}, Nullable: true}, nil case reflect.Slice: if t.Elem().Kind() == reflect.Uint8 { - return Field{Name: name, Type: BinaryType{}, Nullable: true}, nil + // Java infers byte[] as list; the encoder copies the + // bytes straight into the element region. + elem := Field{Name: listItemName, Type: Int8Type{}} + return Field{Name: name, Type: &ListType{Elem: elem}, Nullable: true}, nil + } + if err := checkCycle(t, path); err != nil { + return Field{}, err } - elem, err := inferField(listItemName, t.Elem(), path) + elem, err := inferField(listItemName, t.Elem(), append(path, t)) if err != nil { return Field{}, err } return Field{Name: name, Type: &ListType{Elem: elem}, Nullable: true}, nil case reflect.Map: - if t.Key().Kind() == reflect.Ptr { - return Field{}, fmt.Errorf("row: pointer map key type %v is unsupported", t) + if err := checkCycle(t, path); err != nil { + return Field{}, err + } + if err := validateMapKeyType(t.Key()); err != nil { + return Field{}, err } + path = append(path, t) key, err := inferField(mapKeyName, t.Key(), path) if err != nil { return Field{}, err } - key.Nullable = false value, err := inferField(mapValueName, t.Elem(), path) if err != nil { return Field{}, err } + // Canonical map children, matching Java MapType and the schema + // parser: keys are never nullable, values always are. + key.Nullable = false + value.Nullable = true return Field{Name: name, Type: &MapType{Key: key, Value: value}, Nullable: true}, nil case reflect.Struct: layout, err := inferStructLayout(t, path) @@ -192,34 +236,137 @@ func inferField(name string, t reflect.Type, path []reflect.Type) (Field, error) } } -// hasIgnoreTag mirrors the ignore semantics of the core fory tag -// parser (parseFieldTag in field_spec.go): a whole tag of "-" or an -// ignore/ignore=true part skips the field, ignore=false keeps it, and -// any other ignore value is an error. Other tag keys are not used by -// the row format. +// validateMapKeyType accepts only key types whose encoded fields fully +// determine Go equality, so distinct keys never encode identically and +// decoding never collapses entries: scalars, strings, and structs made +// only of such fields with nothing unexported or ignored. time.Time is +// rejected because its Location is not encoded. +func validateMapKeyType(t reflect.Type) error { + switch t.Kind() { + case reflect.Bool, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int, + reflect.Float32, reflect.Float64, reflect.String: + return nil + case reflect.Struct: + if t == goTimeType { + return fmt.Errorf("row: map key type %v is unsupported: its Location is not encoded", t) + } + for i := 0; i < t.NumField(); i++ { + f := t.Field(i) + if f.PkgPath != "" { + return fmt.Errorf("row: map key type %v has unexported field %s that would not be encoded", t, f.Name) + } + ignored, err := hasIgnoreTag(f.Tag.Get("fory")) + if err != nil { + return err + } + if ignored { + return fmt.Errorf("row: map key type %v has ignored field %s that would not be encoded", t, f.Name) + } + if err := validateMapKeyType(f.Type); err != nil { + return err + } + } + return nil + default: + return fmt.Errorf("row: map key type %v cannot preserve Go equality in row format", t) + } +} + +// Keys accepted by the core fory tag parser; the row format acts only +// on "ignore" but mirrors the grammar so a tag valid for object-graph +// serialization is valid here and vice versa. +var foryTagKeys = map[string]bool{ + "id": true, "nullable": true, "ref": true, "ignore": true, "encoding": true, "type": true, +} + +// hasIgnoreTag mirrors the core fory tag grammar (parseFieldTag in +// field_spec.go): a whole tag of "-" ignores the field; otherwise parts +// split on top-level commas, keys and values are trimmed around '=', +// duplicate keys are errors, and ignore accepts the strict boolean forms +// true/1/yes and false/0/no (case-insensitive), defaulting to true. func hasIgnoreTag(tag string) (bool, error) { if tag == "-" { return true, nil } ignore := false - for _, part := range strings.Split(tag, ",") { + seen := map[string]bool{} + for _, part := range splitTopLevel(tag) { part = strings.TrimSpace(part) - if part == "ignore" { + if part == "" { + continue + } + key, value, hasValue := part, "", false + if idx := indexTopLevel(part, '='); idx >= 0 { + key, value, hasValue = strings.TrimSpace(part[:idx]), strings.TrimSpace(part[idx+1:]), true + } + if !foryTagKeys[key] { + return false, fmt.Errorf("row: unknown fory tag key %q", key) + } + if seen[key] { + return false, fmt.Errorf("row: duplicate fory tag key %q", key) + } + seen[key] = true + if key != "ignore" { + continue + } + if !hasValue { ignore = true - } else if strings.HasPrefix(part, "ignore=") { - switch strings.TrimPrefix(part, "ignore=") { - case "true": - ignore = true - case "false": - ignore = false - default: - return false, fmt.Errorf("row: invalid ignore value in fory tag %q", tag) - } + continue + } + switch strings.ToLower(value) { + case "true", "1", "yes": + ignore = true + case "false", "0", "no": + ignore = false + default: + return false, fmt.Errorf("row: invalid ignore value %q in fory tag", value) } } return ignore, nil } +// splitTopLevel splits on commas outside parentheses, so nested type +// hints such as type=map(key=string,value=int32) stay intact. +func splitTopLevel(input string) []string { + var parts []string + depth, start := 0, 0 + for i := 0; i < len(input); i++ { + switch input[i] { + case '(': + depth++ + case ')': + if depth > 0 { + depth-- + } + case ',': + if depth == 0 { + parts = append(parts, input[start:i]) + start = i + 1 + } + } + } + return append(parts, input[start:]) +} + +func indexTopLevel(input string, target byte) int { + depth := 0 + for i := 0; i < len(input); i++ { + switch input[i] { + case '(': + depth++ + case ')': + if depth > 0 { + depth-- + } + default: + if depth == 0 && input[i] == target { + return i + } + } + } + return -1 +} + func lowerFirst(s string) string { r, size := utf8.DecodeRuneInString(s) if !unicode.IsUpper(r) { diff --git a/go/fory/row/infer_test.go b/go/fory/row/infer_test.go index d62513eb07..3ae76df871 100644 --- a/go/fory/row/infer_test.go +++ b/go/fory/row/infer_test.go @@ -96,7 +96,10 @@ func TestInferTypeMapping(t *testing.T) { require.Equal(t, StringType{}, byName("s").Type) require.True(t, byName("s").Nullable) - require.Equal(t, BinaryType{}, byName("bin").Type) + bin := byName("bin").Type.(*ListType) + require.Equal(t, Int8Type{}, bin.Elem.Type, "[]byte is list, the Java byte[] model") + require.False(t, bin.Elem.Nullable) + require.True(t, byName("bin").Nullable) list := byName("l").Type.(*ListType) require.Equal(t, Int32Type{}, list.Elem.Type) @@ -107,6 +110,7 @@ func TestInferTypeMapping(t *testing.T) { require.Equal(t, StringType{}, m.Key.Type) require.False(t, m.Key.Nullable) require.Equal(t, Int64Type{}, m.Value.Type) + require.True(t, m.Value.Nullable, "map values are canonically nullable, as in Java and the schema parser") in := byName("in").Type.(*StructType) require.Equal(t, "x", in.Fields[0].Name) @@ -119,43 +123,76 @@ func TestInferTypeMapping(t *testing.T) { require.Equal(t, Date32Type{}, byName("d").Type) require.Equal(t, TimestampType{}, byName("t").Type) require.Equal(t, DurationType{}, byName("dur").Type) + for _, name := range []string{"d", "t", "dur"} { + require.True(t, byName(name).Nullable, "%s: Java temporal carriers are nullable objects", name) + } } -// Ignore semantics must match the core fory tag parser: "-", -// "ignore", and "ignore=true" skip; "ignore=false" keeps. +// Tag grammar must match the core fory tag parser: "-", "ignore", +// ignore=true/1/yes skip (case-insensitive, whitespace around '=' +// allowed); ignore=false/0/no keeps; other core keys are accepted and +// ignored; unknown keys, duplicate keys, and bad values are errors. func TestInferSkipsIgnoredAndUnexported(t *testing.T) { type tagged struct { A int64 hidden int64 Skipped string `fory:"ignore"` Dash string `fory:"-"` - True string `fory:"ignore=true"` + Yes string `fory:"ignore = YES"` + One string `fory:"id=3, ignore=1"` Kept int32 `fory:"ignore=false"` + Kept2 int32 `fory:"nullable=true, ignore=no"` + Kept3 int32 `fory:"type=map(key=string,value=int32)"` } s, err := InferSchema(reflect.TypeOf(tagged{})) require.NoError(t, err) - require.Equal(t, 2, s.NumFields()) + require.Equal(t, 4, s.NumFields()) require.Equal(t, "a", s.Field(0).Name) require.Equal(t, "kept", s.Field(1).Name) - - type badTag struct { - A int64 `fory:"ignore=yes"` + require.Equal(t, "kept2", s.Field(2).Name) + require.Equal(t, "kept3", s.Field(3).Name) + + for _, bad := range []any{ + struct { + A int64 `fory:"ignore=maybe"` + }{}, + struct { + A int64 `fory:"ignore, ignore"` + }{}, + struct { + A int64 `fory:"bogus=1"` + }{}, + } { + _, err := InferSchema(reflect.TypeOf(bad)) + require.Error(t, err, "%T", bad) } - _, err = InferSchema(reflect.TypeOf(badTag{})) - requireErrorContains(t, err, "invalid ignore value") } func TestInferRejectsUnsupportedTypes(t *testing.T) { type node struct { Next *node } + type recursiveList []recursiveList + type recursiveMap map[string]recursiveMap + type keyWithHidden struct { + A int32 + b int32 + } cases := []any{ struct{ U uint32 }{}, struct{ C chan int }{}, struct{ A [3]int32 }{}, struct{ PP **int32 }{}, + struct{ PS *[]int32 }{}, + struct{ PM *map[string]int32 }{}, + struct{ PB *[]byte }{}, struct{ M map[*string]int32 }{}, + struct{ M map[keyWithHidden]int32 }{}, + struct{ M map[time.Time]int32 }{}, + struct{ M map[[2]int32]int32 }{}, node{}, + struct{ L recursiveList }{}, + struct{ M recursiveMap }{}, } for _, c := range cases { _, err := InferSchema(reflect.TypeOf(c)) @@ -163,4 +200,21 @@ func TestInferRejectsUnsupportedTypes(t *testing.T) { } _, err := InferSchema(reflect.TypeOf(42)) require.Error(t, err) + _, err = InferSchema(nil) + require.Error(t, err) +} + +// Map keys whose encoded fields fully determine Go equality are +// accepted, including value structs of such fields. +func TestInferAcceptsEqualityPreservingMapKeys(t *testing.T) { + type point struct { + X, Y int32 + } + type keyed struct { + ByDate map[fory.Date]string + ByPoint map[point]string + ByDur map[time.Duration]string + } + _, err := InferSchema(reflect.TypeOf(keyed{})) + require.NoError(t, err) } diff --git a/go/fory/row/row.go b/go/fory/row/row.go index 77e03d819a..be6e0e30ef 100644 --- a/go/fory/row/row.go +++ b/go/fory/row/row.go @@ -30,11 +30,12 @@ import ( // from the underlying data without deserializing other fields; nested // values are sub-slice views, never copies (except String). // -// Fixed-width getters return the zero value for null fields; use -// IsNullAt to distinguish. Variable-width getters return nil (or "") -// for null fields. An out-of-range field index panics; offsets in -// corrupt or untrusted data may panic on slice bounds, so wrap -// untrusted decoding with recover. +// Row and its nested views require trusted, schema-matched bytes: the +// row format is a trusted in-memory format and these views do not +// validate the row graph. Fixed-width getters return the zero value for +// null fields; use IsNullAt to distinguish. Variable-width getters +// return nil (or "") for null fields. Out-of-range indexes and +// malformed bytes panic. type Row struct { fields []Field data []byte @@ -125,7 +126,7 @@ func (r *Row) Timestamp(i int) time.Time { } func (r *Row) Duration(i int) time.Duration { - return time.Duration(r.Int64(i)) * time.Microsecond + return durationFromMicros(r.Int64(i)) } // varData returns the value bytes of variable-width field i, or nil if @@ -180,7 +181,14 @@ type ArrayData struct { } func NewArrayData(elem Field, data []byte) *ArrayData { - numElements := int(binary.LittleEndian.Uint64(data)) + // Narrow the wire count only when it is far below int range, so + // header and size arithmetic cannot overflow on any platform; an + // absurd count becomes -1, which validateBounds and every index + // check reject. + numElements := -1 + if count := binary.LittleEndian.Uint64(data); count <= uint64(math.MaxInt/8) { + numElements = int(count) + } elemSize := elem.Type.ByteWidth() if elemSize < 0 { elemSize = 8 @@ -285,7 +293,7 @@ func (a *ArrayData) Timestamp(i int) time.Time { } func (a *ArrayData) Duration(i int) time.Duration { - return time.Duration(a.Int64(i)) * time.Microsecond + return durationFromMicros(a.Int64(i)) } func (a *ArrayData) varData(i int) []byte { @@ -333,7 +341,11 @@ type MapData struct { } func NewMapData(mapType *MapType, data []byte) *MapData { - keysSize := int(binary.LittleEndian.Uint64(data)) + keysSize64 := binary.LittleEndian.Uint64(data) + if keysSize64 > uint64(len(data)) { + panic(fmt.Sprintf("row: map keys array of %d bytes exceeds the %d-byte map region", keysSize64, len(data))) + } + keysSize := int(keysSize64) keysData := boundedSlice(data, 8, keysSize) valuesData := boundedSlice(data, 8+keysSize, len(data)-8-keysSize) return &MapData{ @@ -368,3 +380,13 @@ func dateFromDays(days int32) fory.Date { d, _ := fory.DateFromEpochDay(int64(days)) return d } + +// durationFromMicros converts wire microseconds to a time.Duration, +// which counts nanoseconds; values the multiplication would wrap are +// rejected instead of silently changing sign. +func durationFromMicros(micros int64) time.Duration { + if micros > math.MaxInt64/1000 || micros < math.MinInt64/1000 { + panic(fmt.Sprintf("row: duration of %d microseconds overflows time.Duration", micros)) + } + return time.Duration(micros) * time.Microsecond +} diff --git a/go/fory/row/row_test.go b/go/fory/row/row_test.go index 428b69449f..3559bb402a 100644 --- a/go/fory/row/row_test.go +++ b/go/fory/row/row_test.go @@ -18,6 +18,7 @@ package row import ( + "math" "strings" "sync" "testing" @@ -245,13 +246,47 @@ func TestOutOfRangeIndexPanics(t *testing.T) { // The wire format packs offset and size into 32 bits each; larger // values must be rejected, never silently truncated. func TestOffsetAndSizeRejectWireLimit(t *testing.T) { + if uint64(math.MaxInt) <= math.MaxUint32 { + t.Skip("int cannot exceed the 32-bit wire limit on this platform") + } + tooBig := int(math.MaxInt) w := NewRowWriter(int64StringSchema()) w.Reset() - require.Error(t, w.SetOffsetAndSize(1, w.Buffer().WriterIndex(), 1<<33)) + require.Error(t, w.SetOffsetAndSize(1, w.Buffer().WriterIndex(), tooBig)) + + aw := NewArrayWriter(List(StringType{}).Elem, w.Buffer()) + require.NoError(t, aw.Reset(1)) + require.Error(t, aw.SetOffsetAndSize(0, w.Buffer().WriterIndex(), tooBig)) +} +// Strings are UTF-8 on the wire; arbitrary Go byte strings are rejected +// so other runtimes never see replacement characters. +func TestWriteStringRejectsInvalidUTF8(t *testing.T) { + w := NewRowWriter(int64StringSchema()) + w.Reset() + require.Error(t, w.WriteString(1, string([]byte{0xff, 'a'}))) aw := NewArrayWriter(List(StringType{}).Elem, w.Buffer()) require.NoError(t, aw.Reset(1)) - require.Error(t, aw.SetOffsetAndSize(0, w.Buffer().WriterIndex(), 1<<33)) + require.Error(t, aw.WriteString(0, string([]byte{0xc3}))) +} + +// Timestamps outside the int64 microsecond range and durations whose +// nanosecond conversion would wrap are rejected instead of corrupted. +func TestTemporalRangeChecks(t *testing.T) { + s := NewSchema([]Field{ + NewField("ts", TimestampType{}, true), + NewField("dur", DurationType{}, true), + }) + w := NewRowWriter(s) + w.Reset() + farFuture := time.Date(300000, time.January, 1, 0, 0, 0, 0, time.UTC) + require.Error(t, w.WriteTimestamp(0, farFuture)) + require.NoError(t, w.WriteTimestamp(0, time.UnixMicro(math.MaxInt64))) + + w.WriteInt64(1, math.MaxInt64) // raw microseconds that overflow time.Duration + r := NewRow(s, w.ToBytes()) + require.Equal(t, int64(math.MaxInt64), r.Timestamp(0).UnixMicro()) + require.Panics(t, func() { r.Duration(1) }) } func TestConcurrentReads(t *testing.T) { diff --git a/go/fory/row/writer.go b/go/fory/row/writer.go index dd898644de..ecc5080a99 100644 --- a/go/fory/row/writer.go +++ b/go/fory/row/writer.go @@ -22,6 +22,7 @@ import ( "fmt" "math" "time" + "unicode/utf8" fory "github.com/apache/fory/go/fory" ) @@ -151,15 +152,26 @@ func (w *RowWriter) WriteDate(i int, d fory.Date) error { return nil } -func (w *RowWriter) WriteTimestamp(i int, t time.Time) { - w.WriteInt64(i, t.UnixMicro()) +func (w *RowWriter) WriteTimestamp(i int, t time.Time) error { + micros, err := timestampMicros(t) + if err != nil { + return err + } + w.WriteInt64(i, micros) + return nil } func (w *RowWriter) WriteDuration(i int, d time.Duration) { w.WriteInt64(i, d.Microseconds()) } +// WriteString writes a UTF-8 string; the row format string type is +// UTF-8, so invalid byte sequences are rejected rather than changed by +// other runtimes on read. func (w *RowWriter) WriteString(i int, s string) error { + if !utf8.ValidString(s) { + return errInvalidUTF8 + } start := appendStringRegion(w.buf, s) return w.SetOffsetAndSize(i, start, len(s)) } @@ -211,10 +223,12 @@ func (w *ArrayWriter) Reset(numElements int) error { if numElements < 0 { return fmt.Errorf("row: negative array length %d", numElements) } - dataBytes := int64(numElements) * int64(w.elemSize) - if dataBytes > maxArrayDataBytes { + // Compare before multiplying so extreme counts cannot overflow the + // size computation and slip past the limit. + if numElements > maxArrayDataBytes/w.elemSize { return fmt.Errorf("row: array of %d elements exceeds maximum size", numElements) } + dataBytes := int64(numElements) * int64(w.elemSize) headerBytes := 8 + bitmapWidthInBytes(numElements) total := headerBytes + roundToWord(int(dataBytes)) base := w.buf.WriterIndex() @@ -293,8 +307,13 @@ func (w *ArrayWriter) WriteDate(i int, d fory.Date) error { return nil } -func (w *ArrayWriter) WriteTimestamp(i int, t time.Time) { - w.WriteInt64(i, t.UnixMicro()) +func (w *ArrayWriter) WriteTimestamp(i int, t time.Time) error { + micros, err := timestampMicros(t) + if err != nil { + return err + } + w.WriteInt64(i, micros) + return nil } func (w *ArrayWriter) WriteDuration(i int, d time.Duration) { @@ -302,6 +321,9 @@ func (w *ArrayWriter) WriteDuration(i int, d time.Duration) { } func (w *ArrayWriter) WriteString(i int, s string) error { + if !utf8.ValidString(s) { + return errInvalidUTF8 + } start := appendStringRegion(w.buf, s) return w.SetOffsetAndSize(i, start, len(s)) } @@ -368,6 +390,24 @@ func appendBytesRegion(buf *fory.ByteBuffer, b []byte) int { return start } +var errInvalidUTF8 = fmt.Errorf("row: string is not valid UTF-8") + +// timestampMicros converts t to microseconds since the Unix epoch with +// overflow checking; time.Time.UnixMicro is undefined outside the +// int64 microsecond range. +func timestampMicros(t time.Time) (int64, error) { + sec := t.Unix() + if sec > math.MaxInt64/1_000_000 || sec < math.MinInt64/1_000_000 { + return 0, fmt.Errorf("row: timestamp %v is outside the int64 microsecond range", t) + } + micros := sec * 1_000_000 + subMicros := int64(t.Nanosecond()) / 1000 + if micros > math.MaxInt64-subMicros { + return 0, fmt.Errorf("row: timestamp %v is outside the int64 microsecond range", t) + } + return micros + subMicros, nil +} + func appendStringRegion(buf *fory.ByteBuffer, s string) int { n := len(s) rounded := roundToWord(n) From 98ac90ad279d810d6d74ce123fe76ffb4dba0189 Mon Sep 17 00:00:00 2001 From: ayush00git Date: Sat, 22 Aug 2026 22:47:49 +0530 Subject: [PATCH 19/20] test(go): cover byte arrays in the Go row cross-language test and fail on setup errors Setting FORY_GO_JAVA_CI=1 opts into GoCrossLanguageTest; once opted in, a missing Go toolchain or a failed peer build now fails the suite instead of skipping it, so a peer-only compile regression cannot leave CI green. A new case exchanges a Java byte[] field with a Go []byte field, pinning the shared list model. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01MwqYMrDU1kD1wcB9snj9mh --- go/fory/tests/row_xlang/row_xlang_main.go | 25 +++++++++ .../fory/format/GoCrossLanguageTest.java | 54 +++++++++++++------ 2 files changed, 62 insertions(+), 17 deletions(-) diff --git a/go/fory/tests/row_xlang/row_xlang_main.go b/go/fory/tests/row_xlang/row_xlang_main.go index f8491cdc17..776eda2410 100644 --- a/go/fory/tests/row_xlang/row_xlang_main.go +++ b/go/fory/tests/row_xlang/row_xlang_main.go @@ -45,6 +45,13 @@ type bar struct { F2 string } +// Mirrors GoCrossLanguageTest.Blob: Java byte[] and Go []byte are both +// list. +type blob struct { + F1 []byte + F2 string +} + // Mirrors CrossLanguageTest.Foo. F3 uses []*string because the Java // fixture contains a null list element. type foo struct { @@ -69,6 +76,8 @@ func main() { fail("test_serialization_with_schema needs ") } testSerializationWithSchema(os.Args[2], os.Args[3]) + case "test_byte_array_carrier": + testByteArrayCarrier(os.Args[2]) default: fail("unknown test case %q", caseName) } @@ -90,6 +99,22 @@ func testMapEncoder(dataFile string) { must(os.WriteFile(dataFile, encoded, 0o644)) } +func testByteArrayCarrier(dataFile string) { + encoder, err := row.NewEncoder[blob]() + must(err) + data, err := os.ReadFile(dataFile) + must(err) + + decoded, err := encoder.Decode(data) + must(err) + expected := blob{F1: []byte{0, 1, 0xff, 127, 0x80}, F2: "bytes"} + check(reflect.DeepEqual(decoded, expected), "decoded %+v, expected %+v", decoded, expected) + + encoded, err := encoder.Encode(&expected) + must(err) + must(os.WriteFile(dataFile, encoded, 0o644)) +} + func testSerializationWithoutSchema(dataFile string) { encoder, err := row.NewEncoder[foo]() must(err) diff --git a/java/fory-format/src/test/java/org/apache/fory/format/GoCrossLanguageTest.java b/java/fory-format/src/test/java/org/apache/fory/format/GoCrossLanguageTest.java index 006a02a977..1c74f7225e 100644 --- a/java/fory-format/src/test/java/org/apache/fory/format/GoCrossLanguageTest.java +++ b/java/fory-format/src/test/java/org/apache/fory/format/GoCrossLanguageTest.java @@ -28,6 +28,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import lombok.Data; import org.apache.fory.format.encoder.Encoders; import org.apache.fory.format.encoder.RowEncoder; import org.apache.fory.format.row.binary.BinaryRow; @@ -43,7 +44,8 @@ /** * Row format cross-language tests against a Go peer built from {@code go/fory/tests/row_xlang}. * Data shapes are shared with {@link CrossLanguageTest} so the Java, Python, and Go peers exercise - * the same schemas. + * the same schemas. Setting {@code FORY_GO_JAVA_CI=1} opts into the suite; once opted in, a missing + * Go toolchain or a peer build failure fails the tests instead of skipping them. */ @Test public class GoCrossLanguageTest { @@ -57,31 +59,49 @@ public void ensureGoPeerReady() { if (!"1".equals(enabled)) { throw new SkipException("Skipping GoCrossLanguageTest: FORY_GO_JAVA_CI not set to 1"); } - boolean goInstalled = true; try { Process process = new ProcessBuilder("go", "version").start(); - if (process.waitFor() != 0) { - goInstalled = false; - } - } catch (IOException | InterruptedException e) { - goInstalled = false; - if (e instanceof InterruptedException) { - Thread.currentThread().interrupt(); - } - } - if (!goInstalled) { - throw new SkipException("Skipping GoCrossLanguageTest: go not installed"); + Assert.assertEquals(process.waitFor(), 0, "go toolchain is required when FORY_GO_JAVA_CI=1"); + } catch (IOException e) { + throw new AssertionError("go toolchain is required when FORY_GO_JAVA_CI=1", e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError("interrupted while probing the go toolchain", e); } List buildCommand = Arrays.asList("go", "build", "-o", "tests/" + GO_BINARY, "./tests/row_xlang"); - boolean buildSuccess = + Assert.assertTrue( TestUtils.executeCommand( - buildCommand, 120, Collections.emptyMap(), new File("../../go/fory")); - if (!buildSuccess || !new File("../../go/fory/tests/" + GO_BINARY).exists()) { - throw new SkipException("Skipping GoCrossLanguageTest: failed to build " + GO_BINARY); + buildCommand, 120, Collections.emptyMap(), new File("../../go/fory")), + "failed to build the Go row format peer " + GO_BINARY); + Assert.assertTrue( + new File("../../go/fory/tests/" + GO_BINARY).exists(), + GO_BINARY + " not found after a successful build"); + } + + /** Keep in sync with {@code blob} in row_xlang_main.go: byte[] is list in both. */ + @Data + public static class Blob { + public byte[] f1; + public String f2; + + public static Blob create() { + Blob blob = new Blob(); + blob.f1 = new byte[] {0, 1, -1, 127, -128}; + blob.f2 = "bytes"; + return blob; } } + public void testByteArrayCarrier() throws IOException { + Blob blob = Blob.create(); + RowEncoder encoder = Encoders.bean(Blob.class); + Path dataFile = createTempFile("row_go_blob"); + Files.write(dataFile, encoder.encode(blob)); + Assert.assertTrue(runGoPeer("test_byte_array_carrier", dataFile)); + Assert.assertEquals(encoder.decode(Files.readAllBytes(dataFile)), blob); + } + public void testMapEncoder() throws IOException { CrossLanguageTest.A a = CrossLanguageTest.A.create(); RowEncoder encoder = Encoders.bean(CrossLanguageTest.A.class); From cc57b039c1daeacc4340f8f03a8c4ace899d519a Mon Sep 17 00:00:00 2001 From: ayush00git Date: Sat, 22 Aug 2026 22:47:49 +0530 Subject: [PATCH 20/20] docs(go): add the Go row format guide and update the support matrix Document the Go Standard Row Format: encoder construction, bare rows versus framed messages, zero-copy views and their lifetimes, nullability and pointer carriers, field ordering and naming, supported types, schema exchange, hand-written rows, and thread safety. List Go in the row format support matrix, the specification's language table, and the README with a Go example. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01MwqYMrDU1kD1wcB9snj9mh --- README.md | 23 ++- docs/row-format/go.md | 258 ++++++++++++++++++++++++++ docs/row-format/index.md | 15 +- docs/row-format/troubleshooting.md | 2 +- docs/specification/row_format_spec.md | 10 +- 5 files changed, 294 insertions(+), 14 deletions(-) create mode 100644 docs/row-format/go.md diff --git a/README.md b/README.md index 470f1bdab9..e65694ad95 100644 --- a/README.md +++ b/README.md @@ -800,11 +800,32 @@ ArrayData scores = scoresField.get(row); int secondScore = scores.getInt32(1); ``` +**Go** + +```go +import "github.com/apache/fory/go/fory/row" + +type User struct { + Id int32 + Name string + Scores []int32 +} + +encoder, _ := row.NewEncoder[User]() +binary, _ := encoder.ToRow(&User{Id: 1, Name: "Alice", Scores: []int32{98, 100, 95}}) + +schema := encoder.Schema() +r := row.NewRow(schema, binary) +name := r.String(schema.FieldIndex("name")) +secondScore := r.Array(schema.FieldIndex("scores")).Int32(1) +``` + For Java imports, nested structs, arrays/maps, Arrow integration, and partial deserialization, see the [Java row-format guide](docs/row-format/java.md), [Python row-format guide](docs/row-format/python.md), [C++ row-format guide](docs/row-format/cpp.md), -[Rust row-format guide](docs/row-format/rust.md), and +[Rust row-format guide](docs/row-format/rust.md), +[Go row-format guide](docs/row-format/go.md), and [row-format specification](docs/specification/row_format_spec.md). ## Fory JSON diff --git a/docs/row-format/go.md b/docs/row-format/go.md new file mode 100644 index 0000000000..2635f46dad --- /dev/null +++ b/docs/row-format/go.md @@ -0,0 +1,258 @@ +--- +title: Go Standard Row Format +sidebar_position: 7 +id: go +license: | + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--- + +Apache Fory™ Go implements the Standard Row Format used by Java, C++, Python, and Rust in the +`github.com/apache/fory/go/fory/row` package. It provides a reflection-based struct encoder, +zero-copy readers for random field access, and the cross-language schema encoding. + +## Overview + +Use Row Format when readers need selected fields or collection elements rather than a fully +reconstructed value. Readers are views over the encoded bytes: reading one field costs a bitmap +test and a slot lookup, and other fields are never touched. + +Row Format is a trusted in-memory format. Decode only bytes produced by a Fory row writer for the +same schema, from a source you trust. Malformed bytes surface as errors from the encoder's decode +methods, but the format does not defend against hostile input the way object serialization does. + +## When to Use Row Format + +- Analytics workloads with selective field access +- Large datasets where only a subset of fields is needed +- Memory-mapped or shared data read by several languages +- High-throughput pipelines that exchange Standard Row bytes with Java, C++, Python, or Rust + +## Basic Usage + +```go +package main + +import ( + "fmt" + + "github.com/apache/fory/go/fory/row" +) + +type UserProfile struct { + Id int64 + Username string + Email *string + Scores []int32 + Preferences map[string]string + IsActive bool +} + +func main() { + encoder, err := row.NewEncoder[UserProfile]() + if err != nil { + panic(err) + } + + email := "alice@example.com" + profile := UserProfile{ + Id: 12345, + Username: "alice", + Email: &email, + Scores: []int32{95, 87, 92, 88}, + Preferences: map[string]string{"theme": "dark", "language": "en"}, + IsActive: true, + } + + rowBytes, err := encoder.ToRow(&profile) + if err != nil { + panic(err) + } + + // Random access without decoding the whole struct. + schema := encoder.Schema() + r := row.NewRow(schema, rowBytes) + fmt.Println(r.String(schema.FieldIndex("username"))) // alice + fmt.Println(r.Array(schema.FieldIndex("scores")).Int32(1)) // 87 + fmt.Println(r.IsNullAt(schema.FieldIndex("email"))) // false + + // Full decode when the whole value is needed. + decoded, err := encoder.FromRow(rowBytes) + if err != nil { + panic(err) + } + fmt.Println(decoded.Preferences["theme"]) // dark +} +``` + +`NewEncoder[T]` infers the schema from the struct type once and compiles the conversion for it. +Creating an encoder is comparatively expensive; create one per struct type and reuse it. + +## Rows and Framed Messages + +`ToRow` and `FromRow` work with bare row bytes, which is what the other languages' `toRow` and +`BinaryRow.pointTo` exchange. `Encode` and `Decode` add the framing used by the Java and Python +row encoders: an 8-byte little-endian schema hash followed by the row. + +```go +framed, err := encoder.Encode(&profile) // hash + row +decoded, err := encoder.Decode(framed) // verifies the hash, then decodes +``` + +The hash is a type-shape fingerprint: it folds the recursive field type ids and nothing else, so +`Decode` rejects a writer whose field types differ but cannot detect renamed fields, changed +nullability, or reordered fields of the same type. Share the schema bytes (see +[Schema Exchange](#schema-exchange)) when peers must agree on more than the type shape. + +`Decode` and `FromRow` return errors for truncated or inconsistent bytes; they never panic. Decoded +values never alias the input, so the input buffer can be reused immediately. + +## Zero-Copy Reading + +`row.NewRow(schema, bytes)` creates a view over a row. `Struct`, `Array`, and `Map` return views +over the nested bytes, and `Binary` returns a sub-slice of the input. These views stay valid only +while the underlying bytes are alive and unmodified. `String` copies, because a Go string must not +observe later changes to the buffer. + +```go +r := row.NewRow(schema, rowBytes) +tags := r.Array(schema.FieldIndex("tags")) +for i := 0; i < tags.NumElements(); i++ { + if !tags.IsNullAt(i) { + fmt.Println(tags.String(i)) + } +} +attrs := r.Map(schema.FieldIndex("attrs")) +fmt.Println(attrs.Keys().String(0), attrs.Values().Int32(0)) +``` + +Fixed-width getters return the zero value for null fields; use `IsNullAt` to distinguish null from +zero. Variable-width getters return `nil` (or `""`) for null. Out-of-range indexes panic, as does +malformed data read through a raw view; use the encoder's decode methods when an error is needed. + +## Nullability + +A Go pointer field is a nullable field: `nil` writes the null bit and decodes back to `nil`. +Slices, maps, and `[]byte` are nullable as well and distinguish `nil` from empty. + +Strings, nested value structs, `fory.Date`, `time.Time`, and `time.Duration` are also nullable in +the schema because their Java carriers are objects, but the Go value cannot hold `nil`. Decoding a +null into one of these fields is an error. Use a pointer carrier (`*string`, `*time.Time`, `*Inner`) +when nulls must round-trip, for example when reading rows written by Java with `null` values. + +Map values are always nullable and map keys never are, matching the other languages. A `[]*int32` +element or `map[string]*int32` value carries a null element; `[]int32` and `map[string]int32` do +not. + +## Field Order and Names + +Fields are sorted by their lowerCamel name and named by its snake_case form (`UserName` becomes +`user_name`), matching Java's schema inference so both languages derive the same schema from +equivalent struct definitions. Unexported fields are skipped. The `fory` struct tag uses the same +grammar as object serialization: `fory:"-"`, `fory:"ignore"`, and `fory:"ignore=true"` skip a +field; other keys are accepted and ignored by Row Format. + +Changing a field name or type changes the schema. Coordinate such changes across all producers and +consumers. + +## Supported Types + +| Go type | Standard Row Format encoding | Nullable | +| --------------------------------------------- | -------------------------------- | -------- | +| `bool`, `int8`, `int16`, `int32`, `int64` | Fixed-width scalar | No | +| `int` | Fixed-width int64 | No | +| `float32`, `float64` | Fixed-width IEEE 754 scalar | No | +| `fory.Date` | Fixed-width date32 in epoch days | Yes | +| `time.Time` | Fixed-width epoch microseconds | Yes | +| `time.Duration` | Fixed-width microseconds | Yes | +| `string` | Variable-width UTF-8 | Yes | +| `[]byte` | Standard array of int8 | Yes | +| `[]T` for supported element types | Standard array | Yes | +| `map[K]V` | Standard map | Yes | +| Nested struct | Nested Standard Row | Yes | +| `*T` for any supported non-slice, non-map `T` | Same encoding as `T` | Yes | + +`[]byte` matches Java's `byte[]`, which Java also infers as a list of int8. The row format's +binary type is available only to hand-built schemas through `RowWriter.WriteBytes` and +`Row.Binary`. + +Strings must be valid UTF-8. Timestamps must fit in an int64 number of microseconds. Map keys must +be scalars, strings, or value structs whose exported, non-ignored fields consist of such types, so +that the encoded key determines Go equality; `time.Time` and pointers are not valid keys. + +Unsupported: unsigned integers, fixed-size arrays, nested pointers, pointers to slices or maps, +interfaces, channels, functions, recursive types, `float16`, and `decimal`. + +## Schema Exchange + +`SchemaToBytes` and `SchemaFromBytes` implement the cross-language schema encoding shared with +Java's `SchemaEncoder` and Python's `Schema.to_bytes`/`from_bytes`. `ComputeSchemaHash` computes +the same type-shape hash used by `Encode`. + +```go +schemaBytes, err := row.SchemaToBytes(encoder.Schema()) +schema, err := row.SchemaFromBytes(schemaBytes) +fmt.Println(schema.Equal(encoder.Schema())) // true +``` + +A Java bean and a Go struct with equivalent fields produce identical schema bytes when their +nullability matches: use pointer fields for Java boxed types (`Integer`, `String` in lists) and +value fields for Java primitives. + +## Writing Rows by Hand + +`RowWriter`, `ArrayWriter`, and `MapWriter` write rows for a schema you construct yourself, without +a Go struct. They share one `fory.ByteBuffer`; a nested value is written at the buffer's current +position and then attached to its parent slot with `SetOffsetAndSize`. + +```go +schema := row.NewSchema([]row.Field{ + row.NewField("id", row.Int64Type{}, false), + row.NewField("tags", row.List(row.StringType{}), true), +}) +w := row.NewRowWriter(schema) +w.Reset() +w.WriteInt64(0, 7) + +tags := row.NewArrayWriter(row.List(row.StringType{}).Elem, w.Buffer()) +start := w.Buffer().WriterIndex() +if err := tags.Reset(2); err != nil { + panic(err) +} +if err := tags.WriteString(0, "go"); err != nil { + panic(err) +} +tags.SetNullAt(1) +if err := w.SetOffsetAndSize(1, start, w.Buffer().WriterIndex()-start); err != nil { + panic(err) +} +rowBytes := w.ToBytes() // valid until the buffer is written to again +``` + +Call `Reset` before each row; to reuse a writer for a new top-level row, set the buffer's writer +index back to zero first. + +## Thread Safety + +An `Encoder` owns a reusable write buffer and is not safe for concurrent use; create one encoder +per goroutine or guard it with a mutex. Writers share the same rule. `Row`, `ArrayData`, and +`MapData` views only read, so one view can be shared by concurrent readers as long as the +underlying bytes are not modified. + +## Related Topics + +- [Basic Serialization](../object-serialization/go/basic-serialization.md) - Object graph serialization +- [Standard Row Format](index.md#standard-row) - Shared layout for Java, Python, C++, Rust, and Go +- [Row Format Specification](../specification/row_format_spec.md) - Protocol details diff --git a/docs/row-format/index.md b/docs/row-format/index.md index 3375195166..ac41122a90 100644 --- a/docs/row-format/index.md +++ b/docs/row-format/index.md @@ -30,20 +30,20 @@ reconstruction as its primary access pattern. ## Choose a Layout -| Layout | Language support | Compatibility | -| ------------ | ----------------------- | -------------------------------- | -| Standard Row | Java, Python, C++, Rust | Shared Standard Row layout | -| Compact Row | Java | Java-only, space-oriented layout | +| Layout | Language support | Compatibility | +| ------------ | --------------------------- | -------------------------------- | +| Standard Row | Java, Python, C++, Rust, Go | Shared Standard Row layout | +| Compact Row | Java | Java-only, space-oriented layout | ## Standard Row -Standard Row is the interoperable layout for Java, Python, C++, and Rust. +Standard Row is the interoperable layout for Java, Python, C++, Rust, and Go. ### Features - **Zero-copy random access**: Read selected fields directly from encoded data. - **Partial deserialization**: Reconstruct only the values an application needs. -- **Cross-language compatibility**: Share Standard Row bytes between Java, Python, C++, and Rust. +- **Cross-language compatibility**: Share Standard Row bytes between Java, Python, C++, Rust, and Go. - **Apache Arrow integration**: Convert rows to Arrow data in Java and Python. ### Layout @@ -61,6 +61,7 @@ layout, alignment rules, type table, and endianness are defined by the | Python | Compatible | [Python](python.md) | PyArrow schema and table conversion | | C++ | Compatible | [C++](cpp.md) | Native row readers and writers | | Rust | Compatible | [Rust](rust.md) | Borrowed struct, array, and map views | +| Go | Compatible | [Go](go.md) | Reflection struct encoder; zero-copy row views | Use the language guides for installation, schema construction, encoding, random access, partial reads, and language-specific integrations. @@ -93,7 +94,7 @@ Reuse the encoder within one thread. Create separate encoders for concurrent thr - Fixed-size nested structs can be stored inline. Choose Compact Row only when every reader is Java and the space reduction justifies the -Java-specific layout. Use Standard Row for Java, Python, C++, and Rust interchange. +Java-specific layout. Use Standard Row for Java, Python, C++, Rust, and Go interchange. See the [Row Format specification](../specification/row_format_spec.md) for the exact Standard and Compact binary layouts. diff --git a/docs/row-format/troubleshooting.md b/docs/row-format/troubleshooting.md index c5fe6330d3..34f51d1c6d 100644 --- a/docs/row-format/troubleshooting.md +++ b/docs/row-format/troubleshooting.md @@ -21,7 +21,7 @@ license: | ## A Standard Row peer cannot read Compact Row bytes -Compact Row is a Java-only row family. Use Standard Row on Java, Python, C++, and Rust for shared +Compact Row is a Java-only row family. Use Standard Row on Java, Python, C++, Rust, and Go for shared bytes. ## A field lookup fails diff --git a/docs/specification/row_format_spec.md b/docs/specification/row_format_spec.md index e005634601..a192e35f1e 100644 --- a/docs/specification/row_format_spec.md +++ b/docs/specification/row_format_spec.md @@ -26,14 +26,14 @@ Apache Fory Row Format is a cache-friendly, random-access binary format designed - **Random Field Access**: Read individual fields without deserializing the entire row - **Zero-Copy Operations**: Direct memory access without data transformation - **Cache-Friendly Layout**: Optimized memory layout for CPU cache efficiency -- **Cross-Language Support**: Consistent binary format across Java, C++, Python, and Rust +- **Cross-Language Support**: Consistent binary format across Java, C++, Python, Rust, and Go Fory provides two row format variants: -| Format | Languages | Use Case | -| --------------- | ----------------------- | ------------------------------ | -| Standard Format | Java, C++, Python, Rust | Cross-language compatibility | -| Compact Format | Java only | Space efficiency, smaller rows | +| Format | Languages | Use Case | +| --------------- | --------------------------- | ------------------------------ | +| Standard Format | Java, C++, Python, Rust, Go | Cross-language compatibility | +| Compact Format | Java only | Space efficiency, smaller rows | ## Format Comparison