diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d7a407b43a..14dc991338 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1692,6 +1692,8 @@ jobs: mvn -T16 --no-transfer-progress clean install -DskipTests -Dmaven.javadoc.skip=true -Dmaven.source.skip=true cd fory-core mvn -T16 --no-transfer-progress test -Dtest=org.apache.fory.xlang.GoXlangTest + cd ../fory-format + mvn -T16 --no-transfer-progress test -Dtest=org.apache.fory.format.GoCrossLanguageTest - name: Run Go IDL Tests run: ./integration_tests/idl_tests/run_go_tests.sh diff --git a/.gitignore b/.gitignore index fcd6e5a199..74350c695c 100644 --- a/.gitignore +++ b/.gitignore @@ -59,6 +59,8 @@ go/fory/tests/xlang_test_main go/fory/tests/xlang/xlang_test_main go/fory/xlang_test_main go/fory/xlang +go/fory/tests/row_xlang_bin +go/fory/tests/row_xlang_bin.exe # Build directories for all languages # Java diff --git a/README.md b/README.md index 470f1bdab9..e65694ad95 100644 --- a/README.md +++ b/README.md @@ -800,11 +800,32 @@ ArrayData scores = scoresField.get(row); int secondScore = scores.getInt32(1); ``` +**Go** + +```go +import "github.com/apache/fory/go/fory/row" + +type User struct { + Id int32 + Name string + Scores []int32 +} + +encoder, _ := row.NewEncoder[User]() +binary, _ := encoder.ToRow(&User{Id: 1, Name: "Alice", Scores: []int32{98, 100, 95}}) + +schema := encoder.Schema() +r := row.NewRow(schema, binary) +name := r.String(schema.FieldIndex("name")) +secondScore := r.Array(schema.FieldIndex("scores")).Int32(1) +``` + For Java imports, nested structs, arrays/maps, Arrow integration, and partial deserialization, see the [Java row-format guide](docs/row-format/java.md), [Python row-format guide](docs/row-format/python.md), [C++ row-format guide](docs/row-format/cpp.md), -[Rust row-format guide](docs/row-format/rust.md), and +[Rust row-format guide](docs/row-format/rust.md), +[Go row-format guide](docs/row-format/go.md), and [row-format specification](docs/specification/row_format_spec.md). ## Fory JSON diff --git a/ci/tasks/go.py b/ci/tasks/go.py index 3df17f81f2..109013cf42 100644 --- a/ci/tasks/go.py +++ b/ci/tasks/go.py @@ -24,4 +24,5 @@ def run(): logging.info("Executing fory go tests") common.cd_project_subdir("go/fory") common.exec_cmd("go test -v") + common.exec_cmd("go test -v ./row/...") logging.info("Executing fory go tests succeeds") diff --git a/docs/row-format/go.md b/docs/row-format/go.md new file mode 100644 index 0000000000..2635f46dad --- /dev/null +++ b/docs/row-format/go.md @@ -0,0 +1,258 @@ +--- +title: Go Standard Row Format +sidebar_position: 7 +id: go +license: | + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--- + +Apache Fory™ Go implements the Standard Row Format used by Java, C++, Python, and Rust in the +`github.com/apache/fory/go/fory/row` package. It provides a reflection-based struct encoder, +zero-copy readers for random field access, and the cross-language schema encoding. + +## Overview + +Use Row Format when readers need selected fields or collection elements rather than a fully +reconstructed value. Readers are views over the encoded bytes: reading one field costs a bitmap +test and a slot lookup, and other fields are never touched. + +Row Format is a trusted in-memory format. Decode only bytes produced by a Fory row writer for the +same schema, from a source you trust. Malformed bytes surface as errors from the encoder's decode +methods, but the format does not defend against hostile input the way object serialization does. + +## When to Use Row Format + +- Analytics workloads with selective field access +- Large datasets where only a subset of fields is needed +- Memory-mapped or shared data read by several languages +- High-throughput pipelines that exchange Standard Row bytes with Java, C++, Python, or Rust + +## Basic Usage + +```go +package main + +import ( + "fmt" + + "github.com/apache/fory/go/fory/row" +) + +type UserProfile struct { + Id int64 + Username string + Email *string + Scores []int32 + Preferences map[string]string + IsActive bool +} + +func main() { + encoder, err := row.NewEncoder[UserProfile]() + if err != nil { + panic(err) + } + + email := "alice@example.com" + profile := UserProfile{ + Id: 12345, + Username: "alice", + Email: &email, + Scores: []int32{95, 87, 92, 88}, + Preferences: map[string]string{"theme": "dark", "language": "en"}, + IsActive: true, + } + + rowBytes, err := encoder.ToRow(&profile) + if err != nil { + panic(err) + } + + // Random access without decoding the whole struct. + schema := encoder.Schema() + r := row.NewRow(schema, rowBytes) + fmt.Println(r.String(schema.FieldIndex("username"))) // alice + fmt.Println(r.Array(schema.FieldIndex("scores")).Int32(1)) // 87 + fmt.Println(r.IsNullAt(schema.FieldIndex("email"))) // false + + // Full decode when the whole value is needed. + decoded, err := encoder.FromRow(rowBytes) + if err != nil { + panic(err) + } + fmt.Println(decoded.Preferences["theme"]) // dark +} +``` + +`NewEncoder[T]` infers the schema from the struct type once and compiles the conversion for it. +Creating an encoder is comparatively expensive; create one per struct type and reuse it. + +## Rows and Framed Messages + +`ToRow` and `FromRow` work with bare row bytes, which is what the other languages' `toRow` and +`BinaryRow.pointTo` exchange. `Encode` and `Decode` add the framing used by the Java and Python +row encoders: an 8-byte little-endian schema hash followed by the row. + +```go +framed, err := encoder.Encode(&profile) // hash + row +decoded, err := encoder.Decode(framed) // verifies the hash, then decodes +``` + +The hash is a type-shape fingerprint: it folds the recursive field type ids and nothing else, so +`Decode` rejects a writer whose field types differ but cannot detect renamed fields, changed +nullability, or reordered fields of the same type. Share the schema bytes (see +[Schema Exchange](#schema-exchange)) when peers must agree on more than the type shape. + +`Decode` and `FromRow` return errors for truncated or inconsistent bytes; they never panic. Decoded +values never alias the input, so the input buffer can be reused immediately. + +## Zero-Copy Reading + +`row.NewRow(schema, bytes)` creates a view over a row. `Struct`, `Array`, and `Map` return views +over the nested bytes, and `Binary` returns a sub-slice of the input. These views stay valid only +while the underlying bytes are alive and unmodified. `String` copies, because a Go string must not +observe later changes to the buffer. + +```go +r := row.NewRow(schema, rowBytes) +tags := r.Array(schema.FieldIndex("tags")) +for i := 0; i < tags.NumElements(); i++ { + if !tags.IsNullAt(i) { + fmt.Println(tags.String(i)) + } +} +attrs := r.Map(schema.FieldIndex("attrs")) +fmt.Println(attrs.Keys().String(0), attrs.Values().Int32(0)) +``` + +Fixed-width getters return the zero value for null fields; use `IsNullAt` to distinguish null from +zero. Variable-width getters return `nil` (or `""`) for null. Out-of-range indexes panic, as does +malformed data read through a raw view; use the encoder's decode methods when an error is needed. + +## Nullability + +A Go pointer field is a nullable field: `nil` writes the null bit and decodes back to `nil`. +Slices, maps, and `[]byte` are nullable as well and distinguish `nil` from empty. + +Strings, nested value structs, `fory.Date`, `time.Time`, and `time.Duration` are also nullable in +the schema because their Java carriers are objects, but the Go value cannot hold `nil`. Decoding a +null into one of these fields is an error. Use a pointer carrier (`*string`, `*time.Time`, `*Inner`) +when nulls must round-trip, for example when reading rows written by Java with `null` values. + +Map values are always nullable and map keys never are, matching the other languages. A `[]*int32` +element or `map[string]*int32` value carries a null element; `[]int32` and `map[string]int32` do +not. + +## Field Order and Names + +Fields are sorted by their lowerCamel name and named by its snake_case form (`UserName` becomes +`user_name`), matching Java's schema inference so both languages derive the same schema from +equivalent struct definitions. Unexported fields are skipped. The `fory` struct tag uses the same +grammar as object serialization: `fory:"-"`, `fory:"ignore"`, and `fory:"ignore=true"` skip a +field; other keys are accepted and ignored by Row Format. + +Changing a field name or type changes the schema. Coordinate such changes across all producers and +consumers. + +## Supported Types + +| Go type | Standard Row Format encoding | Nullable | +| --------------------------------------------- | -------------------------------- | -------- | +| `bool`, `int8`, `int16`, `int32`, `int64` | Fixed-width scalar | No | +| `int` | Fixed-width int64 | No | +| `float32`, `float64` | Fixed-width IEEE 754 scalar | No | +| `fory.Date` | Fixed-width date32 in epoch days | Yes | +| `time.Time` | Fixed-width epoch microseconds | Yes | +| `time.Duration` | Fixed-width microseconds | Yes | +| `string` | Variable-width UTF-8 | Yes | +| `[]byte` | Standard array of int8 | Yes | +| `[]T` for supported element types | Standard array | Yes | +| `map[K]V` | Standard map | Yes | +| Nested struct | Nested Standard Row | Yes | +| `*T` for any supported non-slice, non-map `T` | Same encoding as `T` | Yes | + +`[]byte` matches Java's `byte[]`, which Java also infers as a list of int8. The row format's +binary type is available only to hand-built schemas through `RowWriter.WriteBytes` and +`Row.Binary`. + +Strings must be valid UTF-8. Timestamps must fit in an int64 number of microseconds. Map keys must +be scalars, strings, or value structs whose exported, non-ignored fields consist of such types, so +that the encoded key determines Go equality; `time.Time` and pointers are not valid keys. + +Unsupported: unsigned integers, fixed-size arrays, nested pointers, pointers to slices or maps, +interfaces, channels, functions, recursive types, `float16`, and `decimal`. + +## Schema Exchange + +`SchemaToBytes` and `SchemaFromBytes` implement the cross-language schema encoding shared with +Java's `SchemaEncoder` and Python's `Schema.to_bytes`/`from_bytes`. `ComputeSchemaHash` computes +the same type-shape hash used by `Encode`. + +```go +schemaBytes, err := row.SchemaToBytes(encoder.Schema()) +schema, err := row.SchemaFromBytes(schemaBytes) +fmt.Println(schema.Equal(encoder.Schema())) // true +``` + +A Java bean and a Go struct with equivalent fields produce identical schema bytes when their +nullability matches: use pointer fields for Java boxed types (`Integer`, `String` in lists) and +value fields for Java primitives. + +## Writing Rows by Hand + +`RowWriter`, `ArrayWriter`, and `MapWriter` write rows for a schema you construct yourself, without +a Go struct. They share one `fory.ByteBuffer`; a nested value is written at the buffer's current +position and then attached to its parent slot with `SetOffsetAndSize`. + +```go +schema := row.NewSchema([]row.Field{ + row.NewField("id", row.Int64Type{}, false), + row.NewField("tags", row.List(row.StringType{}), true), +}) +w := row.NewRowWriter(schema) +w.Reset() +w.WriteInt64(0, 7) + +tags := row.NewArrayWriter(row.List(row.StringType{}).Elem, w.Buffer()) +start := w.Buffer().WriterIndex() +if err := tags.Reset(2); err != nil { + panic(err) +} +if err := tags.WriteString(0, "go"); err != nil { + panic(err) +} +tags.SetNullAt(1) +if err := w.SetOffsetAndSize(1, start, w.Buffer().WriterIndex()-start); err != nil { + panic(err) +} +rowBytes := w.ToBytes() // valid until the buffer is written to again +``` + +Call `Reset` before each row; to reuse a writer for a new top-level row, set the buffer's writer +index back to zero first. + +## Thread Safety + +An `Encoder` owns a reusable write buffer and is not safe for concurrent use; create one encoder +per goroutine or guard it with a mutex. Writers share the same rule. `Row`, `ArrayData`, and +`MapData` views only read, so one view can be shared by concurrent readers as long as the +underlying bytes are not modified. + +## Related Topics + +- [Basic Serialization](../object-serialization/go/basic-serialization.md) - Object graph serialization +- [Standard Row Format](index.md#standard-row) - Shared layout for Java, Python, C++, Rust, and Go +- [Row Format Specification](../specification/row_format_spec.md) - Protocol details diff --git a/docs/row-format/index.md b/docs/row-format/index.md index 3375195166..ac41122a90 100644 --- a/docs/row-format/index.md +++ b/docs/row-format/index.md @@ -30,20 +30,20 @@ reconstruction as its primary access pattern. ## Choose a Layout -| Layout | Language support | Compatibility | -| ------------ | ----------------------- | -------------------------------- | -| Standard Row | Java, Python, C++, Rust | Shared Standard Row layout | -| Compact Row | Java | Java-only, space-oriented layout | +| Layout | Language support | Compatibility | +| ------------ | --------------------------- | -------------------------------- | +| Standard Row | Java, Python, C++, Rust, Go | Shared Standard Row layout | +| Compact Row | Java | Java-only, space-oriented layout | ## Standard Row -Standard Row is the interoperable layout for Java, Python, C++, and Rust. +Standard Row is the interoperable layout for Java, Python, C++, Rust, and Go. ### Features - **Zero-copy random access**: Read selected fields directly from encoded data. - **Partial deserialization**: Reconstruct only the values an application needs. -- **Cross-language compatibility**: Share Standard Row bytes between Java, Python, C++, and Rust. +- **Cross-language compatibility**: Share Standard Row bytes between Java, Python, C++, Rust, and Go. - **Apache Arrow integration**: Convert rows to Arrow data in Java and Python. ### Layout @@ -61,6 +61,7 @@ layout, alignment rules, type table, and endianness are defined by the | Python | Compatible | [Python](python.md) | PyArrow schema and table conversion | | C++ | Compatible | [C++](cpp.md) | Native row readers and writers | | Rust | Compatible | [Rust](rust.md) | Borrowed struct, array, and map views | +| Go | Compatible | [Go](go.md) | Reflection struct encoder; zero-copy row views | Use the language guides for installation, schema construction, encoding, random access, partial reads, and language-specific integrations. @@ -93,7 +94,7 @@ Reuse the encoder within one thread. Create separate encoders for concurrent thr - Fixed-size nested structs can be stored inline. Choose Compact Row only when every reader is Java and the space reduction justifies the -Java-specific layout. Use Standard Row for Java, Python, C++, and Rust interchange. +Java-specific layout. Use Standard Row for Java, Python, C++, Rust, and Go interchange. See the [Row Format specification](../specification/row_format_spec.md) for the exact Standard and Compact binary layouts. diff --git a/docs/row-format/troubleshooting.md b/docs/row-format/troubleshooting.md index c5fe6330d3..34f51d1c6d 100644 --- a/docs/row-format/troubleshooting.md +++ b/docs/row-format/troubleshooting.md @@ -21,7 +21,7 @@ license: | ## A Standard Row peer cannot read Compact Row bytes -Compact Row is a Java-only row family. Use Standard Row on Java, Python, C++, and Rust for shared +Compact Row is a Java-only row family. Use Standard Row on Java, Python, C++, Rust, and Go for shared bytes. ## A field lookup fails diff --git a/docs/specification/row_format_spec.md b/docs/specification/row_format_spec.md index e005634601..a192e35f1e 100644 --- a/docs/specification/row_format_spec.md +++ b/docs/specification/row_format_spec.md @@ -26,14 +26,14 @@ Apache Fory Row Format is a cache-friendly, random-access binary format designed - **Random Field Access**: Read individual fields without deserializing the entire row - **Zero-Copy Operations**: Direct memory access without data transformation - **Cache-Friendly Layout**: Optimized memory layout for CPU cache efficiency -- **Cross-Language Support**: Consistent binary format across Java, C++, Python, and Rust +- **Cross-Language Support**: Consistent binary format across Java, C++, Python, Rust, and Go Fory provides two row format variants: -| Format | Languages | Use Case | -| --------------- | ----------------------- | ------------------------------ | -| Standard Format | Java, C++, Python, Rust | Cross-language compatibility | -| Compact Format | Java only | Space efficiency, smaller rows | +| Format | Languages | Use Case | +| --------------- | --------------------------- | ------------------------------ | +| Standard Format | Java, C++, Python, Rust, Go | Cross-language compatibility | +| Compact Format | Java only | Space efficiency, smaller rows | ## Format Comparison diff --git a/go/fory/row/array_test.go b/go/fory/row/array_test.go new file mode 100644 index 0000000000..7faef1c518 --- /dev/null +++ b/go/fory/row/array_test.go @@ -0,0 +1,128 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package row + +import ( + "encoding/binary" + "math" + "testing" + + fory "github.com/apache/fory/go/fory" + "github.com/stretchr/testify/require" +) + +// Arrays store an 8-byte count, a null bitmap rounded to 8-byte words, +// then elements at their natural width, padded to an 8-byte boundary. +func TestArrayExactLayout(t *testing.T) { + buf := fory.NewByteBuffer(nil) + w := NewArrayWriter(List(Int32Type{}).Elem, buf) + require.NoError(t, w.Reset(3)) + w.WriteInt32(0, 1) + w.WriteInt32(1, 2) + w.WriteInt32(2, 3) + require.Equal(t, []byte{ + 3, 0, 0, 0, 0, 0, 0, 0, // element count + 0, 0, 0, 0, 0, 0, 0, 0, // null bitmap + 1, 0, 0, 0, 2, 0, 0, 0, // int32 elements at natural width + 3, 0, 0, 0, 0, 0, 0, 0, // last element + alignment padding + }, w.ToBytes()) + + a := NewArrayData(List(Int32Type{}).Elem, w.ToBytes()) + require.Equal(t, 3, a.NumElements()) + require.Equal(t, int32(2), a.Int32(1)) +} + +func TestStringArrayWithNullElement(t *testing.T) { + buf := fory.NewByteBuffer(nil) + elem := List(StringType{}).Elem + w := NewArrayWriter(elem, buf) + require.NoError(t, w.Reset(3)) + w.WriteString(0, "str1") + w.SetNullAt(1) + w.WriteString(2, "str2") + + a := NewArrayData(elem, w.ToBytes()) + require.Equal(t, 3, a.NumElements()) + require.Equal(t, "str1", a.String(0)) + require.True(t, a.IsNullAt(1)) + require.Equal(t, "", a.String(1)) + require.Equal(t, "str2", a.String(2)) +} + +func TestEmptyArray(t *testing.T) { + buf := fory.NewByteBuffer(nil) + elem := List(Int64Type{}).Elem + w := NewArrayWriter(elem, buf) + require.NoError(t, w.Reset(0)) + require.Equal(t, []byte{0, 0, 0, 0, 0, 0, 0, 0}, w.ToBytes()) + + a := NewArrayData(elem, w.ToBytes()) + require.Equal(t, 0, a.NumElements()) + require.Panics(t, func() { a.Int64(0) }) +} + +func TestNestedArray(t *testing.T) { + buf := fory.NewByteBuffer(nil) + outerElem := List(List(Int32Type{})).Elem // item: list + innerElem := List(Int32Type{}).Elem // item: int32 + + outer := NewArrayWriter(outerElem, buf) + require.NoError(t, outer.Reset(2)) + inner := NewArrayWriter(innerElem, buf) + for i, values := range [][]int32{{1, 2}, {3, 4, 5}} { + start := buf.WriterIndex() + require.NoError(t, inner.Reset(len(values))) + for j, v := range values { + inner.WriteInt32(j, v) + } + outer.SetOffsetAndSize(i, start, buf.WriterIndex()-start) + } + + a := NewArrayData(outerElem, outer.ToBytes()) + require.Equal(t, 2, a.NumElements()) + first := a.Array(0) + require.Equal(t, 2, first.NumElements()) + require.Equal(t, int32(2), first.Int32(1)) + second := a.Array(1) + require.Equal(t, 3, second.NumElements()) + require.Equal(t, int32(5), second.Int32(2)) +} + +func TestArrayResetRejectsInvalidLength(t *testing.T) { + w := NewArrayWriter(List(Int64Type{}).Elem, fory.NewByteBuffer(nil)) + require.Error(t, w.Reset(-1)) + require.Error(t, w.Reset(maxArrayDataBytes/8+1)) + // Extreme counts must fail the limit check, not overflow past it. + require.Error(t, w.Reset(math.MaxInt)) +} + +// Forged element counts must fail validation even when the arithmetic +// would overflow int64 and wrap the required size negative. +func TestArrayValidateBoundsRejectsForgedCounts(t *testing.T) { + for _, count := range []uint64{ + math.MaxUint64, // negative after int conversion + math.MaxInt64, // bitmap width and product overflow + uint64(math.MaxInt64/8) + 1, // product overflow with 8-byte elements + 1 << 32, // plausible-looking but far beyond the data + } { + data := make([]byte, 16) + binary.LittleEndian.PutUint64(data, count) + a := NewArrayData(List(Int64Type{}).Elem, data) + require.Error(t, a.validateBounds(), "count %d", count) + } +} diff --git a/go/fory/row/bitmap.go b/go/fory/row/bitmap.go new file mode 100644 index 0000000000..4e77b0cc9b --- /dev/null +++ b/go/fory/row/bitmap.go @@ -0,0 +1,49 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Package row implements the Fory standard row format. +package row + +// The null bitmap tracks one bit per field or array element. + +// bitmapWidthInBytes returns the null bitmap size for n fields or +// elements, rounded up to a whole 8-byte word. +func bitmapWidthInBytes(n int) int { + return ((n + 63) / 64) * 8 +} + +// setBit marks index i as null. +func setBit(bitmap []byte, i int) { + bitmap[i>>3] |= 1 << (uint(i) & 7) +} + +// clearBit marks index i as not null. +func clearBit(bitmap []byte, i int) { + bitmap[i>>3] &^= 1 << (uint(i) & 7) +} + +// getBit reports whether index i is null. +func getBit(bitmap []byte, i int) bool { + return bitmap[i>>3]&(1<<(uint(i)&7)) != 0 +} + +// roundToWord rounds n up to the nearest multiple of 8. Row Format requires +// every variable-length value and data region to be zero-padded to an +// 8-byte boundary. +func roundToWord(n int) int { + return (n + 7) &^ 7 +} diff --git a/go/fory/row/bitmap_test.go b/go/fory/row/bitmap_test.go new file mode 100644 index 0000000000..37130b6870 --- /dev/null +++ b/go/fory/row/bitmap_test.go @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package row + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestBitmapWidthInBytes(t *testing.T) { + cases := []struct{ n, want int }{ + {0, 0}, + {1, 8}, + {10, 8}, + {63, 8}, + {64, 8}, + {65, 16}, + {128, 16}, + {129, 24}, + } + for _, c := range cases { + require.Equal(t, c.want, bitmapWidthInBytes(c.n), "n=%d", c.n) + } +} + +func TestBitOperations(t *testing.T) { + bitmap := make([]byte, 16) + + // Bit 0 of byte 0 is index 0 (LSB-first). + setBit(bitmap, 0) + require.Equal(t, byte(0b1), bitmap[0]) + require.True(t, getBit(bitmap, 0)) + + setBit(bitmap, 2) + require.Equal(t, byte(0b101), bitmap[0]) + + // Index 7 stays in byte 0; index 8 moves to byte 1. + setBit(bitmap, 7) + require.Equal(t, byte(0b10000101), bitmap[0]) + setBit(bitmap, 8) + require.Equal(t, byte(0b1), bitmap[1]) + + // Word boundary: index 63 is the last bit of the first word, + // index 64 the first bit of the second. + setBit(bitmap, 63) + require.Equal(t, byte(0b10000000), bitmap[7]) + setBit(bitmap, 64) + require.Equal(t, byte(0b1), bitmap[8]) + + // Clearing affects only the targeted bit. + clearBit(bitmap, 2) + require.False(t, getBit(bitmap, 2)) + require.True(t, getBit(bitmap, 0)) + require.True(t, getBit(bitmap, 7)) + + require.False(t, getBit(bitmap, 1)) +} + +func TestRoundToWord(t *testing.T) { + cases := []struct{ n, want int }{ + {0, 0}, + {1, 8}, + {7, 8}, + {8, 8}, + {9, 16}, + {16, 16}, + } + for _, c := range cases { + require.Equal(t, c.want, roundToWord(c.n), "n=%d", c.n) + } +} diff --git a/go/fory/row/datatype.go b/go/fory/row/datatype.go new file mode 100644 index 0000000000..d53ef34fb2 --- /dev/null +++ b/go/fory/row/datatype.go @@ -0,0 +1,314 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package row + +import ( + "fmt" + "strings" + + fory "github.com/apache/fory/go/fory" +) + +// DataType describes the row-format type of a field. +// +// Type ids are the cross-language Fory type ids shared with the Java, +// C++, and Python row-format implementations; they appear verbatim in +// the serialized schema wire format. +type DataType interface { + TypeID() fory.TypeId + // ByteWidth returns the fixed storage width in bytes, or -1 for + // variable-width types stored through an offset+size slot. + ByteWidth() int + String() string +} + +// Cross-language child field names for matching Java DataTypes. +const ( + listItemName = "item" + mapKeyName = "key" + mapValueName = "value" +) + +type BoolType struct{} + +func (BoolType) TypeID() fory.TypeId { return fory.BOOL } +func (BoolType) ByteWidth() int { return 1 } +func (BoolType) String() string { return "bool" } + +type Int8Type struct{} + +func (Int8Type) TypeID() fory.TypeId { return fory.INT8 } +func (Int8Type) ByteWidth() int { return 1 } +func (Int8Type) String() string { return "int8" } + +type Int16Type struct{} + +func (Int16Type) TypeID() fory.TypeId { return fory.INT16 } +func (Int16Type) ByteWidth() int { return 2 } +func (Int16Type) String() string { return "int16" } + +type Int32Type struct{} + +func (Int32Type) TypeID() fory.TypeId { return fory.INT32 } +func (Int32Type) ByteWidth() int { return 4 } +func (Int32Type) String() string { return "int32" } + +type Int64Type struct{} + +func (Int64Type) TypeID() fory.TypeId { return fory.INT64 } +func (Int64Type) ByteWidth() int { return 8 } +func (Int64Type) String() string { return "int64" } + +// Float16Type exists so schemas received from other languages parse, +// the Go writer, reader, and encoder do not support it yet. +type Float16Type struct{} + +func (Float16Type) TypeID() fory.TypeId { return fory.FLOAT16 } +func (Float16Type) ByteWidth() int { return 2 } +func (Float16Type) String() string { return "float16" } + +type Float32Type struct{} + +func (Float32Type) TypeID() fory.TypeId { return fory.FLOAT32 } +func (Float32Type) ByteWidth() int { return 4 } +func (Float32Type) String() string { return "float32" } + +type Float64Type struct{} + +func (Float64Type) TypeID() fory.TypeId { return fory.FLOAT64 } +func (Float64Type) ByteWidth() int { return 8 } +func (Float64Type) String() string { return "float64" } + +// StringType values are UTF-8 bytes in the variable data region, +// stored as (offset + size) in the fixed slot region. +type StringType struct{} + +func (StringType) TypeID() fory.TypeId { return fory.STRING } +func (StringType) ByteWidth() int { return -1 } +func (StringType) String() string { return "string" } + +type BinaryType struct{} + +func (BinaryType) TypeID() fory.TypeId { return fory.BINARY } +func (BinaryType) ByteWidth() int { return -1 } +func (BinaryType) String() string { return "binary" } + +// Date32Type values are days since the Unix epoch (int32). +type Date32Type struct{} + +func (Date32Type) TypeID() fory.TypeId { return fory.DATE } +func (Date32Type) ByteWidth() int { return 4 } +func (Date32Type) String() string { return "date32" } + +// TimestampType values are microseconds since the Unix epoch (int64). +type TimestampType struct{} + +func (TimestampType) TypeID() fory.TypeId { return fory.TIMESTAMP } +func (TimestampType) ByteWidth() int { return 8 } +func (TimestampType) String() string { return "timestamp" } + +// DurationType values are microseconds (int64). +type DurationType struct{} + +func (DurationType) TypeID() fory.TypeId { return fory.DURATION } +func (DurationType) ByteWidth() int { return 8 } +func (DurationType) String() string { return "duration" } + +// DecimalType exists so schemas received from other languages parse; the +// Go writer, reader, and encoder do not support it yet. Use it as a +// value, not a pointer, so schema equality compares precision and scale. +type DecimalType struct { + Precision int + Scale int +} + +func (DecimalType) TypeID() fory.TypeId { return fory.DECIMAL } +func (DecimalType) ByteWidth() int { return -1 } +func (t DecimalType) String() string { + return fmt.Sprintf("decimal(%d, %d)", t.Precision, t.Scale) +} + +// ListType is a variable-length sequence of Elem values. +type ListType struct { + Elem Field +} + +func (*ListType) TypeID() fory.TypeId { return fory.LIST } +func (*ListType) ByteWidth() int { return -1 } +func (t *ListType) String() string { return "list<" + t.Elem.Type.String() + ">" } + +// MapType stores keys and values as two adjacent arrays. +type MapType struct { + Key Field + Value Field +} + +func (*MapType) TypeID() fory.TypeId { return fory.MAP } +func (*MapType) ByteWidth() int { return -1 } +func (t *MapType) String() string { + return "map<" + t.Key.Type.String() + ", " + t.Value.Type.String() + ">" +} + +// StructType is a nested row with its own bitmap, slots, and variable +// data region. +type StructType struct { + Fields []Field +} + +func (*StructType) TypeID() fory.TypeId { return fory.STRUCT } +func (*StructType) ByteWidth() int { return -1 } +func (t *StructType) String() string { + var sb strings.Builder + sb.WriteString("struct<") + for i, f := range t.Fields { + if i > 0 { + sb.WriteString(", ") + } + sb.WriteString(f.Name) + sb.WriteString(": ") + sb.WriteString(f.Type.String()) + } + sb.WriteString(">") + return sb.String() +} + +// Field is a named, optionally nullable slot in a schema or composite type. +type Field struct { + Name string + Type DataType + Nullable bool +} + +func NewField(name string, dataType DataType, nullable bool) Field { + return Field{Name: name, Type: dataType, Nullable: nullable} +} + +func (f Field) Equal(o Field) bool { + return f.Name == o.Name && f.Nullable == o.Nullable && dataTypeEqual(f.Type, o.Type) +} + +// List returns a list type with the cross-language element field name +// "item" and nullable elements. Build ListType directly for +// non-nullable elements. +func List(elem DataType) *ListType { + return &ListType{Elem: Field{Name: listItemName, Type: elem, Nullable: true}} +} + +// Map returns a map type with the cross-language field names +// "key"/"value". Keys are always non-nullable. +func Map(key, value DataType) *MapType { + return &MapType{ + Key: Field{Name: mapKeyName, Type: key, Nullable: false}, + Value: Field{Name: mapValueName, Type: value, Nullable: true}, + } +} + +// Struct returns a struct type over a copy of fields. +func Struct(fields []Field) *StructType { + return &StructType{Fields: append([]Field(nil), fields...)} +} + +func dataTypeEqual(a, b DataType) bool { + switch at := a.(type) { + case *ListType: + bt, ok := b.(*ListType) + return ok && at.Elem.Equal(bt.Elem) + case *MapType: + bt, ok := b.(*MapType) + return ok && at.Key.Equal(bt.Key) && at.Value.Equal(bt.Value) + case *StructType: + bt, ok := b.(*StructType) + if !ok || len(at.Fields) != len(bt.Fields) { + return false + } + for i := range at.Fields { + if !at.Fields[i].Equal(bt.Fields[i]) { + return false + } + } + return true + default: + // Primitive types are empty structs and DecimalType is a + // comparable value, so interface equality is exact. + return a == b + } +} + +// Schema describes the fields of a top-level row. +type Schema struct { + fields []Field + byName map[string]int +} + +// NewSchema builds a schema over a copy of fields in their declared +// order, which fixes the field slot layout. For duplicate names, +// FieldIndex resolves to the last occurrence, matching Java's Schema so +// identical schema bytes resolve names identically in both runtimes. +func NewSchema(fields []Field) *Schema { + owned := append([]Field(nil), fields...) + byName := make(map[string]int, len(owned)) + for i, f := range owned { + byName[f.Name] = i + } + return &Schema{fields: owned, byName: byName} +} + +func (s *Schema) NumFields() int { return len(s.fields) } + +func (s *Schema) Field(i int) Field { return s.fields[i] } + +// Fields returns the backing field slice; callers must not modify it. +func (s *Schema) Fields() []Field { return s.fields } + +// FieldIndex returns the ordinal of the named field, or -1 if absent. +func (s *Schema) FieldIndex(name string) int { + if i, ok := s.byName[name]; ok { + return i + } + return -1 +} + +func (s *Schema) Equal(o *Schema) bool { + if s == o { + return true + } + if o == nil || len(s.fields) != len(o.fields) { + return false + } + for i := range s.fields { + if !s.fields[i].Equal(o.fields[i]) { + return false + } + } + return true +} + +func (s *Schema) String() string { + var sb strings.Builder + sb.WriteString("schema<") + for i, f := range s.fields { + if i > 0 { + sb.WriteString(", ") + } + sb.WriteString(f.Name) + sb.WriteString(": ") + sb.WriteString(f.Type.String()) + } + sb.WriteString(">") + return sb.String() +} diff --git a/go/fory/row/encoder.go b/go/fory/row/encoder.go new file mode 100644 index 0000000000..aa459d9ce9 --- /dev/null +++ b/go/fory/row/encoder.go @@ -0,0 +1,567 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package row + +import ( + "encoding/binary" + "fmt" + "reflect" + "time" + + fory "github.com/apache/fory/go/fory" +) + +// Encoder converts values of struct type T to and from row bytes using +// a codec tree compiled once with reflection at construction time. The +// schema is inferred per InferSchema. +// +// The row format is a trusted in-memory format: FromRow and Decode +// expect trusted, schema-matched bytes produced by a row writer for the +// same schema. Malformed input surfaces as an error rather than a +// panic, but no allocation or work limits are enforced against hostile +// bytes. An Encoder owns a reusable write buffer and is NOT +// goroutine-safe; create one Encoder per goroutine. +type Encoder[T any] struct { + codec *structCodec + hash int64 + buf *fory.ByteBuffer +} + +func NewEncoder[T any]() (*Encoder[T], error) { + t := reflect.TypeOf((*T)(nil)).Elem() + if t.Kind() != reflect.Struct { + return nil, fmt.Errorf("row: Encoder requires a struct type, got %v", t) + } + buf := fory.NewByteBuffer(nil) + codec, err := newStructCodec(t, buf) + if err != nil { + return nil, err + } + return &Encoder[T]{codec: codec, hash: ComputeSchemaHash(codec.schema), buf: buf}, nil +} + +func (e *Encoder[T]) Schema() *Schema { return e.codec.schema } + +// SchemaHash returns the cross-language type-shape fingerprint of the +// schema: a fold over the recursive type ids only. It does not cover +// field names, nullability, decimal parameters, or the order of +// same-typed fields, so Encode/Decode can detect only type-shape +// mismatches between writer and reader. +func (e *Encoder[T]) SchemaHash() int64 { return e.hash } + +// ToRow serializes v to row bytes (no framing, just the row). +func (e *Encoder[T]) ToRow(v *T) ([]byte, error) { + if v == nil { + return nil, errNilValue + } + e.buf.SetWriterIndex(0) + return e.finishRow(v) +} + +// Encode frames the row for cross-language exchange: an int64 +// little-endian schema hash followed by the row bytes, matching the +// Java and Python row encoders. +func (e *Encoder[T]) Encode(v *T) ([]byte, error) { + if v == nil { + return nil, errNilValue + } + // The hash is written into the reusable buffer ahead of the row so + // the framed output needs a single copy. + e.buf.SetWriterIndex(0) + e.buf.Reserve(8) + binary.LittleEndian.PutUint64(e.buf.GetData()[:8], uint64(e.hash)) + e.buf.SetWriterIndex(8) + return e.finishRow(v) +} + +// finishRow writes v at the buffer's current writer index and returns +// an owned copy of everything written so far. +func (e *Encoder[T]) finishRow(v *T) ([]byte, error) { + if err := e.codec.writeStruct(reflect.ValueOf(v).Elem()); err != nil { + return nil, err + } + out := make([]byte, e.buf.WriterIndex()) + copy(out, e.buf.GetData()[:e.buf.WriterIndex()]) + return out, nil +} + +// FromRow deserializes row bytes produced with this schema. +func (e *Encoder[T]) FromRow(data []byte) (T, error) { + var out T + err := e.FromRowInto(data, &out) + return out, err +} + +func (e *Encoder[T]) FromRowInto(data []byte, out *T) (err error) { + if out == nil { + return fmt.Errorf("row: cannot decode into a nil target") + } + defer func() { + if p := recover(); p != nil { + err = fmt.Errorf("row: malformed row data: %v", p) + } + }() + return e.codec.readStruct(NewRow(e.codec.schema, data), reflect.ValueOf(out).Elem()) +} + +// Decode verifies the schema type-shape hash and deserializes the row. +func (e *Encoder[T]) Decode(data []byte) (T, error) { + var out T + if len(data) < 8 { + return out, fmt.Errorf("row: encoded data of %d bytes is too short for the schema hash", len(data)) + } + hash := int64(binary.LittleEndian.Uint64(data)) + if hash != e.hash { + return out, fmt.Errorf("row: schema type-shape hash mismatch: data has %d, encoder expects %d", hash, e.hash) + } + err := e.FromRowInto(data[8:], &out) + return out, err +} + +var errNilValue = fmt.Errorf("row: cannot encode a nil value") + +func nullIntoValueError(goType reflect.Type) error { + return fmt.Errorf("row: null value for non-nullable Go type %v; use a pointer carrier to receive nulls", goType) +} + +// slotWriter is the shared write surface of RowWriter and ArrayWriter, +// letting one value codec serve struct fields and array elements. +type slotWriter interface { + SetNullAt(i int) + WriteBool(i int, v bool) + WriteInt8(i int, v int8) + WriteInt16(i int, v int16) + WriteInt32(i int, v int32) + WriteInt64(i int, v int64) + WriteFloat32(i int, v float32) + WriteFloat64(i int, v float64) + WriteDate(i int, d fory.Date) error + WriteTimestamp(i int, t time.Time) error + WriteDuration(i int, d time.Duration) + WriteString(i int, s string) error + WriteBytes(i int, b []byte) error + SetOffsetAndSize(i, absStart, size int) error +} + +// valueReader is the shared read surface of Row and ArrayData. +type valueReader interface { + IsNullAt(i int) bool + Bool(i int) bool + Int8(i int) int8 + Int16(i int) int16 + Int32(i int) int32 + Int64(i int) int64 + Float32(i int) float32 + Float64(i int) float64 + Date(i int) fory.Date + Timestamp(i int) time.Time + Duration(i int) time.Duration + String(i int) string + Binary(i int) []byte + Struct(i int) *Row + Array(i int) *ArrayData + Map(i int) *MapData +} + +// valueCodec reads or writes one value at slot/element i. Write +// functions handle nil for nilable Go types. Read functions overwrite +// the target even when the value is null so reused targets stay clean, +// and reject null for Go carriers that cannot represent it. +type valueCodec struct { + write func(w slotWriter, i int, v reflect.Value) error + read func(g valueReader, i int, v reflect.Value) error +} + +// structCodec writes and reads one struct level; nested structs hold +// their own child codec with a child RowWriter sharing the buffer. +type structCodec struct { + schema *Schema + writer *RowWriter + fields []fieldCodec +} + +type fieldCodec struct { + goIndex int + codec valueCodec +} + +func newStructCodec(t reflect.Type, buf *fory.ByteBuffer) (*structCodec, error) { + layout, err := inferStructLayout(t, nil) + if err != nil { + return nil, err + } + sc := &structCodec{ + schema: layout.schema, + writer: NewRowWriterWithBuffer(layout.schema, buf), + fields: make([]fieldCodec, 0, layout.schema.NumFields()), + } + for ordinal, goIndex := range layout.indexes { + codec, err := newValueCodec(t.Field(goIndex).Type, layout.schema.Field(ordinal).Type, buf) + if err != nil { + return nil, err + } + sc.fields = append(sc.fields, fieldCodec{goIndex: goIndex, codec: codec}) + } + return sc, nil +} + +func (sc *structCodec) writeStruct(v reflect.Value) error { + sc.writer.Reset() + for ordinal := range sc.fields { + fc := &sc.fields[ordinal] + if err := fc.codec.write(sc.writer, ordinal, v.Field(fc.goIndex)); err != nil { + return err + } + } + return nil +} + +func (sc *structCodec) readStruct(r *Row, v reflect.Value) error { + for ordinal := range sc.fields { + fc := &sc.fields[ordinal] + if err := fc.codec.read(r, ordinal, v.Field(fc.goIndex)); err != nil { + return err + } + } + return nil +} + +func newValueCodec(goType reflect.Type, dataType DataType, buf *fory.ByteBuffer) (valueCodec, error) { + if goType.Kind() == reflect.Ptr { + elemType := goType.Elem() + inner, err := newValueCodec(elemType, dataType, buf) + if err != nil { + return valueCodec{}, err + } + return valueCodec{ + write: func(w slotWriter, i int, v reflect.Value) error { + if v.IsNil() { + w.SetNullAt(i) + return nil + } + return inner.write(w, i, v.Elem()) + }, + read: func(g valueReader, i int, v reflect.Value) error { + if g.IsNullAt(i) { + v.Set(reflect.Zero(goType)) + return nil + } + p := reflect.New(elemType) + if err := inner.read(g, i, p.Elem()); err != nil { + return err + } + v.Set(p) + return nil + }, + }, nil + } + + switch dt := dataType.(type) { + case BoolType: + return scalarCodec(goType, + func(w slotWriter, i int, v reflect.Value) error { w.WriteBool(i, v.Bool()); return nil }, + func(g valueReader, i int, v reflect.Value) error { v.SetBool(g.Bool(i)); return nil }), nil + case Int8Type: + return scalarCodec(goType, + func(w slotWriter, i int, v reflect.Value) error { w.WriteInt8(i, int8(v.Int())); return nil }, + func(g valueReader, i int, v reflect.Value) error { v.SetInt(int64(g.Int8(i))); return nil }), nil + case Int16Type: + return scalarCodec(goType, + func(w slotWriter, i int, v reflect.Value) error { w.WriteInt16(i, int16(v.Int())); return nil }, + func(g valueReader, i int, v reflect.Value) error { v.SetInt(int64(g.Int16(i))); return nil }), nil + case Int32Type: + return scalarCodec(goType, + func(w slotWriter, i int, v reflect.Value) error { w.WriteInt32(i, int32(v.Int())); return nil }, + func(g valueReader, i int, v reflect.Value) error { v.SetInt(int64(g.Int32(i))); return nil }), nil + case Int64Type: + return scalarCodec(goType, + func(w slotWriter, i int, v reflect.Value) error { w.WriteInt64(i, v.Int()); return nil }, + func(g valueReader, i int, v reflect.Value) error { + x := g.Int64(i) + // `int` is 32 bits wide on some platforms. + if v.OverflowInt(x) { + return fmt.Errorf("row: value %d overflows %v", x, goType) + } + v.SetInt(x) + return nil + }), nil + case Float32Type: + return scalarCodec(goType, + func(w slotWriter, i int, v reflect.Value) error { w.WriteFloat32(i, float32(v.Float())); return nil }, + func(g valueReader, i int, v reflect.Value) error { v.SetFloat(float64(g.Float32(i))); return nil }), nil + case Float64Type: + return scalarCodec(goType, + func(w slotWriter, i int, v reflect.Value) error { w.WriteFloat64(i, v.Float()); return nil }, + func(g valueReader, i int, v reflect.Value) error { v.SetFloat(g.Float64(i)); return nil }), nil + case Date32Type: + return scalarCodec(goType, + func(w slotWriter, i int, v reflect.Value) error { return w.WriteDate(i, v.Interface().(fory.Date)) }, + func(g valueReader, i int, v reflect.Value) error { v.Set(reflect.ValueOf(g.Date(i))); return nil }), nil + case TimestampType: + return scalarCodec(goType, + func(w slotWriter, i int, v reflect.Value) error { + return w.WriteTimestamp(i, v.Interface().(time.Time)) + }, + func(g valueReader, i int, v reflect.Value) error { v.Set(reflect.ValueOf(g.Timestamp(i))); return nil }), nil + case DurationType: + return scalarCodec(goType, + func(w slotWriter, i int, v reflect.Value) error { + w.WriteDuration(i, time.Duration(v.Int())) + return nil + }, + func(g valueReader, i int, v reflect.Value) error { v.SetInt(int64(g.Duration(i))); return nil }), nil + case StringType: + return scalarCodec(goType, + func(w slotWriter, i int, v reflect.Value) error { return w.WriteString(i, v.String()) }, + func(g valueReader, i int, v reflect.Value) error { v.SetString(g.String(i)); return nil }), nil + case *ListType: + if goType.Elem().Kind() == reflect.Uint8 { + return newByteListCodec(goType, dt, buf), nil + } + return newListCodec(goType, dt, buf) + case *MapType: + return newMapCodec(goType, dt, buf) + case *StructType: + child, err := newStructCodec(goType, buf) + if err != nil { + return valueCodec{}, err + } + return valueCodec{ + write: func(w slotWriter, i int, v reflect.Value) error { + start := child.writer.buf.WriterIndex() + if err := child.writeStruct(v); err != nil { + return err + } + return w.SetOffsetAndSize(i, start, child.writer.buf.WriterIndex()-start) + }, + read: func(g valueReader, i int, v reflect.Value) error { + nested := g.Struct(i) + if nested == nil { + return nullIntoValueError(goType) + } + return child.readStruct(nested, v) + }, + }, nil + default: + return valueCodec{}, fmt.Errorf("row: data type %s is not supported by the encoder", dataType) + } +} + +// scalarCodec wraps a codec for a Go carrier that cannot hold nil, so a +// null value on read is an error instead of a silent zero value. +func scalarCodec(goType reflect.Type, + write func(w slotWriter, i int, v reflect.Value) error, + read func(g valueReader, i int, v reflect.Value) error) valueCodec { + return valueCodec{ + write: write, + read: func(g valueReader, i int, v reflect.Value) error { + if g.IsNullAt(i) { + return nullIntoValueError(goType) + } + return read(g, i, v) + }, + } +} + +// newByteListCodec handles []byte as list, the Java byte[] model: +// the bytes are copied straight into and out of the int8 element region +// with no per-element reflection. +func newByteListCodec(goType reflect.Type, listType *ListType, buf *fory.ByteBuffer) valueCodec { + writer := NewArrayWriter(listType.Elem, buf) + return valueCodec{ + write: func(w slotWriter, i int, v reflect.Value) error { + if v.IsNil() { + w.SetNullAt(i) + return nil + } + b := v.Bytes() + start := buf.WriterIndex() + if err := writer.Reset(len(b)); err != nil { + return err + } + copy(buf.GetData()[writer.base+writer.headerBytes:], b) + return w.SetOffsetAndSize(i, start, buf.WriterIndex()-start) + }, + read: func(g valueReader, i int, v reflect.Value) error { + arr := g.Array(i) + if arr == nil { + v.Set(reflect.Zero(goType)) + return nil + } + if err := arr.validateBounds(); err != nil { + return err + } + n := arr.NumElements() + for j := 0; j < n; j++ { + if arr.IsNullAt(j) { + return nullIntoValueError(goType.Elem()) + } + } + // Copy so the decoded struct never aliases the input. + owned := make([]byte, n) + copy(owned, arr.data[arr.headerBytes:arr.headerBytes+n]) + v.SetBytes(owned) + return nil + }, + } +} + +func newListCodec(goType reflect.Type, listType *ListType, buf *fory.ByteBuffer) (valueCodec, error) { + elemCodec, err := newValueCodec(goType.Elem(), listType.Elem.Type, buf) + if err != nil { + return valueCodec{}, err + } + writer := NewArrayWriter(listType.Elem, buf) + return valueCodec{ + write: func(w slotWriter, i int, v reflect.Value) error { + if v.IsNil() { + w.SetNullAt(i) + return nil + } + n := v.Len() + start := buf.WriterIndex() + if err := writer.Reset(n); err != nil { + return err + } + for j := 0; j < n; j++ { + if err := elemCodec.write(writer, j, v.Index(j)); err != nil { + return err + } + } + return w.SetOffsetAndSize(i, start, buf.WriterIndex()-start) + }, + read: func(g valueReader, i int, v reflect.Value) error { + arr := g.Array(i) + if arr == nil { + v.Set(reflect.Zero(goType)) + return nil + } + if err := arr.validateBounds(); err != nil { + return err + } + n := arr.NumElements() + s := reflect.MakeSlice(goType, n, n) + for j := 0; j < n; j++ { + if err := elemCodec.read(arr, j, s.Index(j)); err != nil { + return err + } + } + v.Set(s) + return nil + }, + }, nil +} + +func newMapCodec(goType reflect.Type, mapType *MapType, buf *fory.ByteBuffer) (valueCodec, error) { + keyType, valueType := goType.Key(), goType.Elem() + keyCodec, err := newValueCodec(keyType, mapType.Key.Type, buf) + if err != nil { + return valueCodec{}, err + } + valueCodecImpl, err := newValueCodec(valueType, mapType.Value.Type, buf) + if err != nil { + return valueCodec{}, err + } + mapWriter := NewMapWriter(buf) + keysWriter := NewArrayWriter(mapType.Key, buf) + valuesWriter := NewArrayWriter(mapType.Value, buf) + // Scratch values reused across entries: the encoder is single-use + // per goroutine and a map site is never re-entered recursively. + keyScratch := reflect.New(keyType).Elem() + readKey := reflect.New(keyType).Elem() + readValue := reflect.New(valueType).Elem() + values := reflect.MakeSlice(reflect.SliceOf(valueType), 0, 0) + return valueCodec{ + write: func(w slotWriter, i int, v reflect.Value) error { + if v.IsNil() { + w.SetNullAt(i) + return nil + } + n := v.Len() + start := buf.WriterIndex() + mapWriter.Reset() + if err := keysWriter.Reset(n); err != nil { + return err + } + // Keys are written during the single map iteration while + // values are captured into one typed slice, so each entry + // costs no reflect.Value allocation. Go map iteration order + // changes between iterations, which is why both halves + // cannot simply be iterated twice. + if values.Cap() < n { + values = reflect.MakeSlice(reflect.SliceOf(valueType), n, n) + } + values = values.Slice(0, n) + j := 0 + for iter := v.MapRange(); iter.Next(); j++ { + if j >= n { + return fmt.Errorf("row: map changed size during encoding") + } + keyScratch.SetIterKey(iter) + if err := keyCodec.write(keysWriter, j, keyScratch); err != nil { + return err + } + values.Index(j).SetIterValue(iter) + } + if j != n { + return fmt.Errorf("row: map changed size during encoding") + } + mapWriter.FinishKeys() + if err := valuesWriter.Reset(n); err != nil { + return err + } + for j := 0; j < n; j++ { + if err := valueCodecImpl.write(valuesWriter, j, values.Index(j)); err != nil { + return err + } + } + values.Clear() // drop references to the caller's values + return w.SetOffsetAndSize(i, start, buf.WriterIndex()-start) + }, + read: func(g valueReader, i int, v reflect.Value) error { + md := g.Map(i) + if md == nil { + v.Set(reflect.Zero(goType)) + return nil + } + keys, vals := md.Keys(), md.Values() + if err := keys.validateBounds(); err != nil { + return err + } + if err := vals.validateBounds(); err != nil { + return err + } + n := keys.NumElements() + if vals.NumElements() != n { + return fmt.Errorf("row: map has %d keys but %d values", n, vals.NumElements()) + } + m := reflect.MakeMapWithSize(goType, n) + for j := 0; j < n; j++ { + if err := keyCodec.read(keys, j, readKey); err != nil { + return err + } + if err := valueCodecImpl.read(vals, j, readValue); err != nil { + return err + } + m.SetMapIndex(readKey, readValue) + } + v.Set(m) + return nil + }, + }, nil +} diff --git a/go/fory/row/encoder_bench_test.go b/go/fory/row/encoder_bench_test.go new file mode 100644 index 0000000000..e355b57169 --- /dev/null +++ b/go/fory/row/encoder_bench_test.go @@ -0,0 +1,76 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package row + +import ( + "fmt" + "testing" +) + +// Framed encoding must cost one output allocation beyond the row work. +func BenchmarkEncodeFramed(b *testing.B) { + enc, err := NewEncoder[person]() + if err != nil { + b.Fatal(err) + } + value := samplePerson() + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err := enc.Encode(&value); err != nil { + b.Fatal(err) + } + } +} + +type mapHolder struct { + Entries map[string]int64 +} + +// Map codecs must stay linear without per-entry reflection allocations. +func BenchmarkMapRoundTrip(b *testing.B) { + enc, err := NewEncoder[mapHolder]() + if err != nil { + b.Fatal(err) + } + value := mapHolder{Entries: make(map[string]int64, 256)} + for i := 0; i < 256; i++ { + value.Entries[fmt.Sprintf("key-%d", i)] = int64(i) + } + b.Run("encode", func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + if _, err := enc.ToRow(&value); err != nil { + b.Fatal(err) + } + } + }) + rowBytes, err := enc.ToRow(&value) + if err != nil { + b.Fatal(err) + } + b.Run("decode", func(b *testing.B) { + b.ReportAllocs() + var out mapHolder + for i := 0; i < b.N; i++ { + if err := enc.FromRowInto(rowBytes, &out); err != nil { + b.Fatal(err) + } + } + }) +} diff --git a/go/fory/row/encoder_test.go b/go/fory/row/encoder_test.go new file mode 100644 index 0000000000..15e165b08d --- /dev/null +++ b/go/fory/row/encoder_test.go @@ -0,0 +1,299 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package row + +import ( + "encoding/binary" + "testing" + "time" + + fory "github.com/apache/fory/go/fory" + "github.com/stretchr/testify/require" +) + +type address struct { + City string + Zip int32 +} + +type person struct { + Id int64 + Name *string + Score float64 + Tags []string + Attrs map[string]int32 + Addr *address + Data []byte + Born fory.Date + At time.Time + Dur time.Duration +} + +func samplePerson() person { + name := "ayush" + return person{ + Id: 42, + Name: &name, + Score: 99.5, + Tags: []string{"go", "fory"}, + Attrs: map[string]int32{"a": 1, "b": 2}, + Addr: &address{City: "Delhi", Zip: 110001}, + Data: []byte{1, 2, 3}, + Born: fory.Date{Year: 2007, Month: time.June, Day: 1}, + At: time.UnixMicro(1_234_567_890_123_456), + Dur: 90 * time.Second, + } +} + +func TestEncoderRoundTrip(t *testing.T) { + enc, err := NewEncoder[person]() + require.NoError(t, err) + + original := samplePerson() + rowBytes, err := enc.ToRow(&original) + require.NoError(t, err) + + decoded, err := enc.FromRow(rowBytes) + require.NoError(t, err) + require.Equal(t, original, decoded) +} + +func TestEncoderNullsAndEmptiness(t *testing.T) { + enc, err := NewEncoder[person]() + require.NoError(t, err) + + // All nilable fields nil: they round-trip as nil, not zero values. + // Temporal fields stay valid; a zero fory.Date is rejected by + // WriteDate just like in the object-graph serializer. + original := samplePerson() + original.Name, original.Tags, original.Attrs, original.Addr, original.Data = nil, nil, nil, nil, nil + rowBytes, err := enc.ToRow(&original) + require.NoError(t, err) + decoded, err := enc.FromRow(rowBytes) + require.NoError(t, err) + require.Equal(t, original, decoded) + require.Nil(t, decoded.Name) + require.Nil(t, decoded.Tags) + require.Nil(t, decoded.Attrs) + + // Empty is distinct from nil. + original.Tags, original.Attrs, original.Data = []string{}, map[string]int32{}, []byte{} + rowBytes, err = enc.ToRow(&original) + require.NoError(t, err) + decoded, err = enc.FromRow(rowBytes) + require.NoError(t, err) + require.NotNil(t, decoded.Tags) + require.Empty(t, decoded.Tags) + require.NotNil(t, decoded.Attrs) + require.NotNil(t, decoded.Data) +} + +func TestEncoderDeepNesting(t *testing.T) { + type inner struct { + Vals []int64 + KV map[string][]int32 + } + type outer struct { + Rows []inner + Ptrs []*int32 + } + enc, err := NewEncoder[outer]() + require.NoError(t, err) + + three := int32(3) + original := outer{ + Rows: []inner{ + {Vals: []int64{1, 2}, KV: map[string][]int32{"x": {7, 8}}}, + {Vals: nil, KV: nil}, + }, + Ptrs: []*int32{&three, nil}, + } + rowBytes, err := enc.ToRow(&original) + require.NoError(t, err) + decoded, err := enc.FromRow(rowBytes) + require.NoError(t, err) + require.Equal(t, original, decoded) + require.Nil(t, decoded.Ptrs[1]) +} + +func TestEncodeDecodeFraming(t *testing.T) { + enc, err := NewEncoder[person]() + require.NoError(t, err) + original := samplePerson() + + encoded, err := enc.Encode(&original) + require.NoError(t, err) + require.Equal(t, uint64(enc.SchemaHash()), binary.LittleEndian.Uint64(encoded)) + require.Equal(t, ComputeSchemaHash(enc.Schema()), enc.SchemaHash()) + + decoded, err := enc.Decode(encoded) + require.NoError(t, err) + require.Equal(t, original, decoded) + + // Tampered hash is rejected with a descriptive error. + tampered := append([]byte(nil), encoded...) + tampered[0] ^= 0xFF + _, err = enc.Decode(tampered) + requireErrorContains(t, err, "hash mismatch") + + _, err = enc.Decode(encoded[:4]) + requireErrorContains(t, err, "too short") +} + +// Corrupt row bytes must produce errors, never panics. +func TestEncoderCorruptRowData(t *testing.T) { + enc, err := NewEncoder[person]() + require.NoError(t, err) + original := samplePerson() + rowBytes, err := enc.ToRow(&original) + require.NoError(t, err) + + for _, cut := range []int{1, 8, len(rowBytes) / 2, len(rowBytes) - 1} { + _, err := enc.FromRow(rowBytes[:cut]) + require.Error(t, err, "truncated to %d bytes", cut) + } +} + +func TestNewEncoderRejectsNonStruct(t *testing.T) { + _, err := NewEncoder[int]() + require.Error(t, err) + _, err = NewEncoder[map[string]int]() + require.Error(t, err) +} + +// A null for a Go carrier that cannot hold nil is a decode error, not a +// silent zero value: the wire and type hash cannot distinguish the two. +func TestDecodeRejectsNullIntoValueCarrier(t *testing.T) { + type inner struct { + X int32 + } + type carriers struct { + S string + In inner + L []int32 + } + enc, err := NewEncoder[carriers]() + require.NoError(t, err) + schema := enc.Schema() + w := NewRowWriter(schema) + idx := schema.FieldIndex + + // buildRow writes a complete valid row, then nulls one target: + // the string field, the value-struct field, or one list element. + buildRow := func(nullTarget string) []byte { + buf := w.Buffer() + buf.SetWriterIndex(0) + w.Reset() + if nullTarget == "s" { + w.SetNullAt(idx("s")) + } else { + require.NoError(t, w.WriteString(idx("s"), "x")) + } + if nullTarget == "in" { + w.SetNullAt(idx("in")) + } else { + child := NewRowWriterWithBuffer(NewSchema(schema.Field(idx("in")).Type.(*StructType).Fields), buf) + start := buf.WriterIndex() + child.Reset() + child.WriteInt32(0, 1) + require.NoError(t, w.SetOffsetAndSize(idx("in"), start, buf.WriterIndex()-start)) + } + aw := NewArrayWriter(schema.Field(idx("l")).Type.(*ListType).Elem, buf) + start := buf.WriterIndex() + require.NoError(t, aw.Reset(2)) + aw.WriteInt32(0, 1) + if nullTarget == "l[1]" { + aw.SetNullAt(1) + } else { + aw.WriteInt32(1, 2) + } + require.NoError(t, w.SetOffsetAndSize(idx("l"), start, buf.WriterIndex()-start)) + return append([]byte(nil), w.ToBytes()...) + } + + decoded, err := enc.FromRow(buildRow("")) + require.NoError(t, err) + require.Equal(t, carriers{S: "x", In: inner{X: 1}, L: []int32{1, 2}}, decoded) + for _, target := range []string{"s", "in", "l[1]"} { + _, err := enc.FromRow(buildRow(target)) + requireErrorContains(t, err, "null value") + } +} + +// Inferred schemas must survive the wire format unchanged, which +// requires the canonical (nullable) map value field. +func TestInferredSchemaRoundTripsThroughBytes(t *testing.T) { + type primitiveMap struct { + Counts map[string]int32 + Flags map[int64]bool + } + for _, schema := range []*Schema{ + mustSchema(t, NewEncoder[primitiveMap]), + mustSchema(t, NewEncoder[person]), + } { + encoded, err := SchemaToBytes(schema) + require.NoError(t, err) + parsed, err := SchemaFromBytes(encoded) + require.NoError(t, err) + require.True(t, parsed.Equal(schema), "parsed %v != inferred %v", parsed, schema) + } + + enc, err := NewEncoder[primitiveMap]() + require.NoError(t, err) + original := primitiveMap{Counts: map[string]int32{"a": 1, "b": -2}, Flags: map[int64]bool{7: true}} + rowBytes, err := enc.ToRow(&original) + require.NoError(t, err) + decoded, err := enc.FromRow(rowBytes) + require.NoError(t, err) + require.Equal(t, original, decoded) +} + +func mustSchema[T any](t *testing.T, newEncoder func() (*Encoder[T], error)) *Schema { + t.Helper() + enc, err := newEncoder() + require.NoError(t, err) + return enc.Schema() +} + +func TestEncoderRejectsNilArguments(t *testing.T) { + enc, err := NewEncoder[person]() + require.NoError(t, err) + _, err = enc.ToRow(nil) + require.Error(t, err) + _, err = enc.Encode(nil) + require.Error(t, err) + require.Error(t, enc.FromRowInto([]byte{}, nil)) +} + +// Struct-typed map keys must round-trip entry for entry. +func TestEncoderStructMapKeys(t *testing.T) { + type point struct { + X, Y int32 + } + type grid struct { + Cells map[point]string + } + enc, err := NewEncoder[grid]() + require.NoError(t, err) + original := grid{Cells: map[point]string{{1, 2}: "a", {3, 4}: "b"}} + rowBytes, err := enc.ToRow(&original) + require.NoError(t, err) + decoded, err := enc.FromRow(rowBytes) + require.NoError(t, err) + require.Equal(t, original, decoded) +} diff --git a/go/fory/row/infer.go b/go/fory/row/infer.go new file mode 100644 index 0000000000..f0e1eb1978 --- /dev/null +++ b/go/fory/row/infer.go @@ -0,0 +1,399 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package row + +import ( + "fmt" + "reflect" + "sort" + "strings" + "time" + "unicode" + "unicode/utf8" + + fory "github.com/apache/fory/go/fory" +) + +var ( + goDateType = reflect.TypeOf(fory.Date{}) + goTimeType = reflect.TypeOf(time.Time{}) + goDurationType = reflect.TypeOf(time.Duration(0)) +) + +// InferSchema infers the row schema for a struct type or pointer to +// struct, including every exported field not ignored by its fory tag +// (`fory:"-"`, `fory:"ignore"`, or `fory:"ignore=true"`). +// +// Fields are sorted by their lowerCamel name and named by its +// snake_case form (UserName -> user_name), matching Java's schema +// inference so both languages derive identical schemas. +// +// Type mapping: +// - bool, int8/16/32/64, float32/64: same-width row types, non-nullable; +// int maps to int64 +// - string: string, nullable +// - []byte: list, nullable, matching Java's byte[] (the row +// format's binary type is reachable only through explicit schemas) +// - fory.Date, time.Time, time.Duration: date32, timestamp, duration, +// nullable (their Java carriers are objects) +// - slices, maps, nested structs: list, map, and struct, nullable; +// map values are always nullable, map keys never +// - *T: the row type of T, nullable +// +// A nullable field whose Go carrier cannot hold nil (a scalar, string, +// temporal, or value struct) still decodes, but a null value for it is +// a decode error; use a pointer carrier when nulls must round-trip. +// +// Unsupported: unsigned integers, fixed-size arrays, nested pointers, +// pointers to slices or maps (two nil states, one null bit), map keys +// whose encoded fields do not determine Go equality (pointers, +// time.Time, structs with unexported or ignored fields), and recursive +// types. +func InferSchema(t reflect.Type) (*Schema, error) { + if t == nil { + return nil, fmt.Errorf("row: cannot infer a schema from a nil type") + } + layout, err := inferStructLayout(t, nil) + if err != nil { + return nil, err + } + return layout.schema, nil +} + +// structLayout maps schema ordinals back to Go struct field indexes. +type structLayout struct { + schema *Schema + indexes []int // schema ordinal -> reflect struct field index +} + +func inferStructLayout(t reflect.Type, path []reflect.Type) (*structLayout, error) { + if t.Kind() == reflect.Ptr { + t = t.Elem() + } + if t.Kind() != reflect.Struct || t == goDateType || t == goTimeType { + return nil, fmt.Errorf("row: schema inference expects a struct type, got %v", t) + } + if err := checkCycle(t, path); err != nil { + return nil, err + } + path = append(path, t) + + type member struct { + lowerCamel string + goIndex int + fieldType reflect.Type + } + var members []member + for i := 0; i < t.NumField(); i++ { + f := t.Field(i) + if f.PkgPath != "" { + continue + } + ignored, err := hasIgnoreTag(f.Tag.Get("fory")) + if err != nil { + return nil, fmt.Errorf("%w (field %s of %v)", err, f.Name, t) + } + if ignored { + continue + } + members = append(members, member{lowerFirst(f.Name), i, f.Type}) + } + // Java sorts by the lowerCamel member name before converting names + // to snake_case; matching that order is required for schema and + // row-layout compatibility. + sort.Slice(members, func(a, b int) bool { return members[a].lowerCamel < members[b].lowerCamel }) + + layout := &structLayout{indexes: make([]int, 0, len(members))} + fields := make([]Field, 0, len(members)) + for _, m := range members { + f, err := inferField(lowerCamelToLowerUnderscore(m.lowerCamel), m.fieldType, path) + if err != nil { + return nil, fmt.Errorf("%w (field %s of %v)", err, t.Field(m.goIndex).Name, t) + } + fields = append(fields, f) + layout.indexes = append(layout.indexes, m.goIndex) + } + layout.schema = NewSchema(fields) + return layout, nil +} + +// checkCycle rejects a type already on the active inference path. Named +// slice and map types can recurse just like structs (type L []L), so +// every composite type is tracked, not only structs. +func checkCycle(t reflect.Type, path []reflect.Type) error { + for _, seen := range path { + if seen == t { + return fmt.Errorf("row: circular reference through type %v", t) + } + } + return nil +} + +func inferField(name string, t reflect.Type, path []reflect.Type) (Field, error) { + if t.Kind() == reflect.Ptr { + switch t.Elem().Kind() { + case reflect.Ptr: + return Field{}, fmt.Errorf("row: nested pointer type %v is unsupported", t) + case reflect.Slice, reflect.Map: + // A nil pointer and a pointer to a nil container would share + // one null bit, so the value could not round-trip. + return Field{}, fmt.Errorf("row: pointer to slice or map type %v is unsupported; use %v", t, t.Elem()) + } + inner, err := inferField(name, t.Elem(), path) + if err != nil { + return Field{}, err + } + inner.Nullable = true + return inner, nil + } + switch t { + case goDateType: + return Field{Name: name, Type: Date32Type{}, Nullable: true}, nil + case goTimeType: + return Field{Name: name, Type: TimestampType{}, Nullable: true}, nil + case goDurationType: + return Field{Name: name, Type: DurationType{}, Nullable: true}, nil + } + switch t.Kind() { + case reflect.Bool: + return Field{Name: name, Type: BoolType{}}, nil + case reflect.Int8: + return Field{Name: name, Type: Int8Type{}}, nil + case reflect.Int16: + return Field{Name: name, Type: Int16Type{}}, nil + case reflect.Int32: + return Field{Name: name, Type: Int32Type{}}, nil + case reflect.Int64, reflect.Int: + // `int` maps to int64 so the width never depends on the platform. + return Field{Name: name, Type: Int64Type{}}, nil + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return Field{}, fmt.Errorf("row: unsigned type %v is unsupported, use a signed type", t) + case reflect.Float32: + return Field{Name: name, Type: Float32Type{}}, nil + case reflect.Float64: + return Field{Name: name, Type: Float64Type{}}, nil + case reflect.String: + return Field{Name: name, Type: StringType{}, Nullable: true}, nil + case reflect.Slice: + if t.Elem().Kind() == reflect.Uint8 { + // Java infers byte[] as list; the encoder copies the + // bytes straight into the element region. + elem := Field{Name: listItemName, Type: Int8Type{}} + return Field{Name: name, Type: &ListType{Elem: elem}, Nullable: true}, nil + } + if err := checkCycle(t, path); err != nil { + return Field{}, err + } + elem, err := inferField(listItemName, t.Elem(), append(path, t)) + if err != nil { + return Field{}, err + } + return Field{Name: name, Type: &ListType{Elem: elem}, Nullable: true}, nil + case reflect.Map: + if err := checkCycle(t, path); err != nil { + return Field{}, err + } + if err := validateMapKeyType(t.Key()); err != nil { + return Field{}, err + } + path = append(path, t) + key, err := inferField(mapKeyName, t.Key(), path) + if err != nil { + return Field{}, err + } + value, err := inferField(mapValueName, t.Elem(), path) + if err != nil { + return Field{}, err + } + // Canonical map children, matching Java MapType and the schema + // parser: keys are never nullable, values always are. + key.Nullable = false + value.Nullable = true + return Field{Name: name, Type: &MapType{Key: key, Value: value}, Nullable: true}, nil + case reflect.Struct: + layout, err := inferStructLayout(t, path) + if err != nil { + return Field{}, err + } + return Field{Name: name, Type: &StructType{Fields: layout.schema.Fields()}, Nullable: true}, nil + default: + return Field{}, fmt.Errorf("row: type %v is unsupported in row format", t) + } +} + +// validateMapKeyType accepts only key types whose encoded fields fully +// determine Go equality, so distinct keys never encode identically and +// decoding never collapses entries: scalars, strings, and structs made +// only of such fields with nothing unexported or ignored. time.Time is +// rejected because its Location is not encoded. +func validateMapKeyType(t reflect.Type) error { + switch t.Kind() { + case reflect.Bool, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int, + reflect.Float32, reflect.Float64, reflect.String: + return nil + case reflect.Struct: + if t == goTimeType { + return fmt.Errorf("row: map key type %v is unsupported: its Location is not encoded", t) + } + for i := 0; i < t.NumField(); i++ { + f := t.Field(i) + if f.PkgPath != "" { + return fmt.Errorf("row: map key type %v has unexported field %s that would not be encoded", t, f.Name) + } + ignored, err := hasIgnoreTag(f.Tag.Get("fory")) + if err != nil { + return err + } + if ignored { + return fmt.Errorf("row: map key type %v has ignored field %s that would not be encoded", t, f.Name) + } + if err := validateMapKeyType(f.Type); err != nil { + return err + } + } + return nil + default: + return fmt.Errorf("row: map key type %v cannot preserve Go equality in row format", t) + } +} + +// Keys accepted by the core fory tag parser; the row format acts only +// on "ignore" but mirrors the grammar so a tag valid for object-graph +// serialization is valid here and vice versa. +var foryTagKeys = map[string]bool{ + "id": true, "nullable": true, "ref": true, "ignore": true, "encoding": true, "type": true, +} + +// hasIgnoreTag mirrors the core fory tag grammar (parseFieldTag in +// field_spec.go): a whole tag of "-" ignores the field; otherwise parts +// split on top-level commas, keys and values are trimmed around '=', +// duplicate keys are errors, and ignore accepts the strict boolean forms +// true/1/yes and false/0/no (case-insensitive), defaulting to true. +func hasIgnoreTag(tag string) (bool, error) { + if tag == "-" { + return true, nil + } + ignore := false + seen := map[string]bool{} + for _, part := range splitTopLevel(tag) { + part = strings.TrimSpace(part) + if part == "" { + continue + } + key, value, hasValue := part, "", false + if idx := indexTopLevel(part, '='); idx >= 0 { + key, value, hasValue = strings.TrimSpace(part[:idx]), strings.TrimSpace(part[idx+1:]), true + } + if !foryTagKeys[key] { + return false, fmt.Errorf("row: unknown fory tag key %q", key) + } + if seen[key] { + return false, fmt.Errorf("row: duplicate fory tag key %q", key) + } + seen[key] = true + if key != "ignore" { + continue + } + if !hasValue { + ignore = true + continue + } + switch strings.ToLower(value) { + case "true", "1", "yes": + ignore = true + case "false", "0", "no": + ignore = false + default: + return false, fmt.Errorf("row: invalid ignore value %q in fory tag", value) + } + } + return ignore, nil +} + +// splitTopLevel splits on commas outside parentheses, so nested type +// hints such as type=map(key=string,value=int32) stay intact. +func splitTopLevel(input string) []string { + var parts []string + depth, start := 0, 0 + for i := 0; i < len(input); i++ { + switch input[i] { + case '(': + depth++ + case ')': + if depth > 0 { + depth-- + } + case ',': + if depth == 0 { + parts = append(parts, input[start:i]) + start = i + 1 + } + } + } + return append(parts, input[start:]) +} + +func indexTopLevel(input string, target byte) int { + depth := 0 + for i := 0; i < len(input); i++ { + switch input[i] { + case '(': + depth++ + case ')': + if depth > 0 { + depth-- + } + default: + if depth == 0 && input[i] == target { + return i + } + } + } + return -1 +} + +func lowerFirst(s string) string { + r, size := utf8.DecodeRuneInString(s) + if !unicode.IsUpper(r) { + return s + } + return string(unicode.ToLower(r)) + s[size:] +} + +// lowerCamelToLowerUnderscore ports Java StringUtils: every uppercase +// letter becomes '_' plus its lowercase, so userID becomes user_i_d. +func lowerCamelToLowerUnderscore(s string) string { + var b strings.Builder + from := 0 + for i := 0; i < len(s); i++ { + c := s[i] + if c >= 'A' && c <= 'Z' { + b.WriteString(s[from:i]) + b.WriteByte('_') + b.WriteByte(c + ('a' - 'A')) + from = i + 1 + } + } + if from == 0 { + return s + } + if from < len(s) { + b.WriteString(s[from:]) + } + return b.String() +} diff --git a/go/fory/row/infer_test.go b/go/fory/row/infer_test.go new file mode 100644 index 0000000000..3ae76df871 --- /dev/null +++ b/go/fory/row/infer_test.go @@ -0,0 +1,220 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package row + +import ( + "reflect" + "testing" + "time" + + fory "github.com/apache/fory/go/fory" + "github.com/stretchr/testify/require" +) + +func TestLowerCamelToLowerUnderscore(t *testing.T) { + cases := map[string]string{ + "id": "id", + "userName": "user_name", + "userID": "user_i_d", // Java quirk, kept for parity + "f2": "f2", + "aVeryLong": "a_very_long", + } + for in, want := range cases { + require.Equal(t, want, lowerCamelToLowerUnderscore(in), in) + } +} + +// Fields sort by lowerCamel name before snake_case conversion, +// matching Java's Descriptor ordering. +func TestInferFieldOrder(t *testing.T) { + type ordered struct { + ZItem int64 + AlphaBeta string + M2 int32 + } + s, err := InferSchema(reflect.TypeOf(ordered{})) + require.NoError(t, err) + require.Equal(t, 3, s.NumFields()) + require.Equal(t, "alpha_beta", s.Field(0).Name) + require.Equal(t, "m2", s.Field(1).Name) + require.Equal(t, "z_item", s.Field(2).Name) +} + +func TestInferTypeMapping(t *testing.T) { + type inner struct { + X float64 + } + type sample struct { + B bool + I8 int8 + I16 int16 + I32 int32 + I64 int64 + I int + F32 float32 + F64 float64 + S string + Bin []byte + L []int32 + M map[string]int64 + In inner + PI32 *int32 + PIn *inner + D fory.Date + T time.Time + Dur time.Duration + } + s, err := InferSchema(reflect.TypeOf(sample{})) + require.NoError(t, err) + + byName := func(name string) Field { return s.Field(s.FieldIndex(name)) } + + require.Equal(t, BoolType{}, byName("b").Type) + require.False(t, byName("b").Nullable) + require.Equal(t, Int8Type{}, byName("i8").Type) + require.Equal(t, Int16Type{}, byName("i16").Type) + require.Equal(t, Int32Type{}, byName("i32").Type) + require.Equal(t, Int64Type{}, byName("i64").Type) + require.Equal(t, Int64Type{}, byName("i").Type, "int maps to int64") + require.Equal(t, Float32Type{}, byName("f32").Type) + require.Equal(t, Float64Type{}, byName("f64").Type) + + require.Equal(t, StringType{}, byName("s").Type) + require.True(t, byName("s").Nullable) + bin := byName("bin").Type.(*ListType) + require.Equal(t, Int8Type{}, bin.Elem.Type, "[]byte is list, the Java byte[] model") + require.False(t, bin.Elem.Nullable) + require.True(t, byName("bin").Nullable) + + list := byName("l").Type.(*ListType) + require.Equal(t, Int32Type{}, list.Elem.Type) + require.False(t, list.Elem.Nullable, "value-type slice elements cannot be null") + require.True(t, byName("l").Nullable) + + m := byName("m").Type.(*MapType) + require.Equal(t, StringType{}, m.Key.Type) + require.False(t, m.Key.Nullable) + require.Equal(t, Int64Type{}, m.Value.Type) + require.True(t, m.Value.Nullable, "map values are canonically nullable, as in Java and the schema parser") + + in := byName("in").Type.(*StructType) + require.Equal(t, "x", in.Fields[0].Name) + require.True(t, byName("in").Nullable) + + require.Equal(t, Int32Type{}, byName("p_i32").Type) + require.True(t, byName("p_i32").Nullable, "pointer fields are nullable") + require.True(t, byName("p_in").Nullable) + + require.Equal(t, Date32Type{}, byName("d").Type) + require.Equal(t, TimestampType{}, byName("t").Type) + require.Equal(t, DurationType{}, byName("dur").Type) + for _, name := range []string{"d", "t", "dur"} { + require.True(t, byName(name).Nullable, "%s: Java temporal carriers are nullable objects", name) + } +} + +// Tag grammar must match the core fory tag parser: "-", "ignore", +// ignore=true/1/yes skip (case-insensitive, whitespace around '=' +// allowed); ignore=false/0/no keeps; other core keys are accepted and +// ignored; unknown keys, duplicate keys, and bad values are errors. +func TestInferSkipsIgnoredAndUnexported(t *testing.T) { + type tagged struct { + A int64 + hidden int64 + Skipped string `fory:"ignore"` + Dash string `fory:"-"` + Yes string `fory:"ignore = YES"` + One string `fory:"id=3, ignore=1"` + Kept int32 `fory:"ignore=false"` + Kept2 int32 `fory:"nullable=true, ignore=no"` + Kept3 int32 `fory:"type=map(key=string,value=int32)"` + } + s, err := InferSchema(reflect.TypeOf(tagged{})) + require.NoError(t, err) + require.Equal(t, 4, s.NumFields()) + require.Equal(t, "a", s.Field(0).Name) + require.Equal(t, "kept", s.Field(1).Name) + require.Equal(t, "kept2", s.Field(2).Name) + require.Equal(t, "kept3", s.Field(3).Name) + + for _, bad := range []any{ + struct { + A int64 `fory:"ignore=maybe"` + }{}, + struct { + A int64 `fory:"ignore, ignore"` + }{}, + struct { + A int64 `fory:"bogus=1"` + }{}, + } { + _, err := InferSchema(reflect.TypeOf(bad)) + require.Error(t, err, "%T", bad) + } +} + +func TestInferRejectsUnsupportedTypes(t *testing.T) { + type node struct { + Next *node + } + type recursiveList []recursiveList + type recursiveMap map[string]recursiveMap + type keyWithHidden struct { + A int32 + b int32 + } + cases := []any{ + struct{ U uint32 }{}, + struct{ C chan int }{}, + struct{ A [3]int32 }{}, + struct{ PP **int32 }{}, + struct{ PS *[]int32 }{}, + struct{ PM *map[string]int32 }{}, + struct{ PB *[]byte }{}, + struct{ M map[*string]int32 }{}, + struct{ M map[keyWithHidden]int32 }{}, + struct{ M map[time.Time]int32 }{}, + struct{ M map[[2]int32]int32 }{}, + node{}, + struct{ L recursiveList }{}, + struct{ M recursiveMap }{}, + } + for _, c := range cases { + _, err := InferSchema(reflect.TypeOf(c)) + require.Error(t, err, "%T", c) + } + _, err := InferSchema(reflect.TypeOf(42)) + require.Error(t, err) + _, err = InferSchema(nil) + require.Error(t, err) +} + +// Map keys whose encoded fields fully determine Go equality are +// accepted, including value structs of such fields. +func TestInferAcceptsEqualityPreservingMapKeys(t *testing.T) { + type point struct { + X, Y int32 + } + type keyed struct { + ByDate map[fory.Date]string + ByPoint map[point]string + ByDur map[time.Duration]string + } + _, err := InferSchema(reflect.TypeOf(keyed{})) + require.NoError(t, err) +} diff --git a/go/fory/row/map_test.go b/go/fory/row/map_test.go new file mode 100644 index 0000000000..e1f1fa4f93 --- /dev/null +++ b/go/fory/row/map_test.go @@ -0,0 +1,90 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package row + +import ( + "encoding/binary" + "testing" + + fory "github.com/apache/fory/go/fory" + "github.com/stretchr/testify/require" +) + +// Maps are an 8-byte keys-array size followed by two complete arrays; +// both halves must decode independently. +func TestMapLayout(t *testing.T) { + buf := fory.NewByteBuffer(nil) + mapType := Map(StringType{}, Int64Type{}) + m := NewMapWriter(buf) + m.Reset() + keys := NewArrayWriter(mapType.Key, buf) + require.NoError(t, keys.Reset(2)) + keys.WriteString(0, "k1") + keys.WriteString(1, "k2") + m.FinishKeys() + values := NewArrayWriter(mapType.Value, buf) + require.NoError(t, values.Reset(2)) + values.WriteInt64(0, 10) + values.WriteInt64(1, 20) + + data := buf.GetByteSlice(0, buf.WriterIndex()) + // keys array: 8 count + 8 bitmap + 2*8 slots + 2*8 padded strings. + require.Equal(t, uint64(48), binary.LittleEndian.Uint64(data)) + require.Equal(t, 8+48+32, m.Size()) + + md := NewMapData(mapType, data) + require.Equal(t, 2, md.NumElements()) + require.Equal(t, "k1", md.Keys().String(0)) + require.Equal(t, "k2", md.Keys().String(1)) + require.Equal(t, int64(10), md.Values().Int64(0)) + require.Equal(t, int64(20), md.Values().Int64(1)) +} + +func TestMapFieldInRow(t *testing.T) { + mapType := Map(StringType{}, Int32Type{}) + s := NewSchema([]Field{ + NewField("id", Int64Type{}, false), + NewField("attrs", mapType, true), + }) + w := NewRowWriter(s) + w.Reset() + w.WriteInt64(0, 1) + + buf := w.Buffer() + start := buf.WriterIndex() + m := NewMapWriter(buf) + m.Reset() + keys := NewArrayWriter(mapType.Key, buf) + require.NoError(t, keys.Reset(2)) + keys.WriteString(0, "a") + keys.WriteString(1, "b") + m.FinishKeys() + values := NewArrayWriter(mapType.Value, buf) + require.NoError(t, values.Reset(2)) + values.WriteInt32(0, 1) + values.SetNullAt(1) + w.SetOffsetAndSize(1, start, buf.WriterIndex()-start) + + r := NewRow(s, w.ToBytes()) + md := r.Map(1) + require.NotNil(t, md) + require.Equal(t, 2, md.NumElements()) + require.Equal(t, "a", md.Keys().String(0)) + require.Equal(t, int32(1), md.Values().Int32(0)) + require.True(t, md.Values().IsNullAt(1)) +} diff --git a/go/fory/row/row.go b/go/fory/row/row.go new file mode 100644 index 0000000000..be6e0e30ef --- /dev/null +++ b/go/fory/row/row.go @@ -0,0 +1,392 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package row + +import ( + "encoding/binary" + "fmt" + "math" + "time" + + fory "github.com/apache/fory/go/fory" +) + +// Row is a zero-copy reader over one row's bytes. Getters read directly +// from the underlying data without deserializing other fields; nested +// values are sub-slice views, never copies (except String). +// +// Row and its nested views require trusted, schema-matched bytes: the +// row format is a trusted in-memory format and these views do not +// validate the row graph. Fixed-width getters return the zero value for +// null fields; use IsNullAt to distinguish. Variable-width getters +// return nil (or "") for null fields. Out-of-range indexes and +// malformed bytes panic. +type Row struct { + fields []Field + data []byte + bitmapWidth int +} + +func NewRow(schema *Schema, data []byte) *Row { + return newRow(schema.Fields(), data) +} + +func newRow(fields []Field, data []byte) *Row { + return &Row{fields: fields, data: data, bitmapWidth: bitmapWidthInBytes(len(fields))} +} + +func (r *Row) NumFields() int { return len(r.fields) } +func (r *Row) SizeBytes() int { return len(r.data) } + +// Data returns the underlying row bytes; callers must not modify them. +func (r *Row) Data() []byte { return r.data } + +func (r *Row) IsNullAt(i int) bool { + if uint(i) >= uint(len(r.fields)) { + panic(fmt.Sprintf("row: field index %d out of range [0, %d)", i, len(r.fields))) + } + return getBit(r.data, i) +} + +func (r *Row) slot(i int) int { return r.bitmapWidth + i*8 } + +func (r *Row) Bool(i int) bool { + return !r.IsNullAt(i) && r.data[r.slot(i)] != 0 +} + +func (r *Row) Int8(i int) int8 { + if r.IsNullAt(i) { + return 0 + } + return int8(r.data[r.slot(i)]) +} + +func (r *Row) Int16(i int) int16 { + if r.IsNullAt(i) { + return 0 + } + return int16(binary.LittleEndian.Uint16(r.data[r.slot(i):])) +} + +func (r *Row) Int32(i int) int32 { + if r.IsNullAt(i) { + return 0 + } + return int32(binary.LittleEndian.Uint32(r.data[r.slot(i):])) +} + +func (r *Row) Int64(i int) int64 { + if r.IsNullAt(i) { + return 0 + } + return int64(binary.LittleEndian.Uint64(r.data[r.slot(i):])) +} + +func (r *Row) Float32(i int) float32 { + if r.IsNullAt(i) { + return 0 + } + return math.Float32frombits(binary.LittleEndian.Uint32(r.data[r.slot(i):])) +} + +func (r *Row) Float64(i int) float64 { + if r.IsNullAt(i) { + return 0 + } + return math.Float64frombits(binary.LittleEndian.Uint64(r.data[r.slot(i):])) +} + +func (r *Row) Date(i int) fory.Date { + if r.IsNullAt(i) { + return fory.Date{} + } + return dateFromDays(int32(binary.LittleEndian.Uint32(r.data[r.slot(i):]))) +} + +func (r *Row) Timestamp(i int) time.Time { + if r.IsNullAt(i) { + return time.Time{} + } + return time.UnixMicro(r.Int64(i)) +} + +func (r *Row) Duration(i int) time.Duration { + return durationFromMicros(r.Int64(i)) +} + +// varData returns the value bytes of variable-width field i, or nil if +// the field is null. An empty value returns an empty non-nil slice. +func (r *Row) varData(i int) []byte { + if r.IsNullAt(i) { + return nil + } + offset, size := decodeOffsetAndSize(binary.LittleEndian.Uint64(r.data[r.slot(i):])) + return boundedSlice(r.data, offset, size) +} + +// Binary returns a zero-copy view of field i's bytes. +func (r *Row) Binary(i int) []byte { return r.varData(i) } + +func (r *Row) String(i int) string { return string(r.varData(i)) } + +func (r *Row) Struct(i int) *Row { + data := r.varData(i) + if data == nil { + return nil + } + structType := r.fields[i].Type.(*StructType) + return newRow(structType.Fields, data) +} + +func (r *Row) Array(i int) *ArrayData { + data := r.varData(i) + if data == nil { + return nil + } + listType := r.fields[i].Type.(*ListType) + return NewArrayData(listType.Elem, data) +} + +func (r *Row) Map(i int) *MapData { + data := r.varData(i) + if data == nil { + return nil + } + return NewMapData(r.fields[i].Type.(*MapType), data) +} + +// ArrayData is a zero-copy reader over one array's bytes, with the same +// null and bounds semantics as Row. +type ArrayData struct { + elem Field + data []byte + numElements int + elemSize int + headerBytes int +} + +func NewArrayData(elem Field, data []byte) *ArrayData { + // Narrow the wire count only when it is far below int range, so + // header and size arithmetic cannot overflow on any platform; an + // absurd count becomes -1, which validateBounds and every index + // check reject. + numElements := -1 + if count := binary.LittleEndian.Uint64(data); count <= uint64(math.MaxInt/8) { + numElements = int(count) + } + elemSize := elem.Type.ByteWidth() + if elemSize < 0 { + elemSize = 8 + } + return &ArrayData{ + elem: elem, + data: data, + numElements: numElements, + elemSize: elemSize, + headerBytes: 8 + bitmapWidthInBytes(numElements), + } +} + +func (a *ArrayData) NumElements() int { return a.numElements } +func (a *ArrayData) SizeBytes() int { return len(a.data) } + +// validateBounds rejects arrays whose declared element count is not +// covered by the available bytes, so decoders can check before +// allocating from an attacker-declared count. +func (a *ArrayData) validateBounds() error { + // The coarse count check involves no arithmetic, so it cannot be + // defeated by overflow: every element occupies at least one data + // byte. It also bounds numElements low enough that headerBytes + // (computed in NewArrayData) and the product below are exact. + if a.numElements < 0 || a.numElements > len(a.data) { + return fmt.Errorf("row: array declares %d elements but holds only %d bytes", a.numElements, len(a.data)) + } + need := int64(a.headerBytes) + int64(a.numElements)*int64(a.elemSize) + if need > int64(len(a.data)) { + return fmt.Errorf("row: array declares %d elements needing %d bytes but holds only %d", a.numElements, need, len(a.data)) + } + return nil +} + +func (a *ArrayData) IsNullAt(i int) bool { + if uint(i) >= uint(a.numElements) { + panic(fmt.Sprintf("row: array index %d out of range [0, %d)", i, a.numElements)) + } + return getBit(a.data[8:a.headerBytes], i) +} + +func (a *ArrayData) slot(i int) int { return a.headerBytes + i*a.elemSize } + +func (a *ArrayData) Bool(i int) bool { + return !a.IsNullAt(i) && a.data[a.slot(i)] != 0 +} + +func (a *ArrayData) Int8(i int) int8 { + if a.IsNullAt(i) { + return 0 + } + return int8(a.data[a.slot(i)]) +} + +func (a *ArrayData) Int16(i int) int16 { + if a.IsNullAt(i) { + return 0 + } + return int16(binary.LittleEndian.Uint16(a.data[a.slot(i):])) +} + +func (a *ArrayData) Int32(i int) int32 { + if a.IsNullAt(i) { + return 0 + } + return int32(binary.LittleEndian.Uint32(a.data[a.slot(i):])) +} + +func (a *ArrayData) Int64(i int) int64 { + if a.IsNullAt(i) { + return 0 + } + return int64(binary.LittleEndian.Uint64(a.data[a.slot(i):])) +} + +func (a *ArrayData) Float32(i int) float32 { + if a.IsNullAt(i) { + return 0 + } + return math.Float32frombits(binary.LittleEndian.Uint32(a.data[a.slot(i):])) +} + +func (a *ArrayData) Float64(i int) float64 { + if a.IsNullAt(i) { + return 0 + } + return math.Float64frombits(binary.LittleEndian.Uint64(a.data[a.slot(i):])) +} + +func (a *ArrayData) Date(i int) fory.Date { + if a.IsNullAt(i) { + return fory.Date{} + } + return dateFromDays(int32(binary.LittleEndian.Uint32(a.data[a.slot(i):]))) +} + +func (a *ArrayData) Timestamp(i int) time.Time { + if a.IsNullAt(i) { + return time.Time{} + } + return time.UnixMicro(a.Int64(i)) +} + +func (a *ArrayData) Duration(i int) time.Duration { + return durationFromMicros(a.Int64(i)) +} + +func (a *ArrayData) varData(i int) []byte { + if a.IsNullAt(i) { + return nil + } + offset, size := decodeOffsetAndSize(binary.LittleEndian.Uint64(a.data[a.slot(i):])) + return boundedSlice(a.data, offset, size) +} + +// Binary returns a zero-copy view of element i's bytes. +func (a *ArrayData) Binary(i int) []byte { return a.varData(i) } + +func (a *ArrayData) String(i int) string { return string(a.varData(i)) } + +func (a *ArrayData) Struct(i int) *Row { + data := a.varData(i) + if data == nil { + return nil + } + return newRow(a.elem.Type.(*StructType).Fields, data) +} + +func (a *ArrayData) Array(i int) *ArrayData { + data := a.varData(i) + if data == nil { + return nil + } + return NewArrayData(a.elem.Type.(*ListType).Elem, data) +} + +func (a *ArrayData) Map(i int) *MapData { + data := a.varData(i) + if data == nil { + return nil + } + return NewMapData(a.elem.Type.(*MapType), data) +} + +// MapData is a zero-copy reader over one map's bytes: the keys array +// and values array views share the map's underlying data. +type MapData struct { + keys *ArrayData + values *ArrayData +} + +func NewMapData(mapType *MapType, data []byte) *MapData { + keysSize64 := binary.LittleEndian.Uint64(data) + if keysSize64 > uint64(len(data)) { + panic(fmt.Sprintf("row: map keys array of %d bytes exceeds the %d-byte map region", keysSize64, len(data))) + } + keysSize := int(keysSize64) + keysData := boundedSlice(data, 8, keysSize) + valuesData := boundedSlice(data, 8+keysSize, len(data)-8-keysSize) + return &MapData{ + keys: NewArrayData(mapType.Key, keysData), + values: NewArrayData(mapType.Value, valuesData), + } +} + +// boundedSlice checks declared bounds against the data LENGTH +// and caps the view so nested +// readers cannot reach outside it either. Out-of-bounds panics are +// converted to errors by the encoder's decode entry points. +func boundedSlice(data []byte, offset, size int) []byte { + end := offset + size + if offset < 0 || size < 0 || end > len(data) { + panic(fmt.Sprintf("row: value bytes [%d:%d] exceed the enclosing %d-byte region", offset, end, len(data))) + } + return data[offset:end:end] +} + +func (m *MapData) NumElements() int { return m.keys.numElements } +func (m *MapData) Keys() *ArrayData { return m.keys } +func (m *MapData) Values() *ArrayData { return m.values } + +func decodeOffsetAndSize(slotValue uint64) (offset, size int) { + return int(slotValue >> 32), int(uint32(slotValue)) +} + +// dateFromDays converts int32 epoch days to a Date. The int32 range +// always yields an in-range year, so the conversion cannot fail. +func dateFromDays(days int32) fory.Date { + d, _ := fory.DateFromEpochDay(int64(days)) + return d +} + +// durationFromMicros converts wire microseconds to a time.Duration, +// which counts nanoseconds; values the multiplication would wrap are +// rejected instead of silently changing sign. +func durationFromMicros(micros int64) time.Duration { + if micros > math.MaxInt64/1000 || micros < math.MinInt64/1000 { + panic(fmt.Sprintf("row: duration of %d microseconds overflows time.Duration", micros)) + } + return time.Duration(micros) * time.Microsecond +} diff --git a/go/fory/row/row_test.go b/go/fory/row/row_test.go new file mode 100644 index 0000000000..3559bb402a --- /dev/null +++ b/go/fory/row/row_test.go @@ -0,0 +1,311 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package row + +import ( + "math" + "strings" + "sync" + "testing" + "time" + + fory "github.com/apache/fory/go/fory" + "github.com/stretchr/testify/require" +) + +func int64StringSchema() *Schema { + return NewSchema([]Field{ + NewField("f1", Int64Type{}, false), + NewField("f2", StringType{}, true), + }) +} + +// The full byte image is fixed by the spec: 8-byte bitmap, one 8-byte +// slot per field (variable-width slots hold offset<<32|size relative to +// the row base), then the variable region zero-padded to 8 bytes. +func TestRowExactLayout(t *testing.T) { + w := NewRowWriter(int64StringSchema()) + w.Reset() + w.WriteInt64(0, 0x0102030405060708) + w.WriteString(1, "ab") + require.Equal(t, []byte{ + 0, 0, 0, 0, 0, 0, 0, 0, // null bitmap + 0x08, 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, // f1 slot + 0x02, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, // f2 slot: size=2, offset=24 + 'a', 'b', 0, 0, 0, 0, 0, 0, // variable region, zero-padded + }, w.ToBytes()) + + r := NewRow(w.Schema(), w.ToBytes()) + require.Equal(t, int64(0x0102030405060708), r.Int64(0)) + require.Equal(t, "ab", r.String(1)) +} + +func TestNullAndEmptyString(t *testing.T) { + w := NewRowWriter(int64StringSchema()) + w.Reset() + w.WriteInt64(0, 42) + w.SetNullAt(1) + require.Equal(t, []byte{ + 0x02, 0, 0, 0, 0, 0, 0, 0, // bit 1 set: f2 is null + 42, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, // null slot zeroed + }, w.ToBytes()) + r := NewRow(w.Schema(), w.ToBytes()) + require.True(t, r.IsNullAt(1)) + require.Nil(t, r.Binary(1)) + require.Equal(t, "", r.String(1)) + + // An empty string is not null: bit clear, size 0, no variable bytes. + w.Buffer().SetWriterIndex(0) + w.Reset() + w.WriteInt64(0, 42) + w.WriteString(1, "") + r = NewRow(w.Schema(), w.ToBytes()) + require.Equal(t, 24, r.SizeBytes()) + require.False(t, r.IsNullAt(1)) + require.Equal(t, "", r.String(1)) + require.NotNil(t, r.Binary(1)) + require.Empty(t, r.Binary(1)) +} + +func TestPrimitiveRoundTrip(t *testing.T) { + s := NewSchema([]Field{ + NewField("b", BoolType{}, false), + NewField("i8", Int8Type{}, false), + NewField("i16", Int16Type{}, false), + NewField("i32", Int32Type{}, false), + NewField("i64", Int64Type{}, false), + NewField("f32", Float32Type{}, false), + NewField("f64", Float64Type{}, false), + NewField("bin", BinaryType{}, true), + }) + w := NewRowWriter(s) + w.Reset() + w.WriteBool(0, true) + w.WriteInt8(1, -5) + w.WriteInt16(2, -1000) + w.WriteInt32(3, -100000) + w.WriteInt64(4, -5_000_000_000_000) + w.WriteFloat32(5, 3.5) + w.WriteFloat64(6, -2.25) + w.WriteBytes(7, []byte{1, 2, 3}) + + r := NewRow(s, w.ToBytes()) + require.True(t, r.Bool(0)) + require.Equal(t, int8(-5), r.Int8(1)) + require.Equal(t, int16(-1000), r.Int16(2)) + require.Equal(t, int32(-100000), r.Int32(3)) + require.Equal(t, int64(-5_000_000_000_000), r.Int64(4)) + require.Equal(t, float32(3.5), r.Float32(5)) + require.Equal(t, -2.25, r.Float64(6)) + require.Equal(t, []byte{1, 2, 3}, r.Binary(7)) +} + +func TestTemporalRoundTrip(t *testing.T) { + s := NewSchema([]Field{ + NewField("d", Date32Type{}, false), + NewField("ts", TimestampType{}, false), + NewField("dur", DurationType{}, false), + }) + w := NewRowWriter(s) + w.Reset() + date := fory.Date{Year: 2023, Month: time.March, Day: 15} + require.NoError(t, w.WriteDate(0, date)) + w.WriteTimestamp(1, time.UnixMicro(1_234_567_890_123_456)) + w.WriteDuration(2, 90*time.Second) + + r := NewRow(s, w.ToBytes()) + require.Equal(t, date, r.Date(0)) + require.Equal(t, int64(1_234_567_890_123_456), r.Timestamp(1).UnixMicro()) + require.Equal(t, 90*time.Second, r.Duration(2)) +} + +// A 65-field schema needs a 16-byte bitmap, shifting every slot by 16. +func TestMultiWordBitmap(t *testing.T) { + fields := make([]Field, 65) + for i := range fields { + fields[i] = NewField("f"+string(rune('a'+i%26))+string(rune('0'+i/26)), Int64Type{}, true) + } + s := NewSchema(fields) + w := NewRowWriter(s) + w.Reset() + w.WriteInt64(0, 100) + w.WriteInt64(64, 200) + w.SetNullAt(63) + + data := w.ToBytes() + require.Equal(t, 16+65*8, len(data)) + require.Equal(t, byte(0x80), data[7], "bit 63 is the top bit of bitmap byte 7") + + r := NewRow(s, data) + require.Equal(t, int64(100), r.Int64(0)) + require.Equal(t, int64(200), r.Int64(64)) + require.True(t, r.IsNullAt(63)) + require.Equal(t, int64(0), r.Int64(63)) +} + +// The buffer reuses dirty capacity, so a rewritten smaller row must +// still produce byte-identical output with zeroed padding and slots. +func TestWriterReuseZeroesDirtyBuffer(t *testing.T) { + w := NewRowWriter(int64StringSchema()) + w.Reset() + w.WriteInt64(0, 1) + w.WriteString(1, strings.Repeat("Z", 64)) + + w.Buffer().SetWriterIndex(0) + w.Reset() + w.WriteInt64(0, 0x0102030405060708) + w.WriteString(1, "ab") + require.Equal(t, []byte{ + 0, 0, 0, 0, 0, 0, 0, 0, + 0x08, 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, + 0x02, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, + 'a', 'b', 0, 0, 0, 0, 0, 0, + }, w.ToBytes()) + + w.Buffer().SetWriterIndex(0) + w.Reset() + w.WriteInt64(0, 42) + w.SetNullAt(1) + require.Equal(t, []byte{ + 0x02, 0, 0, 0, 0, 0, 0, 0, + 42, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + }, w.ToBytes()) +} + +func TestNestedStruct(t *testing.T) { + inner := Struct([]Field{ + NewField("x", Float64Type{}, false), + NewField("s", StringType{}, true), + }) + s := NewSchema([]Field{ + NewField("id", Int64Type{}, false), + NewField("inner", inner, true), + }) + w := NewRowWriter(s) + w.Reset() + w.WriteInt64(0, 7) + child := NewRowWriterWithBuffer(NewSchema(inner.Fields), w.Buffer()) + start := w.Buffer().WriterIndex() + child.Reset() + child.WriteFloat64(0, 3.5) + child.WriteString(1, "hi") + w.SetOffsetAndSize(1, start, w.Buffer().WriterIndex()-start) + + r := NewRow(s, w.ToBytes()) + require.Equal(t, int64(7), r.Int64(0)) + nested := r.Struct(1) + require.NotNil(t, nested) + require.Equal(t, 3.5, nested.Float64(0)) + require.Equal(t, "hi", nested.String(1)) +} + +// A value larger than the initial buffer forces mid-row growth; slots +// patched after reallocation must still land in the right place. +func TestLargeStringGrowth(t *testing.T) { + s := NewSchema([]Field{NewField("s", StringType{}, true)}) + w := NewRowWriter(s) + w.Reset() + big := strings.Repeat("x", 5000) + w.WriteString(0, big) + + r := NewRow(s, w.ToBytes()) + require.Equal(t, big, r.String(0)) + require.Equal(t, 8+8+roundToWord(5000), r.SizeBytes()) +} + +// One-past-the-end indices must be rejected, matching the other +// row format implementations. +func TestOutOfRangeIndexPanics(t *testing.T) { + w := NewRowWriter(int64StringSchema()) + w.Reset() + require.Panics(t, func() { w.WriteInt64(2, 1) }) + require.Panics(t, func() { w.SetNullAt(-1) }) + + r := NewRow(w.Schema(), w.ToBytes()) + require.Panics(t, func() { r.Int64(2) }) + require.Panics(t, func() { r.IsNullAt(-1) }) +} + +// The wire format packs offset and size into 32 bits each; larger +// values must be rejected, never silently truncated. +func TestOffsetAndSizeRejectWireLimit(t *testing.T) { + if uint64(math.MaxInt) <= math.MaxUint32 { + t.Skip("int cannot exceed the 32-bit wire limit on this platform") + } + tooBig := int(math.MaxInt) + w := NewRowWriter(int64StringSchema()) + w.Reset() + require.Error(t, w.SetOffsetAndSize(1, w.Buffer().WriterIndex(), tooBig)) + + aw := NewArrayWriter(List(StringType{}).Elem, w.Buffer()) + require.NoError(t, aw.Reset(1)) + require.Error(t, aw.SetOffsetAndSize(0, w.Buffer().WriterIndex(), tooBig)) +} + +// Strings are UTF-8 on the wire; arbitrary Go byte strings are rejected +// so other runtimes never see replacement characters. +func TestWriteStringRejectsInvalidUTF8(t *testing.T) { + w := NewRowWriter(int64StringSchema()) + w.Reset() + require.Error(t, w.WriteString(1, string([]byte{0xff, 'a'}))) + aw := NewArrayWriter(List(StringType{}).Elem, w.Buffer()) + require.NoError(t, aw.Reset(1)) + require.Error(t, aw.WriteString(0, string([]byte{0xc3}))) +} + +// Timestamps outside the int64 microsecond range and durations whose +// nanosecond conversion would wrap are rejected instead of corrupted. +func TestTemporalRangeChecks(t *testing.T) { + s := NewSchema([]Field{ + NewField("ts", TimestampType{}, true), + NewField("dur", DurationType{}, true), + }) + w := NewRowWriter(s) + w.Reset() + farFuture := time.Date(300000, time.January, 1, 0, 0, 0, 0, time.UTC) + require.Error(t, w.WriteTimestamp(0, farFuture)) + require.NoError(t, w.WriteTimestamp(0, time.UnixMicro(math.MaxInt64))) + + w.WriteInt64(1, math.MaxInt64) // raw microseconds that overflow time.Duration + r := NewRow(s, w.ToBytes()) + require.Equal(t, int64(math.MaxInt64), r.Timestamp(0).UnixMicro()) + require.Panics(t, func() { r.Duration(1) }) +} + +func TestConcurrentReads(t *testing.T) { + w := NewRowWriter(int64StringSchema()) + w.Reset() + w.WriteInt64(0, 99) + w.WriteString(1, "shared") + r := NewRow(w.Schema(), w.ToBytes()) + + var wg sync.WaitGroup + for g := 0; g < 4; g++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < 200; j++ { + require.Equal(t, int64(99), r.Int64(0)) + require.Equal(t, "shared", r.String(1)) + } + }() + } + wg.Wait() +} diff --git a/go/fory/row/schema_bytes.go b/go/fory/row/schema_bytes.go new file mode 100644 index 0000000000..f2279cb3ec --- /dev/null +++ b/go/fory/row/schema_bytes.go @@ -0,0 +1,417 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package row + +import ( + "fmt" + "math" + + fory "github.com/apache/fory/go/fory" + "github.com/apache/fory/go/fory/meta" +) + +// Schema wire format, byte-compatible with Java SchemaEncoder: +// +// | version (1 byte) | num_fields (var uint32 small7) | fields... | +// +// Field: | header (1 byte) | name size (varint, only if large) | name +// bytes | type info |. Header bits 0-1 hold the index into +// fieldNameEncodings, bits 2-5 hold nameSize-1 (15 means a varint with +// nameSize-15 follows), bit 6 is the nullable flag. Type info is the +// type id byte, plus precision+scale for DECIMAL, a recursive element +// field for LIST, recursive key and value fields for MAP, and a field +// count plus recursive fields for STRUCT. +const ( + schemaVersion = 1 + fieldNameSizeThreshold = 15 + // Caps readField/readType recursion. + maxSchemaNestingDepth = 64 +) + +// Header bits 0-1 store the INDEX into this table (matching Java +// SchemaEncoder.FIELD_NAME_ENCODINGS order), not the meta.Encoding +// wire values, which differ. +var fieldNameEncodings = [3]meta.Encoding{ + meta.UTF_8, + meta.ALL_TO_LOWER_SPECIAL, + meta.LOWER_UPPER_DIGIT_SPECIAL, +} + +// Stateless after construction, safe for concurrent use. +var ( + fieldNameEncoder = meta.NewEncoder('$', '_') + fieldNameDecoder = meta.NewDecoder('$', '_') +) + +func encodingIndexOf(encoding meta.Encoding) int { + for i, e := range fieldNameEncodings { + if e == encoding { + return i + } + } + return -1 +} + +// SchemaToBytes serializes a schema to the cross-language wire format. +func SchemaToBytes(s *Schema) ([]byte, error) { + buf := fory.NewByteBuffer(nil) + if err := SchemaToBuffer(s, buf); err != nil { + return nil, err + } + return buf.GetByteSlice(0, buf.WriterIndex()), nil +} + +// SchemaToBuffer serializes a schema into an existing buffer. The +// schema is validated in full before any byte is written, so a +// rejected schema leaves the buffer untouched. +func SchemaToBuffer(s *Schema, buf *fory.ByteBuffer) error { + if s == nil { + return fmt.Errorf("row: cannot serialize a nil schema") + } + for _, f := range s.Fields() { + if err := validateSchemaField(f, nil, 0); err != nil { + return err + } + } + buf.WriteByte_(schemaVersion) + buf.WriteVarUint32Small7(uint32(s.NumFields())) + for _, f := range s.Fields() { + if err := writeSchemaField(buf, f); err != nil { + return err + } + } + return nil +} + +// validateSchemaField checks what the wire format can express before +// writing: nesting within maxSchemaNestingDepth (so the reader accepts +// the output), no composite type reachable from itself, decimal +// parameters within one byte, and only the DataType implementations +// the format defines. A depth of d means the field sits d levels below +// the top-level schema, matching the reader's depth counter. +func validateSchemaField(f Field, path []DataType, depth int) error { + if depth >= maxSchemaNestingDepth { + return fmt.Errorf("row: schema nesting exceeds %d levels", maxSchemaNestingDepth) + } + if f.Name == "" { + return fmt.Errorf("row: schema field with empty name") + } + switch t := f.Type.(type) { + case BoolType, Int8Type, Int16Type, Int32Type, Int64Type, Float16Type, Float32Type, + Float64Type, StringType, BinaryType, Date32Type, TimestampType, DurationType: + return nil + case DecimalType: + if t.Precision < 0 || t.Precision > math.MaxUint8 || t.Scale < 0 || t.Scale > math.MaxUint8 { + return fmt.Errorf("row: decimal(%d, %d) in field %q is outside the wire format's single-byte range", t.Precision, t.Scale, f.Name) + } + return nil + case *ListType, *MapType, *StructType: + for _, seen := range path { + if seen == f.Type { + return fmt.Errorf("row: schema type %T in field %q refers to itself", f.Type, f.Name) + } + } + path = append(path, f.Type) + switch t := t.(type) { + case *ListType: + return validateSchemaField(t.Elem, path, depth+1) + case *MapType: + if err := validateSchemaField(t.Key, path, depth+1); err != nil { + return err + } + return validateSchemaField(t.Value, path, depth+1) + case *StructType: + for _, child := range t.Fields { + if err := validateSchemaField(child, path, depth+1); err != nil { + return err + } + } + return nil + } + return nil + case nil: + return fmt.Errorf("row: schema field %q has no type", f.Name) + default: + return fmt.Errorf("row: schema field %q uses %T, which the wire format does not define (use DecimalType by value)", f.Name, f.Type) + } +} + +func writeSchemaField(buf *fory.ByteBuffer, f Field) error { + encoding := fieldNameEncoder.ComputeEncodingWith(f.Name, fieldNameEncodings[:]) + metaString, err := fieldNameEncoder.EncodeWithEncoding(f.Name, encoding) + if err != nil { + return fmt.Errorf("row: encoding field name %q: %w", f.Name, err) + } + nameBytes := metaString.GetEncodedBytes() + nameSize := len(nameBytes) + encodingIndex := encodingIndexOf(metaString.GetEncoding()) + if encodingIndex < 0 { + return fmt.Errorf("row: field name %q got unsupported encoding %d", f.Name, metaString.GetEncoding()) + } + + header := encodingIndex & 0x03 + bigSize := nameSize > fieldNameSizeThreshold + if bigSize { + header |= fieldNameSizeThreshold << 2 + } else { + header |= (nameSize - 1) << 2 + } + if f.Nullable { + header |= 0x40 + } + buf.WriteByte_(byte(header)) + if bigSize { + buf.WriteVarUint32Small7(uint32(nameSize - fieldNameSizeThreshold)) + } + buf.WriteBinary(nameBytes) + return writeSchemaType(buf, f.Type) +} + +func writeSchemaType(buf *fory.ByteBuffer, dataType DataType) error { + buf.WriteByte_(byte(dataType.TypeID())) + switch t := dataType.(type) { + case DecimalType: + buf.WriteByte_(byte(t.Precision)) + buf.WriteByte_(byte(t.Scale)) + case *ListType: + return writeSchemaField(buf, t.Elem) + case *MapType: + if err := writeSchemaField(buf, t.Key); err != nil { + return err + } + return writeSchemaField(buf, t.Value) + case *StructType: + buf.WriteVarUint32Small7(uint32(len(t.Fields))) + for _, f := range t.Fields { + if err := writeSchemaField(buf, f); err != nil { + return err + } + } + } + return nil +} + +// SchemaFromBytes deserializes a schema from the cross-language wire +// format. Schema bytes are trusted input, like the row format itself; +// declared counts are still bounded by the remaining bytes so +// malformed schemas fail fast instead of over-allocating. +func SchemaFromBytes(data []byte) (*Schema, error) { + r := &schemaReader{buf: fory.NewByteBuffer(data), size: len(data)} + version := r.buf.ReadUint8(&r.err) + if err := r.err.CheckError(); err != nil { + return nil, err + } + if version != schemaVersion { + return nil, fmt.Errorf("row: unsupported schema version %d, expected %d", version, schemaVersion) + } + numFields, err := r.readCount("schema") + if err != nil { + return nil, err + } + fields := make([]Field, 0, numFields) + for i := 0; i < numFields; i++ { + f, err := r.readField() + if err != nil { + return nil, err + } + fields = append(fields, f) + } + return NewSchema(fields), nil +} + +type schemaReader struct { + buf *fory.ByteBuffer + size int + depth int + err fory.Error +} + +func (r *schemaReader) remaining() int { return r.size - r.buf.ReaderIndex() } + +// readCount reads a field count and bounds it by the remaining bytes +// (every field costs at least one byte) BEFORE narrowing to int, so a +// large wire value cannot wrap negative on 32-bit platforms or drive an +// oversized allocation. +func (r *schemaReader) readCount(owner string) (int, error) { + count := r.buf.ReadVarUint32Small7(&r.err) + if err := r.err.CheckError(); err != nil { + return 0, err + } + if uint64(count) > uint64(r.remaining()) { + return 0, fmt.Errorf("row: %s declares %d fields but only %d bytes remain", owner, count, r.remaining()) + } + return int(count), nil +} + +func (r *schemaReader) readField() (Field, error) { + if r.depth >= maxSchemaNestingDepth { + return Field{}, fmt.Errorf("row: schema nesting exceeds %d levels", maxSchemaNestingDepth) + } + r.depth++ + defer func() { r.depth-- }() + header := int(r.buf.ReadUint8(&r.err)) + if err := r.err.CheckError(); err != nil { + return Field{}, err + } + encodingIndex := header & 0x03 + if encodingIndex >= len(fieldNameEncodings) { + return Field{}, fmt.Errorf("row: invalid field name encoding index %d", encodingIndex) + } + nameSizeMinus1 := (header >> 2) & 0x0F + nullable := header&0x40 != 0 + + nameSize64 := uint64(nameSizeMinus1 + 1) + if nameSizeMinus1 == fieldNameSizeThreshold { + nameSize64 = uint64(r.buf.ReadVarUint32Small7(&r.err)) + fieldNameSizeThreshold + } + if err := r.err.CheckError(); err != nil { + return Field{}, err + } + if nameSize64 > uint64(r.remaining()) { + return Field{}, fmt.Errorf("row: field name of %d bytes exceeds %d remaining", nameSize64, r.remaining()) + } + nameSize := int(nameSize64) + nameBytes := r.buf.ReadBinary(nameSize, &r.err) + if err := r.err.CheckError(); err != nil { + return Field{}, err + } + name, err := fieldNameDecoder.Decode(nameBytes, fieldNameEncodings[encodingIndex]) + if err != nil { + return Field{}, fmt.Errorf("row: decoding field name: %w", err) + } + dataType, err := r.readType() + if err != nil { + return Field{}, err + } + return Field{Name: name, Type: dataType, Nullable: nullable}, nil +} + +func (r *schemaReader) readType() (DataType, error) { + typeID := fory.TypeId(r.buf.ReadUint8(&r.err)) + if err := r.err.CheckError(); err != nil { + return nil, err + } + switch typeID { + case fory.BOOL: + return BoolType{}, nil + case fory.INT8: + return Int8Type{}, nil + case fory.INT16: + return Int16Type{}, nil + case fory.INT32: + return Int32Type{}, nil + case fory.INT64: + return Int64Type{}, nil + case fory.FLOAT16: + return Float16Type{}, nil + case fory.FLOAT32: + return Float32Type{}, nil + case fory.FLOAT64: + return Float64Type{}, nil + case fory.STRING: + return StringType{}, nil + case fory.BINARY: + return BinaryType{}, nil + case fory.DURATION: + return DurationType{}, nil + case fory.TIMESTAMP: + return TimestampType{}, nil + case fory.DATE: + return Date32Type{}, nil + case fory.DECIMAL: + precision := int(r.buf.ReadUint8(&r.err)) + scale := int(r.buf.ReadUint8(&r.err)) + if err := r.err.CheckError(); err != nil { + return nil, err + } + return DecimalType{Precision: precision, Scale: scale}, nil + case fory.LIST: + elem, err := r.readField() + if err != nil { + return nil, err + } + return &ListType{Elem: elem}, nil + case fory.MAP: + keyField, err := r.readField() + if err != nil { + return nil, err + } + valueField, err := r.readField() + if err != nil { + return nil, err + } + // Java rebuilds maps from the child types only, restoring the + // canonical key/value names and nullability. + return Map(keyField.Type, valueField.Type), nil + case fory.STRUCT: + numFields, err := r.readCount("struct") + if err != nil { + return nil, err + } + fields := make([]Field, 0, numFields) + for i := 0; i < numFields; i++ { + f, err := r.readField() + if err != nil { + return nil, err + } + fields = append(fields, f) + } + return &StructType{Fields: fields}, nil + default: + return nil, fmt.Errorf("row: unknown type id %d in schema", typeID) + } +} + +// ComputeSchemaHash computes the cross-language schema hash, matching +// Java DataTypes.computeSchemaHash and the Python equivalent: fold +// hash*31 + typeId over fields, recursing into list elements, map keys +// and values, and struct fields; on signed-64 overflow, arithmetic- +// shift the hash right by 2 and retry. +func ComputeSchemaHash(s *Schema) int64 { + hash := int64(17) + for _, f := range s.Fields() { + hash = computeFieldHash(hash, f) + } + return hash +} + +func computeFieldHash(hash int64, f Field) int64 { + typeID := int64(f.Type.TypeID()) + for { + if hash <= math.MaxInt64/31 && hash >= math.MinInt64/31 { + product := hash * 31 + if product <= math.MaxInt64-typeID { + hash = product + typeID + break + } + } + hash >>= 2 + } + switch t := f.Type.(type) { + case *ListType: + hash = computeFieldHash(hash, t.Elem) + case *MapType: + hash = computeFieldHash(hash, t.Key) + hash = computeFieldHash(hash, t.Value) + case *StructType: + for _, child := range t.Fields { + hash = computeFieldHash(hash, child) + } + } + return hash +} diff --git a/go/fory/row/schema_bytes_test.go b/go/fory/row/schema_bytes_test.go new file mode 100644 index 0000000000..9521412be6 --- /dev/null +++ b/go/fory/row/schema_bytes_test.go @@ -0,0 +1,221 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package row + +import ( + "fmt" + "testing" + + fory "github.com/apache/fory/go/fory" + "github.com/stretchr/testify/require" +) + +// Golden bytes and hashes generated by Java +// org.apache.fory.format.type.SchemaEncoder (SCHEMA_VERSION=1) and +// DataTypes.computeSchemaHash. Regenerate with those classes if the +// Java wire format ever changes; the Java<->Go cross-language test is +// the live guard. +var schemaGoldens = []struct { + name string + schema func() *Schema + bytes []byte + hash int64 +}{ + { + name: "simple", + schema: func() *Schema { + return NewSchema([]Field{ + NewField("id", Int32Type{}, true), + NewField("name", StringType{}, true), + NewField("score", Float64Type{}, true), + NewField("active", BoolType{}, true), + }) + }, + bytes: []byte{ + 0x01, 0x04, 0x45, 0xa0, 0x60, 0x04, 0x49, 0x34, 0x0c, 0x20, 0x15, 0x4d, + 0xc8, 0x4e, 0x89, 0x00, 0x14, 0x4d, 0x00, 0x53, 0x45, 0x48, 0x01, + }, + hash: 15839823, + }, + { + name: "nested", + schema: func() *Schema { + return NewSchema([]Field{ + NewField("person", Struct([]Field{ + NewField("name", StringType{}, true), + NewField("age", Int32Type{}, false), + }), true), + NewField("tags", List(StringType{}), true), + NewField("attrs", Map(StringType{}, Int64Type{}), true), + }) + }, + bytes: []byte{ + 0x01, 0x03, 0x4d, 0x3c, 0x91, 0x93, 0x9a, 0x1b, 0x02, 0x49, 0x34, 0x0c, + 0x20, 0x15, 0x05, 0x00, 0xc4, 0x04, 0x49, 0x4c, 0x06, 0x90, 0x16, 0x49, + 0x22, 0x64, 0x60, 0x15, 0x4d, 0x82, 0x73, 0x8c, 0x80, 0x18, 0x05, 0x28, + 0x98, 0x15, 0x4d, 0xd4, 0x0b, 0xa1, 0x00, 0x06, + }, + hash: 15260761278193, + }, + { + name: "special", + schema: func() *Schema { + return NewSchema([]Field{ + NewField("a_very_long_field_name_exceeding_limit", Int64Type{}, false), + NewField("field2", StringType{}, true), + NewField("$special_name", Date32Type{}, true), + NewField("CamelCase", TimestampType{}, false), + NewField("price", DecimalType{Precision: 10, Scale: 2}, true), + }) + }, + bytes: []byte{ + 0x01, 0x05, 0x3d, 0x09, 0x03, 0x75, 0x24, 0x71, 0xb5, 0xb9, 0xa6, 0xd9, + 0x50, 0x45, 0x8f, 0x6d, 0x03, 0x09, 0xb2, 0x5c, 0x44, 0x20, 0xd0, 0xd3, + 0x6d, 0x68, 0x62, 0x26, 0x06, 0x52, 0x0a, 0x40, 0x85, 0x87, 0xb0, 0x15, + 0x61, 0xf2, 0x4f, 0x20, 0x90, 0x05, 0xed, 0xa0, 0x61, 0x00, 0x27, 0x1a, + 0x38, 0x01, 0x82, 0x16, 0xe0, 0x09, 0x08, 0x26, 0x4d, 0xbe, 0x28, 0x11, + 0x00, 0x28, 0x0a, 0x02, + }, + hash: 492901001, + }, +} + +func TestSchemaBytesGolden(t *testing.T) { + for _, g := range schemaGoldens { + schema := g.schema() + + got, err := SchemaToBytes(schema) + require.NoError(t, err, g.name) + require.Equal(t, g.bytes, got, "%s: bytes must match Java SchemaEncoder output", g.name) + + parsed, err := SchemaFromBytes(g.bytes) + require.NoError(t, err, g.name) + require.True(t, parsed.Equal(schema), "%s: parsed schema %v != %v", g.name, parsed, schema) + } +} + +func TestSchemaHashGolden(t *testing.T) { + for _, g := range schemaGoldens { + require.Equal(t, g.hash, ComputeSchemaHash(g.schema()), g.name) + } + + // 20 int64 fields overflow the int64 hash, exercising Java's + // shift-right-2-and-retry path. + fields := make([]Field, 20) + for i := range fields { + fields[i] = NewField(fmt.Sprintf("f%d", i), Int64Type{}, true) + } + require.Equal(t, int64(2627256684647412568), ComputeSchemaHash(NewSchema(fields))) +} + +func requireErrorContains(t *testing.T, err error, substr string) { + t.Helper() + require.Error(t, err) + require.Contains(t, err.Error(), substr) +} + +func TestSchemaFromBytesErrors(t *testing.T) { + // Unsupported version. + _, err := SchemaFromBytes([]byte{0x02, 0x00}) + requireErrorContains(t, err, "schema version") + + // Invalid field name encoding index (bits 0-1 == 3). + _, err = SchemaFromBytes([]byte{0x01, 0x01, 0x07}) + requireErrorContains(t, err, "encoding index") + + // Unknown type id. + corrupt := append([]byte(nil), schemaGoldens[0].bytes...) + corrupt[len(corrupt)-1] = 0x63 + _, err = SchemaFromBytes(corrupt) + requireErrorContains(t, err, "unknown type id") + + // Field count larger than the remaining input. + _, err = SchemaFromBytes([]byte{0x01, 0x7f}) + requireErrorContains(t, err, "fields") + + // Every strict prefix of a valid encoding must fail, never panic. + golden := schemaGoldens[1].bytes + for i := 0; i < len(golden); i++ { + _, err := SchemaFromBytes(golden[:i]) + require.Error(t, err, "prefix of length %d", i) + } +} + +func TestSchemaToBytesRejectsEmptyName(t *testing.T) { + _, err := SchemaToBytes(NewSchema([]Field{NewField("", Int32Type{}, true)})) + requireErrorContains(t, err, "empty name") +} + +// A crafted chain of nested LIST type-infos costs ~3 bytes per level; +// without a depth limit it would fatally overflow the goroutine stack. +func TestSchemaFromBytesRejectsDeepNesting(t *testing.T) { + data := []byte{0x01, 0x01} + for i := 0; i < 10*maxSchemaNestingDepth; i++ { + // field: header (UTF_8 encoding, 1-byte name, non-null), + // name "a", type LIST -> recurses into the next field. + data = append(data, 0x00, 'a', 0x16) + } + _, err := SchemaFromBytes(data) + requireErrorContains(t, err, "nesting") +} + +type unknownDataType struct{} + +func (unknownDataType) TypeID() fory.TypeId { return 99 } +func (unknownDataType) ByteWidth() int { return -1 } +func (unknownDataType) String() string { return "unknown" } + +// The writer validates everything the wire format cannot express +// before emitting a byte, so its output is always readable. +func TestSchemaToBytesRejectsUnwritableSchemas(t *testing.T) { + _, err := SchemaToBytes(nil) + require.Error(t, err) + + // One level deeper than the reader accepts. + var deep DataType = Int32Type{} + for i := 0; i < maxSchemaNestingDepth; i++ { + deep = List(deep) + } + _, err = SchemaToBytes(NewSchema([]Field{NewField("deep", deep, true)})) + requireErrorContains(t, err, "nesting") + // Exactly the reader's limit still round-trips. + var ok DataType = Int32Type{} + for i := 0; i < maxSchemaNestingDepth-1; i++ { + ok = List(ok) + } + okBytes, err := SchemaToBytes(NewSchema([]Field{NewField("ok", ok, true)})) + require.NoError(t, err) + _, err = SchemaFromBytes(okBytes) + require.NoError(t, err) + + // A type that refers to itself. + loop := &ListType{} + loop.Elem = NewField("item", loop, true) + _, err = SchemaToBytes(NewSchema([]Field{NewField("loop", loop, true)})) + requireErrorContains(t, err, "refers to itself") + + // Decimal parameters wider than the single wire byte. + _, err = SchemaToBytes(NewSchema([]Field{NewField("d", DecimalType{Precision: 300, Scale: 2}, true)})) + requireErrorContains(t, err, "decimal") + + // Pointer decimal and foreign DataType implementations. + _, err = SchemaToBytes(NewSchema([]Field{NewField("d", &DecimalType{Precision: 10, Scale: 2}, true)})) + require.Error(t, err) + _, err = SchemaToBytes(NewSchema([]Field{NewField("u", unknownDataType{}, true)})) + require.Error(t, err) +} diff --git a/go/fory/row/schema_test.go b/go/fory/row/schema_test.go new file mode 100644 index 0000000000..32042e7e8a --- /dev/null +++ b/go/fory/row/schema_test.go @@ -0,0 +1,155 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package row + +import ( + "testing" + + fory "github.com/apache/fory/go/fory" + "github.com/stretchr/testify/require" +) + +// Type ids are serialized into the cross-language schema bytes, so they +// are pinned to their literal values here. +func TestTypeIDs(t *testing.T) { + cases := []struct { + dataType DataType + id fory.TypeId + }{ + {BoolType{}, 1}, + {Int8Type{}, 2}, + {Int16Type{}, 3}, + {Int32Type{}, 4}, + {Int64Type{}, 6}, + {Float16Type{}, 17}, + {Float32Type{}, 19}, + {Float64Type{}, 20}, + {StringType{}, 21}, + {List(Int32Type{}), 22}, + {Map(StringType{}, Int64Type{}), 24}, + {Struct(nil), 27}, + {DurationType{}, 37}, + {TimestampType{}, 38}, + {Date32Type{}, 39}, + {DecimalType{Precision: 10, Scale: 2}, 40}, + {BinaryType{}, 41}, + } + for _, c := range cases { + require.Equal(t, c.id, c.dataType.TypeID(), "%s", c.dataType) + } +} + +func TestByteWidths(t *testing.T) { + cases := []struct { + dataType DataType + width int + }{ + {BoolType{}, 1}, + {Int8Type{}, 1}, + {Int16Type{}, 2}, + {Int32Type{}, 4}, + {Int64Type{}, 8}, + {Float16Type{}, 2}, + {Float32Type{}, 4}, + {Float64Type{}, 8}, + {Date32Type{}, 4}, + {TimestampType{}, 8}, + {DurationType{}, 8}, + {StringType{}, -1}, + {BinaryType{}, -1}, + {DecimalType{}, -1}, + {List(Int32Type{}), -1}, + {Map(StringType{}, Int64Type{}), -1}, + {Struct(nil), -1}, + } + for _, c := range cases { + require.Equal(t, c.width, c.dataType.ByteWidth(), "%s", c.dataType) + } +} + +func TestCompositeFactories(t *testing.T) { + list := List(StringType{}) + require.Equal(t, "item", list.Elem.Name) + require.True(t, list.Elem.Nullable) + + m := Map(StringType{}, Int32Type{}) + require.Equal(t, "key", m.Key.Name) + require.False(t, m.Key.Nullable, "map keys must be non-nullable") + require.Equal(t, "value", m.Value.Name) + require.True(t, m.Value.Nullable) + + st := Struct([]Field{NewField("a", Int32Type{}, false)}) + require.Len(t, st.Fields, 1) + require.Equal(t, "a", st.Fields[0].Name) +} + +func TestSchemaLookup(t *testing.T) { + s := NewSchema([]Field{ + NewField("id", Int64Type{}, false), + NewField("name", StringType{}, true), + }) + require.Equal(t, 2, s.NumFields()) + require.Equal(t, "id", s.Field(0).Name) + require.Equal(t, 1, s.FieldIndex("name")) + require.Equal(t, -1, s.FieldIndex("missing")) + + // Duplicate names resolve to the last occurrence, as in Java. + dup := NewSchema([]Field{ + NewField("x", Int32Type{}, false), + NewField("x", StringType{}, true), + }) + require.Equal(t, 1, dup.FieldIndex("x")) +} + +// Schemas and struct types own a copy of their fields, so later +// mutation of the caller's slice cannot desynchronize name lookups. +func TestSchemaCopiesFields(t *testing.T) { + fields := []Field{NewField("a", Int32Type{}, false)} + s := NewSchema(fields) + st := Struct(fields) + fields[0].Name = "changed" + require.Equal(t, "a", s.Field(0).Name) + require.Equal(t, 0, s.FieldIndex("a")) + require.Equal(t, "a", st.Fields[0].Name) +} + +func TestSchemaEqual(t *testing.T) { + nested := func() *Schema { + return NewSchema([]Field{ + NewField("id", Int64Type{}, false), + NewField("tags", List(StringType{}), true), + NewField("attrs", Map(StringType{}, Int32Type{}), true), + NewField("inner", Struct([]Field{NewField("x", Float64Type{}, false)}), true), + NewField("price", DecimalType{Precision: 10, Scale: 2}, true), + }) + } + require.True(t, nested().Equal(nested())) + + base := NewSchema([]Field{NewField("a", Int32Type{}, false)}) + require.False(t, base.Equal(nil)) + require.False(t, base.Equal(NewSchema([]Field{NewField("b", Int32Type{}, false)}))) + require.False(t, base.Equal(NewSchema([]Field{NewField("a", Int64Type{}, false)}))) + require.False(t, base.Equal(NewSchema([]Field{NewField("a", Int32Type{}, true)}))) + require.False(t, base.Equal(NewSchema(nil))) + + // Composite mismatches must compare structurally, not by pointer. + require.False(t, NewSchema([]Field{NewField("l", List(Int32Type{}), true)}). + Equal(NewSchema([]Field{NewField("l", List(Int64Type{}), true)}))) + require.False(t, NewSchema([]Field{NewField("d", DecimalType{Precision: 10, Scale: 2}, true)}). + Equal(NewSchema([]Field{NewField("d", DecimalType{Precision: 12, Scale: 2}, true)}))) +} diff --git a/go/fory/row/writer.go b/go/fory/row/writer.go new file mode 100644 index 0000000000..ecc5080a99 --- /dev/null +++ b/go/fory/row/writer.go @@ -0,0 +1,421 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package row + +import ( + "encoding/binary" + "fmt" + "math" + "time" + "unicode/utf8" + + fory "github.com/apache/fory/go/fory" +) + +const maxArrayDataBytes = math.MaxInt32 - 15 + +// RowWriter writes one row into a ByteBuffer: null bitmap, fixed 8-byte +// field slots, then the variable data region. All slot and bitmap +// offsets are relative to the row base, so a nested RowWriter can share +// the parent's buffer and produce a self-contained row. +// +// Reset rebases the writer at the buffer's current writer index and must +// be called before writing each row. Writers are not goroutine-safe. +type RowWriter struct { + schema *Schema + buf *fory.ByteBuffer + base int + numFields int + bitmapWidth int + fixedSize int +} + +func NewRowWriter(schema *Schema) *RowWriter { + return NewRowWriterWithBuffer(schema, fory.NewByteBuffer(nil)) +} + +// NewRowWriterWithBuffer shares an existing buffer, e.g. to write a +// nested struct row into its parent's variable data region. +func NewRowWriterWithBuffer(schema *Schema, buf *fory.ByteBuffer) *RowWriter { + n := schema.NumFields() + bitmapWidth := bitmapWidthInBytes(n) + return &RowWriter{ + schema: schema, + buf: buf, + numFields: n, + bitmapWidth: bitmapWidth, + fixedSize: bitmapWidth + n*8, + } +} + +func (w *RowWriter) Schema() *Schema { return w.schema } +func (w *RowWriter) Buffer() *fory.ByteBuffer { return w.buf } + +// Reset rebases the writer at the current writer index and zeroes the +// whole fixed region. The buffer reuses dirty capacity after shrinking, +// so zeroing here is what makes null slots and padding deterministic; +// it also lets narrow fixed-width writes store just the value. +func (w *RowWriter) Reset() { + w.base = w.buf.WriterIndex() + w.buf.Reserve(w.fixedSize) + data := w.buf.GetData() + clear(data[w.base : w.base+w.fixedSize]) + w.buf.SetWriterIndex(w.base + w.fixedSize) +} + +// Size returns the bytes written for this row so far. +func (w *RowWriter) Size() int { return w.buf.WriterIndex() - w.base } + +// ToBytes returns a view of the row bytes; it stays valid only until the +// buffer is written to or reset again. +func (w *RowWriter) ToBytes() []byte { + return w.buf.GetByteSlice(w.base, w.buf.WriterIndex()) +} + +func (w *RowWriter) slot(i int) int { + if uint(i) >= uint(w.numFields) { + panic(fmt.Sprintf("row: field index %d out of range [0, %d)", i, w.numFields)) + } + return w.base + w.bitmapWidth + i*8 +} + +// SetNullAt marks field i null and zeroes its slot. +func (w *RowWriter) SetNullAt(i int) { + slot := w.slot(i) + data := w.buf.GetData() + setBit(data[w.base:w.base+w.bitmapWidth], i) + binary.LittleEndian.PutUint64(data[slot:], 0) +} + +func (w *RowWriter) SetNotNullAt(i int) { + w.slot(i) // bounds check + clearBit(w.buf.GetData()[w.base:w.base+w.bitmapWidth], i) +} + +func (w *RowWriter) WriteBool(i int, v bool) { + slot := w.slot(i) + if v { + w.buf.GetData()[slot] = 1 + } else { + w.buf.GetData()[slot] = 0 + } +} + +func (w *RowWriter) WriteInt8(i int, v int8) { + w.buf.GetData()[w.slot(i)] = byte(v) +} + +func (w *RowWriter) WriteInt16(i int, v int16) { + binary.LittleEndian.PutUint16(w.buf.GetData()[w.slot(i):], uint16(v)) +} + +func (w *RowWriter) WriteInt32(i int, v int32) { + binary.LittleEndian.PutUint32(w.buf.GetData()[w.slot(i):], uint32(v)) +} + +func (w *RowWriter) WriteInt64(i int, v int64) { + binary.LittleEndian.PutUint64(w.buf.GetData()[w.slot(i):], uint64(v)) +} + +func (w *RowWriter) WriteFloat32(i int, v float32) { + binary.LittleEndian.PutUint32(w.buf.GetData()[w.slot(i):], math.Float32bits(v)) +} + +func (w *RowWriter) WriteFloat64(i int, v float64) { + binary.LittleEndian.PutUint64(w.buf.GetData()[w.slot(i):], math.Float64bits(v)) +} + +func (w *RowWriter) WriteDate(i int, d fory.Date) error { + days, err := fory.DateToEpochDay(d) + if err != nil { + return err + } + if days < math.MinInt32 || days > math.MaxInt32 { + return fmt.Errorf("row: date %v out of date32 range", d) + } + w.WriteInt32(i, int32(days)) + return nil +} + +func (w *RowWriter) WriteTimestamp(i int, t time.Time) error { + micros, err := timestampMicros(t) + if err != nil { + return err + } + w.WriteInt64(i, micros) + return nil +} + +func (w *RowWriter) WriteDuration(i int, d time.Duration) { + w.WriteInt64(i, d.Microseconds()) +} + +// WriteString writes a UTF-8 string; the row format string type is +// UTF-8, so invalid byte sequences are rejected rather than changed by +// other runtimes on read. +func (w *RowWriter) WriteString(i int, s string) error { + if !utf8.ValidString(s) { + return errInvalidUTF8 + } + start := appendStringRegion(w.buf, s) + return w.SetOffsetAndSize(i, start, len(s)) +} + +func (w *RowWriter) WriteBytes(i int, b []byte) error { + start := appendBytesRegion(w.buf, b) + return w.SetOffsetAndSize(i, start, len(b)) +} + +// SetOffsetAndSize patches field i's slot with the row-relative offset +// and byte size of a value already appended to the variable data region. +// Use it after writing a nested struct, array, or map at absStart. +// Offsets and sizes beyond 32 bits are rejected: the wire format packs +// both into one slot, and truncating would corrupt the row silently. +func (w *RowWriter) SetOffsetAndSize(i, absStart, size int) error { + rel := absStart - w.base + if uint64(rel) > math.MaxUint32 || uint64(size) > math.MaxUint32 { + return fmt.Errorf("row: value at offset %d with %d bytes exceeds the 32-bit wire format limit", rel, size) + } + binary.LittleEndian.PutUint64(w.buf.GetData()[w.slot(i):], uint64(rel)<<32|uint64(size)) + return nil +} + +// ArrayWriter writes one array: an 8-byte element count, a null bitmap, +// then elements at their natural width (variable-width elements use +// 8-byte offset+size slots). Offsets are relative to the array base. +type ArrayWriter struct { + elem Field + buf *fory.ByteBuffer + base int + elemSize int + numElements int + headerBytes int +} + +func NewArrayWriter(elem Field, buf *fory.ByteBuffer) *ArrayWriter { + elemSize := elem.Type.ByteWidth() + if elemSize < 0 { + elemSize = 8 + } + return &ArrayWriter{elem: elem, buf: buf, elemSize: elemSize} +} + +func (w *ArrayWriter) Buffer() *fory.ByteBuffer { return w.buf } + +// Reset rebases the writer at the current writer index and writes the +// array header plus a zeroed element region for numElements elements. +func (w *ArrayWriter) Reset(numElements int) error { + if numElements < 0 { + return fmt.Errorf("row: negative array length %d", numElements) + } + // Compare before multiplying so extreme counts cannot overflow the + // size computation and slip past the limit. + if numElements > maxArrayDataBytes/w.elemSize { + return fmt.Errorf("row: array of %d elements exceeds maximum size", numElements) + } + dataBytes := int64(numElements) * int64(w.elemSize) + headerBytes := 8 + bitmapWidthInBytes(numElements) + total := headerBytes + roundToWord(int(dataBytes)) + base := w.buf.WriterIndex() + w.buf.Reserve(total) + data := w.buf.GetData() + binary.LittleEndian.PutUint64(data[base:], uint64(numElements)) + clear(data[base+8 : base+total]) + w.buf.SetWriterIndex(base + total) + w.base, w.numElements, w.headerBytes = base, numElements, headerBytes + return nil +} + +func (w *ArrayWriter) Size() int { return w.buf.WriterIndex() - w.base } + +func (w *ArrayWriter) ToBytes() []byte { + return w.buf.GetByteSlice(w.base, w.buf.WriterIndex()) +} + +func (w *ArrayWriter) slot(i int) int { + if uint(i) >= uint(w.numElements) { + panic(fmt.Sprintf("row: array index %d out of range [0, %d)", i, w.numElements)) + } + return w.base + w.headerBytes + i*w.elemSize +} + +// SetNullAt marks element i null and re-zeroes its element bytes. +func (w *ArrayWriter) SetNullAt(i int) { + slot := w.slot(i) + data := w.buf.GetData() + setBit(data[w.base+8:w.base+w.headerBytes], i) + clear(data[slot : slot+w.elemSize]) +} + +func (w *ArrayWriter) WriteBool(i int, v bool) { + slot := w.slot(i) + if v { + w.buf.GetData()[slot] = 1 + } else { + w.buf.GetData()[slot] = 0 + } +} + +func (w *ArrayWriter) WriteInt8(i int, v int8) { + w.buf.GetData()[w.slot(i)] = byte(v) +} + +func (w *ArrayWriter) WriteInt16(i int, v int16) { + binary.LittleEndian.PutUint16(w.buf.GetData()[w.slot(i):], uint16(v)) +} + +func (w *ArrayWriter) WriteInt32(i int, v int32) { + binary.LittleEndian.PutUint32(w.buf.GetData()[w.slot(i):], uint32(v)) +} + +func (w *ArrayWriter) WriteInt64(i int, v int64) { + binary.LittleEndian.PutUint64(w.buf.GetData()[w.slot(i):], uint64(v)) +} + +func (w *ArrayWriter) WriteFloat32(i int, v float32) { + binary.LittleEndian.PutUint32(w.buf.GetData()[w.slot(i):], math.Float32bits(v)) +} + +func (w *ArrayWriter) WriteFloat64(i int, v float64) { + binary.LittleEndian.PutUint64(w.buf.GetData()[w.slot(i):], math.Float64bits(v)) +} + +func (w *ArrayWriter) WriteDate(i int, d fory.Date) error { + days, err := fory.DateToEpochDay(d) + if err != nil { + return err + } + if days < math.MinInt32 || days > math.MaxInt32 { + return fmt.Errorf("row: date %v out of date32 range", d) + } + w.WriteInt32(i, int32(days)) + return nil +} + +func (w *ArrayWriter) WriteTimestamp(i int, t time.Time) error { + micros, err := timestampMicros(t) + if err != nil { + return err + } + w.WriteInt64(i, micros) + return nil +} + +func (w *ArrayWriter) WriteDuration(i int, d time.Duration) { + w.WriteInt64(i, d.Microseconds()) +} + +func (w *ArrayWriter) WriteString(i int, s string) error { + if !utf8.ValidString(s) { + return errInvalidUTF8 + } + start := appendStringRegion(w.buf, s) + return w.SetOffsetAndSize(i, start, len(s)) +} + +func (w *ArrayWriter) WriteBytes(i int, b []byte) error { + start := appendBytesRegion(w.buf, b) + return w.SetOffsetAndSize(i, start, len(b)) +} + +// SetOffsetAndSize patches element i's slot with the array-relative +// offset and byte size of a value already appended after the array, +// with the same 32-bit wire limit as RowWriter.SetOffsetAndSize. +func (w *ArrayWriter) SetOffsetAndSize(i, absStart, size int) error { + rel := absStart - w.base + if uint64(rel) > math.MaxUint32 || uint64(size) > math.MaxUint32 { + return fmt.Errorf("row: value at offset %d with %d bytes exceeds the 32-bit wire format limit", rel, size) + } + binary.LittleEndian.PutUint64(w.buf.GetData()[w.slot(i):], uint64(rel)<<32|uint64(size)) + return nil +} + +// MapWriter writes one map: an 8-byte keys-array size, the keys array, +// then the values array. Write flow: Reset, write the keys with an +// ArrayWriter, FinishKeys, write the values with another ArrayWriter. +type MapWriter struct { + buf *fory.ByteBuffer + base int +} + +func NewMapWriter(buf *fory.ByteBuffer) *MapWriter { + return &MapWriter{buf: buf} +} + +// Reset rebases the writer at the current writer index and reserves the +// keys-array size word, which FinishKeys patches later. +func (w *MapWriter) Reset() { + w.base = w.buf.WriterIndex() + w.buf.Reserve(8) + data := w.buf.GetData() + clear(data[w.base : w.base+8]) + w.buf.SetWriterIndex(w.base + 8) +} + +// FinishKeys records the keys array size; call it after the keys array +// is fully written and before starting the values array. +func (w *MapWriter) FinishKeys() { + keysSize := w.buf.WriterIndex() - w.base - 8 + binary.LittleEndian.PutUint64(w.buf.GetData()[w.base:], uint64(keysSize)) +} + +func (w *MapWriter) Size() int { return w.buf.WriterIndex() - w.base } + +// appendBytesRegion appends b to the variable data region, zero-padded +// to an 8-byte boundary, and returns its buffer offset. +func appendBytesRegion(buf *fory.ByteBuffer, b []byte) int { + n := len(b) + rounded := roundToWord(n) + start := buf.WriterIndex() + buf.Reserve(rounded) + data := buf.GetData() + clear(data[start+n : start+rounded]) + copy(data[start:], b) + buf.SetWriterIndex(start + rounded) + return start +} + +var errInvalidUTF8 = fmt.Errorf("row: string is not valid UTF-8") + +// timestampMicros converts t to microseconds since the Unix epoch with +// overflow checking; time.Time.UnixMicro is undefined outside the +// int64 microsecond range. +func timestampMicros(t time.Time) (int64, error) { + sec := t.Unix() + if sec > math.MaxInt64/1_000_000 || sec < math.MinInt64/1_000_000 { + return 0, fmt.Errorf("row: timestamp %v is outside the int64 microsecond range", t) + } + micros := sec * 1_000_000 + subMicros := int64(t.Nanosecond()) / 1000 + if micros > math.MaxInt64-subMicros { + return 0, fmt.Errorf("row: timestamp %v is outside the int64 microsecond range", t) + } + return micros + subMicros, nil +} + +func appendStringRegion(buf *fory.ByteBuffer, s string) int { + n := len(s) + rounded := roundToWord(n) + start := buf.WriterIndex() + buf.Reserve(rounded) + data := buf.GetData() + clear(data[start+n : start+rounded]) + copy(data[start:], s) + buf.SetWriterIndex(start + rounded) + return start +} diff --git a/go/fory/tests/row_xlang/row_xlang_main.go b/go/fory/tests/row_xlang/row_xlang_main.go new file mode 100644 index 0000000000..776eda2410 --- /dev/null +++ b/go/fory/tests/row_xlang/row_xlang_main.go @@ -0,0 +1,183 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Row format cross-language peer driven by Java's +// org.apache.fory.format.GoCrossLanguageTest: each case reads bytes +// written by Java, verifies them field by field, re-encodes the same +// value, and overwrites the file for Java to check. +package main + +import ( + "bytes" + "fmt" + "os" + "reflect" + + "github.com/apache/fory/go/fory/row" +) + +// Mirrors org.apache.fory.format.CrossLanguageTest.A: pointer and +// value types are chosen so the inferred schema matches Java's +// (boxed Java types are nullable, so they map to Go pointers; string +// fields are nullable in both languages). +type a struct { + F1 *int32 + F2 map[string]string +} + +// Mirrors CrossLanguageTest.Bar. +type bar struct { + F1 *int32 + F2 string +} + +// Mirrors GoCrossLanguageTest.Blob: Java byte[] and Go []byte are both +// list. +type blob struct { + F1 []byte + F2 string +} + +// Mirrors CrossLanguageTest.Foo. F3 uses []*string because the Java +// fixture contains a null list element. +type foo struct { + F1 *int32 + F2 string + F3 []*string + F4 map[string]*int32 + F5 *bar +} + +func main() { + if len(os.Args) < 3 { + fail("usage: row_xlang_bin ...") + } + switch caseName := os.Args[1]; caseName { + case "test_map_encoder": + testMapEncoder(os.Args[2]) + case "test_serialization_without_schema": + testSerializationWithoutSchema(os.Args[2]) + case "test_serialization_with_schema": + if len(os.Args) < 4 { + fail("test_serialization_with_schema needs ") + } + testSerializationWithSchema(os.Args[2], os.Args[3]) + case "test_byte_array_carrier": + testByteArrayCarrier(os.Args[2]) + default: + fail("unknown test case %q", caseName) + } +} + +func testMapEncoder(dataFile string) { + encoder, err := row.NewEncoder[a]() + must(err) + data, err := os.ReadFile(dataFile) + must(err) + + decoded, err := encoder.Decode(data) + must(err) + expected := a{F1: int32Ptr(1), F2: map[string]string{"pid": "12345", "ip": "0.0.0.0", "k1": "v1"}} + check(reflect.DeepEqual(decoded, expected), "decoded %+v, expected %+v", decoded, expected) + + encoded, err := encoder.Encode(&expected) + must(err) + must(os.WriteFile(dataFile, encoded, 0o644)) +} + +func testByteArrayCarrier(dataFile string) { + encoder, err := row.NewEncoder[blob]() + must(err) + data, err := os.ReadFile(dataFile) + must(err) + + decoded, err := encoder.Decode(data) + must(err) + expected := blob{F1: []byte{0, 1, 0xff, 127, 0x80}, F2: "bytes"} + check(reflect.DeepEqual(decoded, expected), "decoded %+v, expected %+v", decoded, expected) + + encoded, err := encoder.Encode(&expected) + must(err) + must(os.WriteFile(dataFile, encoded, 0o644)) +} + +func testSerializationWithoutSchema(dataFile string) { + encoder, err := row.NewEncoder[foo]() + must(err) + data, err := os.ReadFile(dataFile) + must(err) + + decoded, err := encoder.FromRow(data) + must(err) + expected := expectedFoo() + check(reflect.DeepEqual(decoded, expected), "decoded %+v, expected %+v", decoded, expected) + + rowBytes, err := encoder.ToRow(&expected) + must(err) + must(os.WriteFile(dataFile, rowBytes, 0o644)) +} + +func testSerializationWithSchema(schemaFile, dataFile string) { + encoder, err := row.NewEncoder[foo]() + must(err) + schemaBytes, err := os.ReadFile(schemaFile) + must(err) + + parsed, err := row.SchemaFromBytes(schemaBytes) + must(err) + check(parsed.Equal(encoder.Schema()), "Java schema %v, inferred schema %v", parsed, encoder.Schema()) + check(row.ComputeSchemaHash(parsed) == encoder.SchemaHash(), + "schema hash %d, encoder hash %d", row.ComputeSchemaHash(parsed), encoder.SchemaHash()) + reencoded, err := row.SchemaToBytes(encoder.Schema()) + must(err) + check(bytes.Equal(reencoded, schemaBytes), "re-encoded schema bytes differ from Java's") + + testSerializationWithoutSchema(dataFile) +} + +func expectedFoo() foo { + return foo{ + F1: int32Ptr(1), + F2: "str", + F3: []*string{stringPtr("str1"), nil, stringPtr("str2")}, + F4: map[string]*int32{ + "k1": int32Ptr(1), "k2": int32Ptr(2), "k3": int32Ptr(3), + "k4": int32Ptr(4), "k5": int32Ptr(5), "k6": int32Ptr(6), + }, + F5: &bar{F1: int32Ptr(1), F2: "str"}, + } +} + +func int32Ptr(v int32) *int32 { return &v } +func stringPtr(s string) *string { return &s } + +func check(ok bool, format string, args ...any) { + if !ok { + fail(format, args...) + } +} + +func must(err error) { + if err != nil { + fail("%v", err) + } +} + +func fail(format string, args ...any) { + fmt.Fprintf(os.Stderr, "row_xlang: "+format+"\n", args...) + os.Exit(1) +} diff --git a/java/fory-format/src/test/java/org/apache/fory/format/GoCrossLanguageTest.java b/java/fory-format/src/test/java/org/apache/fory/format/GoCrossLanguageTest.java new file mode 100644 index 0000000000..1c74f7225e --- /dev/null +++ b/java/fory-format/src/test/java/org/apache/fory/format/GoCrossLanguageTest.java @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.format; + +import com.google.common.collect.ImmutableMap; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import lombok.Data; +import org.apache.fory.format.encoder.Encoders; +import org.apache.fory.format.encoder.RowEncoder; +import org.apache.fory.format.row.binary.BinaryRow; +import org.apache.fory.format.type.DataTypes; +import org.apache.fory.memory.MemoryBuffer; +import org.apache.fory.memory.MemoryUtils; +import org.apache.fory.test.TestUtils; +import org.testng.Assert; +import org.testng.SkipException; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +/** + * Row format cross-language tests against a Go peer built from {@code go/fory/tests/row_xlang}. + * Data shapes are shared with {@link CrossLanguageTest} so the Java, Python, and Go peers exercise + * the same schemas. Setting {@code FORY_GO_JAVA_CI=1} opts into the suite; once opted in, a missing + * Go toolchain or a peer build failure fails the tests instead of skipping them. + */ +@Test +public class GoCrossLanguageTest { + private static final boolean IS_WINDOWS = + System.getProperty("os.name").toLowerCase().contains("windows"); + private static final String GO_BINARY = IS_WINDOWS ? "row_xlang_bin.exe" : "row_xlang_bin"; + + @BeforeClass + public void ensureGoPeerReady() { + String enabled = System.getenv("FORY_GO_JAVA_CI"); + if (!"1".equals(enabled)) { + throw new SkipException("Skipping GoCrossLanguageTest: FORY_GO_JAVA_CI not set to 1"); + } + try { + Process process = new ProcessBuilder("go", "version").start(); + Assert.assertEquals(process.waitFor(), 0, "go toolchain is required when FORY_GO_JAVA_CI=1"); + } catch (IOException e) { + throw new AssertionError("go toolchain is required when FORY_GO_JAVA_CI=1", e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError("interrupted while probing the go toolchain", e); + } + List buildCommand = + Arrays.asList("go", "build", "-o", "tests/" + GO_BINARY, "./tests/row_xlang"); + Assert.assertTrue( + TestUtils.executeCommand( + buildCommand, 120, Collections.emptyMap(), new File("../../go/fory")), + "failed to build the Go row format peer " + GO_BINARY); + Assert.assertTrue( + new File("../../go/fory/tests/" + GO_BINARY).exists(), + GO_BINARY + " not found after a successful build"); + } + + /** Keep in sync with {@code blob} in row_xlang_main.go: byte[] is list in both. */ + @Data + public static class Blob { + public byte[] f1; + public String f2; + + public static Blob create() { + Blob blob = new Blob(); + blob.f1 = new byte[] {0, 1, -1, 127, -128}; + blob.f2 = "bytes"; + return blob; + } + } + + public void testByteArrayCarrier() throws IOException { + Blob blob = Blob.create(); + RowEncoder encoder = Encoders.bean(Blob.class); + Path dataFile = createTempFile("row_go_blob"); + Files.write(dataFile, encoder.encode(blob)); + Assert.assertTrue(runGoPeer("test_byte_array_carrier", dataFile)); + Assert.assertEquals(encoder.decode(Files.readAllBytes(dataFile)), blob); + } + + public void testMapEncoder() throws IOException { + CrossLanguageTest.A a = CrossLanguageTest.A.create(); + RowEncoder encoder = Encoders.bean(CrossLanguageTest.A.class); + Path dataFile = createTempFile("row_go_map"); + Files.write(dataFile, encoder.encode(a)); + Assert.assertTrue(runGoPeer("test_map_encoder", dataFile)); + Assert.assertEquals(encoder.decode(Files.readAllBytes(dataFile)), a); + } + + public void testSerializationWithoutSchema() throws IOException { + CrossLanguageTest.Foo foo = CrossLanguageTest.Foo.create(); + RowEncoder encoder = Encoders.bean(CrossLanguageTest.Foo.class); + Path dataFile = createTempFile("row_go_foo"); + Files.write(dataFile, encoder.toRow(foo).toBytes()); + Assert.assertTrue(runGoPeer("test_serialization_without_schema", dataFile)); + Assert.assertEquals(readFoo(encoder, dataFile), foo); + } + + public void testSerializationWithSchema() throws IOException { + CrossLanguageTest.Foo foo = CrossLanguageTest.Foo.create(); + RowEncoder encoder = Encoders.bean(CrossLanguageTest.Foo.class); + Path dataFile = createTempFile("row_go_foo"); + Path schemaFile = createTempFile("row_go_foo_schema"); + BinaryRow row = encoder.toRow(foo); + Files.write(dataFile, row.toBytes()); + Files.write(schemaFile, DataTypes.serializeSchema(row.getSchema())); + Assert.assertTrue(runGoPeer("test_serialization_with_schema", schemaFile, dataFile)); + Assert.assertEquals(readFoo(encoder, dataFile), foo); + } + + private static CrossLanguageTest.Foo readFoo( + RowEncoder encoder, Path dataFile) throws IOException { + MemoryBuffer buffer = MemoryUtils.wrap(Files.readAllBytes(dataFile)); + BinaryRow row = new BinaryRow(encoder.schema()); + row.pointTo(buffer, 0, buffer.size()); + return encoder.fromRow(row); + } + + private static Path createTempFile(String prefix) throws IOException { + Path file = Files.createTempFile(prefix, "data"); + file.toFile().deleteOnExit(); + return file; + } + + private boolean runGoPeer(String caseName, Path... files) { + List command = new ArrayList<>(); + command.add(IS_WINDOWS ? GO_BINARY : "./" + GO_BINARY); + command.add(caseName); + for (Path file : files) { + command.add(file.toAbsolutePath().toString()); + } + return TestUtils.executeCommand( + command, + 30, + ImmutableMap.of("ENABLE_CROSS_LANGUAGE_TESTS", "true"), + new File("../../go/fory/tests")); + } +}