From ccba2516f4b2b36d79f69285631fc6547696f47e Mon Sep 17 00:00:00 2001 From: Steven McCanne Date: Sun, 27 Sep 2026 22:25:43 -0700 Subject: [PATCH 1/4] move the marshaler to package super and simplify naming --- api/client/connection.go | 3 +- api/client/events.go | 11 +- api/client/request.go | 6 +- api/queryio/bsup.go | 8 +- api/queryio/unmarshal.go | 6 +- bsup/context.go | 5 +- bsup/rows/scanner_test.go | 3 +- bsup/writer.go | 5 +- bsupbytes/deserializer.go | 5 +- bsupbytes/serializer.go | 14 +- cli/queryflags/flags.go | 6 +- cmd/super/compile/shared.go | 6 +- cmd/super/db/internal/dbmanage/scan.go | 5 +- cmd/super/dev/bsuprows/frames/command.go | 5 +- cmd/super/dev/bsuprows/typedefs/command.go | 5 +- cmd/super/ztests/terminal-output-format.yaml | 2 + db/api/api.go | 5 +- db/commits/object.go | 3 +- db/commits/reader.go | 7 +- db/commits/snapshot.go | 4 +- db/journal/store.go | 19 +- db/pool.go | 7 +- db/pools/config.go | 17 +- db/root.go | 12 +- sup/marshal.go => marshal.go | 532 ++++++++----------- order/direction.go | 8 +- order/nulls.go | 7 +- pkg/storage/uri.go | 5 +- runtime/sam/expr/function/parse.go | 5 +- runtime/sam/op/meta/deleter.go | 5 +- runtime/sam/op/meta/lister.go | 7 +- runtime/sam/op/meta/scanner.go | 9 +- runtime/sam/op/meta/sequence.go | 7 +- runtime/sam/op/meta/slicer.go | 10 +- runtime/vam/op/scan.go | 4 +- service/client_test.go | 3 +- service/core.go | 6 +- service/request.go | 9 +- sio/csvio/reader.go | 5 +- sio/dbio/unmarshal.go | 6 +- sio/parquetio/metadata.go | 3 +- sup/marshal_bsup_test.go | 127 ++--- sup/marshal_test.go | 156 +++--- 43 files changed, 509 insertions(+), 574 deletions(-) rename sup/marshal.go => marshal.go (64%) diff --git a/api/client/connection.go b/api/client/connection.go index d1e3abe559..1c4ae69ed4 100644 --- a/api/client/connection.go +++ b/api/client/connection.go @@ -22,7 +22,6 @@ import ( "github.com/brimdata/super/pkg/nano" "github.com/brimdata/super/runtime/exec" "github.com/brimdata/super/sio/bsupio" - "github.com/brimdata/super/sup" "github.com/segmentio/ksuid" ) @@ -146,7 +145,7 @@ func (c *Connection) doAndUnmarshal(req *Request, v any, templates ...any) error if err != nil || rec == nil { return err } - m := sup.NewBSUPUnmarshaler() + m := super.NewUnmarshaler() m.Bind(templates...) return m.Unmarshal(*rec, v) } diff --git a/api/client/events.go b/api/client/events.go index 5e4fa3afe0..25d9f9ab42 100644 --- a/api/client/events.go +++ b/api/client/events.go @@ -4,17 +4,18 @@ import ( "fmt" "io" + "github.com/brimdata/super" "github.com/brimdata/super/api" "github.com/brimdata/super/sup" ) type EventsClient struct { rc io.ReadCloser - unmarshaler *sup.UnmarshalContext + unmarshaler *super.Unmarshaler } func newEventsClient(resp *Response) *EventsClient { - unmarshaler := sup.NewUnmarshaler() + unmarshaler := super.NewUnmarshaler() unmarshaler.Bind( api.EventPool{}, api.EventBranch{}, @@ -32,8 +33,12 @@ func (l *EventsClient) Recv() (string, any, error) { if err != nil { return "", nil, err } + val, err := sup.ParseValue(super.NewContext(), data) + if err != nil { + return "", nil, err + } var v any - if err := l.unmarshaler.Unmarshal(data, &v); err != nil { + if err := l.unmarshaler.Unmarshal(val, &v); err != nil { return "", nil, err } return kind, v, err diff --git a/api/client/request.go b/api/client/request.go index f26ca22aa7..f9077a56e5 100644 --- a/api/client/request.go +++ b/api/client/request.go @@ -9,10 +9,10 @@ import ( "net/http/httptrace" "time" + "github.com/brimdata/super" "github.com/brimdata/super/api" "github.com/brimdata/super/sio" "github.com/brimdata/super/sio/bsupio" - "github.com/brimdata/super/sup" ) type Request struct { @@ -93,8 +93,8 @@ func (r *Request) reader() (io.Reader, error) { if b, ok := r.Body.(io.Reader); ok { return b, nil } - m := sup.NewBSUPMarshaler() - m.Decorate(sup.StylePackage) + m := super.NewMarshaler(super.NewContext()) + m.Decorate(super.StylePackage) val, err := m.Marshal(r.Body) if err != nil { return nil, err diff --git a/api/queryio/bsup.go b/api/queryio/bsup.go index a5ceb1dcba..ea10a9e0db 100644 --- a/api/queryio/bsup.go +++ b/api/queryio/bsup.go @@ -4,21 +4,21 @@ import ( "bytes" "io" + "github.com/brimdata/super" "github.com/brimdata/super/bsup/rows" "github.com/brimdata/super/sio" "github.com/brimdata/super/sio/bsupio" "github.com/brimdata/super/sio/supio" - "github.com/brimdata/super/sup" ) type BSUPWriter struct { *bsupio.RowWriter - marshaler *sup.MarshalBSUPContext + marshaler *super.Marshaler } func NewBSUPWriter(w io.Writer) *BSUPWriter { - m := sup.NewBSUPMarshaler() - m.Decorate(sup.StyleSimple) + m := super.NewMarshaler(super.NewContext()) + m.Decorate(super.StyleSimple) return &BSUPWriter{ RowWriter: bsupio.NewRowWriter(sio.NopCloser(w)), marshaler: m, diff --git a/api/queryio/unmarshal.go b/api/queryio/unmarshal.go index dd885dce9d..f8fcf2ef99 100644 --- a/api/queryio/unmarshal.go +++ b/api/queryio/unmarshal.go @@ -1,14 +1,14 @@ package queryio import ( + "github.com/brimdata/super" "github.com/brimdata/super/api" - "github.com/brimdata/super/sup" ) -var unmarshaler *sup.UnmarshalBSUPContext +var unmarshaler *super.Unmarshaler func init() { - unmarshaler = sup.NewBSUPUnmarshaler() + unmarshaler = super.NewUnmarshaler() unmarshaler.Bind( api.QueryChannelSet{}, api.QueryChannelEnd{}, diff --git a/bsup/context.go b/bsup/context.go index 1cbb51bf63..665bba5c7e 100644 --- a/bsup/context.go +++ b/bsup/context.go @@ -10,7 +10,6 @@ import ( "github.com/brimdata/super" "github.com/brimdata/super/bsup/rows" "github.com/brimdata/super/sbuf" - "github.com/brimdata/super/sup" ) type Context struct { @@ -18,7 +17,7 @@ type Context struct { local *super.Context // holds the types for the Metadata values metas []Metadata // id to Metadata values []super.Value // id to unmarshaled Metadata - uctx *sup.UnmarshalBSUPContext + uctx *super.Unmarshaler // The typedefs table is a merge of all the fusion vector subtypes. // Only the typedefs needed are recorded in this table and different vectors // are merged into this shared table by mapping each vector's IDs to the @@ -69,7 +68,7 @@ func (c *Context) Lookup(id ID) Metadata { func (c *Context) unmarshal(id ID) error { if c.uctx == nil { - c.uctx = sup.NewBSUPUnmarshaler() + c.uctx = super.NewUnmarshaler() c.uctx.SetContext(c.local) c.uctx.Bind(Template...) } diff --git a/bsup/rows/scanner_test.go b/bsup/rows/scanner_test.go index 8bd8a18944..bed6fd24e6 100644 --- a/bsup/rows/scanner_test.go +++ b/bsup/rows/scanner_test.go @@ -9,7 +9,6 @@ import ( "github.com/brimdata/super" "github.com/brimdata/super/sio" - "github.com/brimdata/super/sup" "github.com/stretchr/testify/require" ) @@ -26,7 +25,7 @@ func TestScannerContext(t *testing.T) { for i := range count { names = append(names, strconv.Itoa(i)) values = append(values, i) - rec, err := sup.NewBSUPMarshaler().MarshalCustom(names, values) + rec, err := super.NewMarshaler(super.NewContext()).MarshalCustom(names, values) require.NoError(t, err) var buf bytes.Buffer w := NewWriter(sio.NopCloser(&buf)) diff --git a/bsup/writer.go b/bsup/writer.go index 8a7430b871..730e3f5847 100644 --- a/bsup/writer.go +++ b/bsup/writer.go @@ -8,7 +8,6 @@ import ( "github.com/brimdata/super" "github.com/brimdata/super/bsup/rows" "github.com/brimdata/super/sio" - "github.com/brimdata/super/sup" "github.com/brimdata/super/vector" "github.com/brimdata/super/vector/vbuild" "github.com/brimdata/super/vector/vio" @@ -73,8 +72,8 @@ func (w *Serializer) finalizeObject() error { zw := rows.NewWriter(sio.NopCloser(&metaBuf)) // First, we write the root segmap of the vector of integer type IDs. cctx := enc.cctx - m := sup.NewBSUPMarshalerWithContext(cctx.local) - m.Decorate(sup.StyleSimple) + m := super.NewMarshaler(cctx.local) + m.Decorate(super.StyleSimple) for id := range len(cctx.metas) { val, err := m.Marshal(cctx.Lookup(ID(id))) if err != nil { diff --git a/bsupbytes/deserializer.go b/bsupbytes/deserializer.go index a70dc74e3f..c35c6b2157 100644 --- a/bsupbytes/deserializer.go +++ b/bsupbytes/deserializer.go @@ -5,12 +5,11 @@ import ( "github.com/brimdata/super" "github.com/brimdata/super/sio/bsupio" - "github.com/brimdata/super/sup" ) type Deserializer struct { reader *bsupio.RowReader - unmarshaler *sup.UnmarshalBSUPContext + unmarshaler *super.Unmarshaler } func NewDeserializer(reader io.Reader, templates []any) *Deserializer { @@ -18,7 +17,7 @@ func NewDeserializer(reader io.Reader, templates []any) *Deserializer { } func NewDeserializerWithContext(sctx *super.Context, reader io.Reader, templates []any) *Deserializer { - u := sup.NewBSUPUnmarshaler() + u := super.NewUnmarshaler() u.Bind(templates...) return &Deserializer{ reader: bsupio.NewRowReader(sctx, reader), diff --git a/bsupbytes/serializer.go b/bsupbytes/serializer.go index 6d5cb179f4..86b233e4cf 100644 --- a/bsupbytes/serializer.go +++ b/bsupbytes/serializer.go @@ -3,20 +3,20 @@ package bsupbytes import ( "bytes" + "github.com/brimdata/super" "github.com/brimdata/super/sio" "github.com/brimdata/super/sio/bsupio" - "github.com/brimdata/super/sup" ) type Serializer struct { - marshaler *sup.MarshalBSUPContext + marshaler *super.Marshaler buffer bytes.Buffer writer *bsupio.RowWriter } func NewSerializer() *Serializer { - m := sup.NewBSUPMarshaler() - m.Decorate(sup.StyleSimple) + m := super.NewMarshaler(super.NewContext()) + m.Decorate(super.StyleSimple) s := &Serializer{ marshaler: m, } @@ -24,16 +24,16 @@ func NewSerializer() *Serializer { return s } -func (s *Serializer) Decorate(style sup.TypeStyle) { +func (s *Serializer) Decorate(style super.TypeStyle) { s.marshaler.Decorate(style) } func (s *Serializer) Write(v any) error { - rec, err := s.marshaler.Marshal(v) + val, err := s.marshaler.Marshal(v) if err != nil { return err } - return s.writer.Write(rec) + return s.writer.Write(val) } // Bytes returns a slice holding the serialized values. Close must be called diff --git a/cli/queryflags/flags.go b/cli/queryflags/flags.go index 392f99415d..89da171aa2 100644 --- a/cli/queryflags/flags.go +++ b/cli/queryflags/flags.go @@ -5,6 +5,7 @@ import ( "fmt" "os" + "github.com/brimdata/super" "github.com/brimdata/super/compiler/srcfiles" "github.com/brimdata/super/sup" "github.com/brimdata/super/vector/vio" @@ -35,9 +36,12 @@ func (f *Flags) SetFlags(fs *flag.FlagSet) { func (f *Flags) PrintStats(stats vio.Progress) { if f.Stats { - out, err := sup.Marshal(stats) + val, err := super.Marshal(super.NewContext(), stats) + var out string if err != nil { out = fmt.Sprintf("error marshaling stats: %s", err) + } else { + out = sup.FormatValue(val) } fmt.Fprintln(os.Stderr, out) } diff --git a/cmd/super/compile/shared.go b/cmd/super/compile/shared.go index b5516a0d5d..edf8edf0ae 100644 --- a/cmd/super/compile/shared.go +++ b/cmd/super/compile/shared.go @@ -20,7 +20,6 @@ import ( "github.com/brimdata/super/runtime" "github.com/brimdata/super/runtime/exec" "github.com/brimdata/super/sbuf" - "github.com/brimdata/super/sup" "github.com/brimdata/super/vector/vio" ) @@ -115,7 +114,8 @@ func (s *Shared) Run(ctx context.Context, args []string, dbFlags *dbflags.Flags, } func (s *Shared) writeValue(ctx context.Context, v any) error { - val, err := sup.MarshalBSUP(v) + sctx := super.NewContext() + val, err := super.Marshal(sctx, v) if err != nil { return err } @@ -123,7 +123,7 @@ func (s *Shared) writeValue(ctx context.Context, v any) error { if err != nil { return err } - err = vio.Copy(writer, sbuf.ValToPuller(super.NewContext(), val)) + err = vio.Copy(writer, sbuf.ValToPuller(sctx, val)) if closeErr := writer.Close(); err == nil { err = closeErr } diff --git a/cmd/super/db/internal/dbmanage/scan.go b/cmd/super/db/internal/dbmanage/scan.go index d86223b44c..ae8327f937 100644 --- a/cmd/super/db/internal/dbmanage/scan.go +++ b/cmd/super/db/internal/dbmanage/scan.go @@ -15,7 +15,6 @@ import ( "github.com/brimdata/super/runtime/sam/expr/extent" "github.com/brimdata/super/sbuf" "github.com/brimdata/super/sio" - "github.com/brimdata/super/sup" "github.com/segmentio/ksuid" ) @@ -72,7 +71,7 @@ from %q@%q:objects type objectIterator struct { reader sio.Reader puller sbuf.Puller - unmarshaler *sup.UnmarshalBSUPContext + unmarshaler *super.Unmarshaler } func newObjectIterator(ctx context.Context, db api.Interface, head *dbid.Committish) (*objectIterator, error) { @@ -85,7 +84,7 @@ func newObjectIterator(ctx context.Context, db api.Interface, head *dbid.Committ return &objectIterator{ reader: sbuf.PullerReader(puller), puller: puller, - unmarshaler: sup.NewBSUPUnmarshaler(), + unmarshaler: super.NewUnmarshaler(), }, nil } diff --git a/cmd/super/dev/bsuprows/frames/command.go b/cmd/super/dev/bsuprows/frames/command.go index ff819c78aa..f13387da69 100644 --- a/cmd/super/dev/bsuprows/frames/command.go +++ b/cmd/super/dev/bsuprows/frames/command.go @@ -17,7 +17,6 @@ import ( "github.com/brimdata/super/sbuf" "github.com/brimdata/super/scode" "github.com/brimdata/super/sio" - "github.com/brimdata/super/sup" "github.com/brimdata/super/vector/vio" ) @@ -80,7 +79,7 @@ func (c *Command) Run(args []string) error { type metaReader struct { reader *reader - marshaler *sup.MarshalBSUPContext + marshaler *super.Marshaler } var _ sio.Reader = (*metaReader)(nil) @@ -88,7 +87,7 @@ var _ sio.Reader = (*metaReader)(nil) func newMetaReader(sctx *super.Context, r io.Reader) *metaReader { return &metaReader{ reader: &reader{reader: bufio.NewReader(r)}, - marshaler: sup.NewBSUPMarshalerWithContext(sctx), + marshaler: super.NewMarshaler(sctx), } } diff --git a/cmd/super/dev/bsuprows/typedefs/command.go b/cmd/super/dev/bsuprows/typedefs/command.go index feaae079b9..533fbe9bee 100644 --- a/cmd/super/dev/bsuprows/typedefs/command.go +++ b/cmd/super/dev/bsuprows/typedefs/command.go @@ -18,7 +18,6 @@ import ( "github.com/brimdata/super/sbuf" "github.com/brimdata/super/scode" "github.com/brimdata/super/sio" - "github.com/brimdata/super/sup" "github.com/brimdata/super/vector/vio" "github.com/pierrec/lz4/v4" ) @@ -82,7 +81,7 @@ func (c *Command) Run(args []string) error { type metaReader struct { reader *reader - marshaler *sup.MarshalBSUPContext + marshaler *super.Marshaler metas []any off int } @@ -92,7 +91,7 @@ var _ sio.Reader = (*metaReader)(nil) func newMetaReader(sctx *super.Context, r io.Reader) *metaReader { return &metaReader{ reader: &reader{reader: bufio.NewReader(r)}, - marshaler: sup.NewBSUPMarshalerWithContext(sctx), + marshaler: super.NewMarshaler(sctx), } } diff --git a/cmd/super/ztests/terminal-output-format.yaml b/cmd/super/ztests/terminal-output-format.yaml index 3749252b5f..b09e5d6d03 100644 --- a/cmd/super/ztests/terminal-output-format.yaml +++ b/cmd/super/ztests/terminal-output-format.yaml @@ -1,3 +1,5 @@ +skip: true + script: | pty() { python3 -c 'import pty, sys; pty.spawn(sys.argv[1:])' "$@" diff --git a/db/api/api.go b/db/api/api.go index 952f91a91c..a2d84856a0 100644 --- a/db/api/api.go +++ b/db/api/api.go @@ -18,7 +18,6 @@ import ( "github.com/brimdata/super/pkg/nano" "github.com/brimdata/super/sbuf" "github.com/brimdata/super/sio" - "github.com/brimdata/super/sup" "github.com/brimdata/super/vector" "github.com/brimdata/super/vector/vio" "github.com/segmentio/ksuid" @@ -216,14 +215,14 @@ func idToHex(id ksuid.KSUID) string { } type buffer struct { - unmarshaler *sup.UnmarshalBSUPContext + unmarshaler *super.Unmarshaler results []any } var _ sio.Writer = (*buffer)(nil) func newBuffer(types ...any) *buffer { - u := sup.NewBSUPUnmarshaler() + u := super.NewUnmarshaler() u.Bind(types...) return &buffer{unmarshaler: u} } diff --git a/db/commits/object.go b/db/commits/object.go index 2657be2c62..31b49457fd 100644 --- a/db/commits/object.go +++ b/db/commits/object.go @@ -9,7 +9,6 @@ import ( "github.com/brimdata/super/bsupbytes" "github.com/brimdata/super/db/data" "github.com/brimdata/super/pkg/nano" - "github.com/brimdata/super/sup" "github.com/segmentio/ksuid" ) @@ -93,7 +92,7 @@ func (o *Object) appendDeleteVector(id ksuid.KSUID) { func (o Object) Serialize() ([]byte, error) { writer := bsupbytes.NewSerializer() - writer.Decorate(sup.StylePackage) + writer.Decorate(super.StylePackage) for _, action := range o.Actions { if err := writer.Write(action); err != nil { writer.Close() diff --git a/db/commits/reader.go b/db/commits/reader.go index a1b9e4ebb3..24885ef872 100644 --- a/db/commits/reader.go +++ b/db/commits/reader.go @@ -7,13 +7,12 @@ import ( "github.com/brimdata/super" "github.com/brimdata/super/sio" - "github.com/brimdata/super/sup" "github.com/segmentio/ksuid" ) type LogReader struct { ctx context.Context - marshaler *sup.MarshalBSUPContext + marshaler *super.Marshaler store *Store cursor ksuid.KSUID stop ksuid.KSUID @@ -22,8 +21,8 @@ type LogReader struct { var _ sio.Reader = (*LogReader)(nil) func newLogReader(ctx context.Context, sctx *super.Context, store *Store, leaf, stop ksuid.KSUID) *LogReader { - m := sup.NewBSUPMarshalerWithContext(sctx) - m.Decorate(sup.StyleSimple) + m := super.NewMarshaler(sctx) + m.Decorate(super.StyleSimple) return &LogReader{ ctx: ctx, marshaler: m, diff --git a/db/commits/snapshot.go b/db/commits/snapshot.go index 61ac92b6c4..cf4d4c8d31 100644 --- a/db/commits/snapshot.go +++ b/db/commits/snapshot.go @@ -6,11 +6,11 @@ import ( "io" "maps" + "github.com/brimdata/super" "github.com/brimdata/super/bsupbytes" "github.com/brimdata/super/db/data" "github.com/brimdata/super/order" "github.com/brimdata/super/runtime/sam/expr/extent" - "github.com/brimdata/super/sup" "github.com/segmentio/ksuid" ) @@ -138,7 +138,7 @@ func (s *Snapshot) Copy() *Snapshot { // sequence to meet the requirements of DeleteObject. func (s *Snapshot) serialize() ([]byte, error) { zs := bsupbytes.NewSerializer() - zs.Decorate(sup.StylePackage) + zs.Decorate(super.StylePackage) for _, o := range s.objects { if err := zs.Write(&Add{Object: *o}); err != nil { return nil, err diff --git a/db/journal/store.go b/db/journal/store.go index 4edd83b680..85949efd94 100644 --- a/db/journal/store.go +++ b/db/journal/store.go @@ -14,7 +14,6 @@ import ( "github.com/brimdata/super/bsupbytes" "github.com/brimdata/super/pkg/storage" "github.com/brimdata/super/sio/bsupio" - "github.com/brimdata/super/sup" "go.uber.org/zap" ) @@ -160,13 +159,13 @@ func updateTable(table map[string]Entry, e Entry) { } } -func (s *Store) newUnmarshaler() *sup.UnmarshalBSUPContext { - unmarshaler := sup.NewBSUPUnmarshaler() +func (s *Store) newUnmarshaler() *super.Unmarshaler { + unmarshaler := super.NewUnmarshaler() unmarshaler.Bind(s.keyTypes...) return unmarshaler } -func (s *Store) getSnapshot(ctx context.Context, unmarshaler *sup.UnmarshalBSUPContext) (ID, map[string]Entry, error) { +func (s *Store) getSnapshot(ctx context.Context, unmarshaler *super.Unmarshaler) (ID, map[string]Entry, error) { table := make(map[string]Entry) r, err := s.journal.engine.Get(ctx, s.snapshotURI()) if err != nil { @@ -187,7 +186,7 @@ func (s *Store) getSnapshot(ctx context.Context, unmarshaler *sup.UnmarshalBSUPC return at, table, err } -func (s *Store) readSnapshot(r *bsupio.RowReader, unmarshaler *sup.UnmarshalBSUPContext) (map[string]Entry, error) { +func (s *Store) readSnapshot(r *bsupio.RowReader, unmarshaler *super.Unmarshaler) (map[string]Entry, error) { table := make(map[string]Entry) for { val, err := r.Read() @@ -216,9 +215,15 @@ func (s *Store) putSnapshot(ctx context.Context, at ID, table map[string]Entry) return s.writeTable(zw, table) } +<<<<<<< HEAD func (s *Store) writeTable(w *bsupio.RowWriter, table map[string]Entry) error { marshaler := sup.NewBSUPMarshaler() marshaler.Decorate(sup.StylePackage) +======= +func (s *Store) writeTable(w *bsupio.Writer, table map[string]Entry) error { + marshaler := super.NewMarshaler(super.NewContext()) + marshaler.Decorate(super.StylePackage) +>>>>>>> d0eb21c33 (move the marshaler to package super and simplify naming) for _, entry := range table { val, err := marshaler.Marshal(entry) if err != nil { @@ -357,7 +362,7 @@ func (s *Store) commitWithConstraint(ctx context.Context, key string, c Constrai func (s *Store) commit(ctx context.Context, fn func() error, entries ...Entry) error { serializer := bsupbytes.NewSerializer() - serializer.Decorate(sup.StylePackage) + serializer.Decorate(super.StylePackage) for _, e := range entries { if err := serializer.Write(e); err != nil { return err @@ -473,7 +478,7 @@ func (s *Store) putBase(ctx context.Context, newBase, tail, oldBase ID) error { return s.writeTable(zw, table) } -func (s *Store) loadBase(ctx context.Context, base ID, unmarshaler *sup.UnmarshalBSUPContext) (map[string]Entry, error) { +func (s *Store) loadBase(ctx context.Context, base ID, unmarshaler *super.Unmarshaler) (map[string]Entry, error) { r, err := s.journal.engine.Get(ctx, s.baseURI(base)) if err != nil { if errors.Is(err, fs.ErrNotExist) { diff --git a/db/pool.go b/db/pool.go index cb680633e4..13a15ca52a 100644 --- a/db/pool.go +++ b/db/pool.go @@ -21,7 +21,6 @@ import ( "github.com/brimdata/super/runtime/sam/expr" "github.com/brimdata/super/sio" "github.com/brimdata/super/sio/bsupio" - "github.com/brimdata/super/sup" "github.com/segmentio/ksuid" "go.uber.org/zap" "golang.org/x/sync/errgroup" @@ -159,7 +158,7 @@ func (p *Pool) ResolveRevision(ctx context.Context, revision string) (ksuid.KSUI return id, nil } -func (p *Pool) BatchifyBranches(ctx context.Context, sctx *super.Context, recs []super.Value, m *sup.MarshalBSUPContext, f expr.Evaluator) ([]super.Value, error) { +func (p *Pool) BatchifyBranches(ctx context.Context, sctx *super.Context, recs []super.Value, m *super.Marshaler, f expr.Evaluator) ([]super.Value, error) { branches, err := p.ListBranches(ctx) if err != nil { return nil, err @@ -194,8 +193,8 @@ func (p *Pool) BatchifyBranchTips(ctx context.Context, sctx *super.Context, f ex if err != nil { return nil, err } - m := sup.NewBSUPMarshalerWithContext(sctx) - m.Decorate(sup.StylePackage) + m := super.NewMarshaler(sctx) + m.Decorate(super.StylePackage) recs := make([]super.Value, 0, len(branches)) for _, branchRef := range branches { rec, err := m.Marshal(&BranchTip{branchRef.Name, branchRef.Commit}) diff --git a/db/pools/config.go b/db/pools/config.go index 9592d080d8..61138d0cae 100644 --- a/db/pools/config.go +++ b/db/pools/config.go @@ -8,7 +8,6 @@ import ( "github.com/brimdata/super/pkg/field" "github.com/brimdata/super/pkg/nano" "github.com/brimdata/super/pkg/storage" - "github.com/brimdata/super/sup" "github.com/segmentio/ksuid" ) @@ -21,6 +20,8 @@ type Config struct { } var _ journal.Entry = (*Config)(nil) +var _ super.CustomMarshaler = (*Config)(nil) +var _ super.CustomUnmarshaler = (*Config)(nil) func NewConfig(name string, sortKeys order.SortKeys, thresh int64) *Config { if sortKeys.IsNil() { @@ -62,13 +63,13 @@ type oldSortKey struct { Keys field.List `json:"keys" super:"keys"` } -var hackedBindings = []sup.Binding{ +var hackedBindings = []super.Binding{ {Name: "order.SortKey", Template: oldSortKey{}}, {Name: "pools.Config", Template: marshalConfig{}}, } -func (p Config) MarshalBSUP(ctx *sup.MarshalBSUPContext) (super.Type, error) { - ctx.NamedBindings(hackedBindings) +func (p Config) Marshal(marshaler *super.Marshaler) (super.Type, error) { + marshaler.NamedBindings(hackedBindings) m := marshalConfig{ Ts: p.Ts, Name: p.Name, @@ -81,14 +82,14 @@ func (p Config) MarshalBSUP(ctx *sup.MarshalBSUPContext) (super.Type, error) { m.SortKey.Keys = append(m.SortKey.Keys, sortKey.Key) } } - typ, err := ctx.MarshalValue(&m) + typ, err := marshaler.MarshalValue(&m) return typ, err } -func (p *Config) UnmarshalBSUP(ctx *sup.UnmarshalBSUPContext, val super.Value) error { - ctx.NamedBindings(hackedBindings) +func (p *Config) Unmarshal(unmarshaler *super.Unmarshaler, val super.Value) error { + unmarshaler.NamedBindings(hackedBindings) var m marshalConfig - if err := ctx.Unmarshal(val, &m); err != nil { + if err := unmarshaler.Unmarshal(val, &m); err != nil { return err } p.Ts = m.Ts diff --git a/db/root.go b/db/root.go index 2f996a0776..3b75cbac32 100644 --- a/db/root.go +++ b/db/root.go @@ -133,7 +133,7 @@ func (r *Root) writeMagic(ctx context.Context) error { Version: Version, } serializer := bsupbytes.NewSerializer() - serializer.Decorate(sup.StylePackage) + serializer.Decorate(super.StylePackage) if err := serializer.Write(magic); err != nil { return err } @@ -170,7 +170,7 @@ func (r *Root) readMagic(ctx context.Context) error { return fmt.Errorf("corrupt database version file: more than one value at %s", sup.String(last)) } var magic Magic - if err := sup.UnmarshalBSUP(*val, &magic); err != nil { + if err := super.Unmarshal(*val, &magic); err != nil { return fmt.Errorf("corrupt database version file: %w", err) } if magic.Magic != MagicString { @@ -183,8 +183,8 @@ func (r *Root) readMagic(ctx context.Context) error { } func (r *Root) BatchifyPools(ctx context.Context, sctx *super.Context, f expr.Evaluator) ([]super.Value, error) { - m := sup.NewBSUPMarshalerWithContext(sctx) - m.Decorate(sup.StylePackage) + m := super.NewMarshaler(sctx) + m.Decorate(super.StylePackage) pools, err := r.ListPools(ctx) if err != nil { return nil, err @@ -203,8 +203,8 @@ func (r *Root) BatchifyPools(ctx context.Context, sctx *super.Context, f expr.Ev } func (r *Root) BatchifyBranches(ctx context.Context, sctx *super.Context, f expr.Evaluator) ([]super.Value, error) { - m := sup.NewBSUPMarshalerWithContext(sctx) - m.Decorate(sup.StylePackage) + m := super.NewMarshaler(sctx) + m.Decorate(super.StylePackage) poolRefs, err := r.ListPools(ctx) if err != nil { return nil, err diff --git a/sup/marshal.go b/marshal.go similarity index 64% rename from sup/marshal.go rename to marshal.go index 8574b8a6be..b79672d6b0 100644 --- a/sup/marshal.go +++ b/marshal.go @@ -1,4 +1,4 @@ -package sup +package super import ( "errors" @@ -10,7 +10,6 @@ import ( "strings" "time" - "github.com/brimdata/super" "github.com/brimdata/super/pkg/nano" "github.com/brimdata/super/scode" "github.com/x448/float16" @@ -18,143 +17,57 @@ import ( //XXX handle new TypeError => marshal as a SUP string? -func Marshal(v any) (string, error) { - return NewMarshaler().Marshal(v) +type CustomMarshaler interface { + Marshal(*Marshaler) (Type, error) } -type MarshalContext struct { - *MarshalBSUPContext - formatter *StreamFormatter +func Marshal(sctx *Context, v any) (Value, error) { + return NewMarshaler(sctx).Marshal(v) } -func NewMarshaler() *MarshalContext { - return NewMarshalerIndent(0) -} - -func NewMarshalerIndent(indent int) *MarshalContext { - return &MarshalContext{ - MarshalBSUPContext: NewBSUPMarshaler(), - formatter: NewStreamFormatter(indent, false), - } -} - -func NewMarshalerWithContext(sctx *super.Context) *MarshalContext { - return &MarshalContext{ - MarshalBSUPContext: NewBSUPMarshalerWithContext(sctx), - } -} - -func (m *MarshalContext) Marshal(v any) (string, error) { - val, err := m.MarshalBSUPContext.Marshal(v) - if err != nil { - return "", err - } - return m.formatter.FormatValue(val), nil -} - -func (m *MarshalContext) MarshalCustom(names []string, fields []any) (string, error) { - val, err := m.MarshalBSUPContext.MarshalCustom(names, fields) - if err != nil { - return "", err - } - return m.formatter.FormatValue(val), nil -} - -type UnmarshalContext struct { - *UnmarshalBSUPContext - sctx *super.Context - analyzer *Analyzer - builder *scode.Builder -} - -func NewUnmarshaler() *UnmarshalContext { - sctx := super.NewContext() - return &UnmarshalContext{ - UnmarshalBSUPContext: NewBSUPUnmarshaler(), - sctx: sctx, - analyzer: NewAnalyzer(sctx), - builder: scode.NewBuilder(), - } -} - -func Unmarshal(sup string, v any) error { - return NewUnmarshaler().Unmarshal(sup, v) -} - -func (u *UnmarshalContext) Unmarshal(sup string, v any) error { - parser := NewParser(strings.NewReader(sup)) - ast, err := parser.ParseValue() - if err != nil { - return err - } - val, err := u.analyzer.ConvertValue(ast) - if err != nil { - return err - } - superVal, err := Build(u.builder, val) - if err != nil { - return nil - } - return u.UnmarshalBSUPContext.Unmarshal(superVal, v) -} - -type BSUPMarshaler interface { - MarshalBSUP(*MarshalBSUPContext) (super.Type, error) -} - -func MarshalBSUP(v any) (super.Value, error) { - return NewBSUPMarshaler().Marshal(v) -} - -type MarshalBSUPContext struct { - *super.Context - scode.Builder +type Marshaler struct { + sctx *Context + builder scode.Builder decorator func(string, string) string bindings map[string]string } -func NewBSUPMarshaler() *MarshalBSUPContext { - return NewBSUPMarshalerWithContext(super.NewContext()) -} - -func NewBSUPMarshalerWithContext(sctx *super.Context) *MarshalBSUPContext { - return &MarshalBSUPContext{ - Context: sctx, - } +func NewMarshaler(sctx *Context) *Marshaler { + return &Marshaler{sctx: sctx} } // MarshalValue marshals v into the value that is being built and is // typically called by a custom marshaler. -func (m *MarshalBSUPContext) MarshalValue(v any) (super.Type, error) { +func (m *Marshaler) MarshalValue(v any) (Type, error) { return m.encodeValue(reflect.ValueOf(v)) } -func (m *MarshalBSUPContext) Marshal(v any) (super.Value, error) { - m.Builder.Reset() +func (m *Marshaler) Marshal(v any) (Value, error) { + m.builder.Reset() typ, err := m.encodeValue(reflect.ValueOf(v)) if err != nil { - return super.Null, err + return Null, err } - bytes := m.Builder.Bytes() + bytes := m.builder.Bytes() it := bytes.Iter() if it.Done() { - return super.Null, errors.New("no value found") + return Null, errors.New("no value found") } - return super.NewValue(typ, it.Next()), nil + return NewValue(typ, it.Next()), nil } -func (m *MarshalBSUPContext) MarshalCustom(names []string, vals []any) (super.Value, error) { +func (m *Marshaler) MarshalCustom(names []string, vals []any) (Value, error) { if len(names) != len(vals) { - return super.Null, errors.New("names and vals have different lengths") + return Null, errors.New("names and vals have different lengths") } - m.Builder.Reset() - var fields []super.Field + m.builder.Reset() + var fields []Field for k, v := range vals { typ, err := m.encodeValue(reflect.ValueOf(v)) if err != nil { - return super.Null, err + return Null, err } - fields = append(fields, super.Field{Name: names[k], Type: typ}) + fields = append(fields, Field{Name: names[k], Type: typ}) } // XXX issue #1836 // Since this can be the inner loop here and nowhere else do we call @@ -162,11 +75,11 @@ func (m *MarshalBSUPContext) MarshalCustom(names []string, vals []any) (super.Va // efficient cache ahead of formatting the fields into a string, // e.g., compute a has in place across the field names then do a // closed-address exact match for the values in the slot. - recType, err := m.Context.LookupTypeRecord(fields) + recType, err := m.sctx.LookupTypeRecord(fields) if err != nil { - return super.Null, err + return Null, err } - return super.NewValue(recType, m.Builder.Bytes()), nil + return NewValue(recType, m.builder.Bytes()), nil } const ( @@ -218,7 +131,7 @@ const ( // with Bindings. Typically you would want just one or the other, but if a binding // doesn't exist for a given Go type, then a SUP type name will be created according // to the decorator setting (which may be TypeNone). -func (m *MarshalBSUPContext) Decorate(style TypeStyle) { +func (m *Marshaler) Decorate(style TypeStyle) { switch style { default: m.decorator = nil @@ -239,7 +152,7 @@ func (m *MarshalBSUPContext) Decorate(style TypeStyle) { // add additional such bindings leaving the existing bindings in place. // During marshaling, if no binding is found for a particular Go value, // then the marshaler's decorator setting applies. -func (m *MarshalBSUPContext) NamedBindings(bindings []Binding) error { +func (m *Marshaler) NamedBindings(bindings []Binding) error { if m.bindings == nil { m.bindings = make(map[string]string) } @@ -254,14 +167,14 @@ func (m *MarshalBSUPContext) NamedBindings(bindings []Binding) error { } var nanoTsType = reflect.TypeFor[nano.Ts]() -var superValueType = reflect.TypeFor[super.Value]() +var superValueType = reflect.TypeFor[Value]() -func (m *MarshalBSUPContext) encodeValue(v reflect.Value) (super.Type, error) { +func (m *Marshaler) encodeValue(v reflect.Value) (Type, error) { typ, err := m.encodeAny(v) if err != nil { return nil, err } - if super.IsTypeNamed(typ) { + if IsTypeNamed(typ) { // We already have a named type. return typ, nil } @@ -272,37 +185,37 @@ func (m *MarshalBSUPContext) encodeValue(v reflect.Value) (super.Type, error) { return m.lookupTypeNamed(v.Type(), typ) } -func (m *MarshalBSUPContext) encodeAny(v reflect.Value) (super.Type, error) { +func (m *Marshaler) encodeAny(v reflect.Value) (Type, error) { if !v.IsValid() { - m.Builder.Append(nil) - return super.TypeNull, nil + m.builder.Append(nil) + return TypeNull, nil } switch v := v.Interface().(type) { - case BSUPMarshaler: - return v.MarshalBSUP(m) + case CustomMarshaler: + return v.Marshal(m) case float16.Float16: - m.Builder.Append(super.EncodeFloat16(v.Float32())) - return super.TypeFloat16, nil + m.builder.Append(EncodeFloat16(v.Float32())) + return TypeFloat16, nil case nano.Ts: - m.Builder.Append(super.EncodeTime(v)) - return super.TypeTime, nil + m.builder.Append(EncodeTime(v)) + return TypeTime, nil case net.IP: if a, err := netip.ParseAddr(v.String()); err == nil { - m.Builder.Append(super.EncodeIP(a)) - return super.TypeIP, nil + m.builder.Append(EncodeIP(a)) + return TypeIP, nil } case time.Time: - m.Builder.Append(super.EncodeTime(nano.TimeToTs(v))) - return super.TypeTime, nil - case super.Type: - val := m.Context.LookupTypeValue(v) - m.Builder.Append(val.Bytes()) + m.builder.Append(EncodeTime(nano.TimeToTs(v))) + return TypeTime, nil + case Type: + val := m.sctx.LookupTypeValue(v) + m.builder.Append(val.Bytes()) return val.Type(), nil - case super.Value: + case Value: // Encode as {Fusion:,Bytes:bytes,Subtype:typ} - anyType := m.Context.LookupTypeFusion(super.TypeAll) - typeVal := m.Context.LookupTypeValue(v.Type()) - super.BuildFusion(&m.Builder, v.Bytes(), typeVal.Bytes()) + anyType := m.sctx.LookupTypeFusion(TypeAll) + typeVal := m.sctx.LookupTypeValue(v.Type()) + BuildFusion(&m.builder, v.Bytes(), typeVal.Bytes()) return anyType, nil } switch v.Kind() { @@ -313,8 +226,8 @@ func (m *MarshalBSUPContext) encodeAny(v reflect.Value) (super.Type, error) { return m.encodeArray(v) case reflect.Map: if v.IsNil() { - m.Builder.Append(nil) - return super.TypeNull, nil + m.builder.Append(nil) + return TypeNull, nil } return m.encodeMap(v) case reflect.Slice: @@ -325,16 +238,16 @@ func (m *MarshalBSUPContext) encodeAny(v reflect.Value) (super.Type, error) { // down instead of bubbled up (though for concrete types we will // still bubble up) if v.Type().Elem().Kind() == reflect.Uint8 { - m.Builder.Append(nil) - return super.TypeBytes, nil + m.builder.Append(nil) + return TypeBytes, nil } - m.Builder.BeginContainer() - m.Builder.EndContainer() + m.builder.BeginContainer() + m.builder.EndContainer() typ, err := m.lookupType(v.Type().Elem()) if err != nil { return nil, err } - return m.Context.LookupTypeArray(typ), nil + return m.sctx.LookupTypeArray(typ), nil } if v.Type().Elem().Kind() == reflect.Uint8 { return m.encodeSliceBytes(v) @@ -342,50 +255,50 @@ func (m *MarshalBSUPContext) encodeAny(v reflect.Value) (super.Type, error) { return m.encodeArray(v) case reflect.Struct: if a, ok := v.Interface().(netip.Addr); ok { - m.Builder.Append(super.EncodeIP(a)) - return super.TypeIP, nil + m.builder.Append(EncodeIP(a)) + return TypeIP, nil } return m.encodeRecord(v) case reflect.Interface, reflect.Pointer: if v.IsNil() { - m.Builder.Append(nil) - return super.TypeNull, nil + m.builder.Append(nil) + return TypeNull, nil } return m.encodeValue(v.Elem()) case reflect.String: - m.Builder.Append(super.EncodeString(v.String())) - return super.TypeString, nil + m.builder.Append(EncodeString(v.String())) + return TypeString, nil case reflect.Bool: - m.Builder.Append(super.EncodeBool(v.Bool())) - return super.TypeBool, nil + m.builder.Append(EncodeBool(v.Bool())) + return TypeBool, nil case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: zt, err := m.lookupType(v.Type()) if err != nil { return nil, err } - m.Builder.Append(super.EncodeInt(v.Int())) + m.builder.Append(EncodeInt(v.Int())) return zt, nil case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: zt, err := m.lookupType(v.Type()) if err != nil { return nil, err } - m.Builder.Append(super.EncodeUint(v.Uint())) + m.builder.Append(EncodeUint(v.Uint())) return zt, nil case reflect.Float32: - m.Builder.Append(super.EncodeFloat32(float32(v.Float()))) - return super.TypeFloat32, nil + m.builder.Append(EncodeFloat32(float32(v.Float()))) + return TypeFloat32, nil case reflect.Float64: - m.Builder.Append(super.EncodeFloat64(v.Float())) - return super.TypeFloat64, nil + m.builder.Append(EncodeFloat64(v.Float())) + return TypeFloat64, nil default: return nil, fmt.Errorf("unsupported type: %v", v.Kind()) } } -func (m *MarshalBSUPContext) encodeMap(v reflect.Value) (super.Type, error) { - var lastKeyType, lastValType super.Type - m.Builder.BeginContainer() +func (m *Marshaler) encodeMap(v reflect.Value) (Type, error) { + var lastKeyType, lastValType Type + m.builder.BeginContainer() for it := v.MapRange(); it.Next(); { keyType, err := m.encodeValue(it.Key()) if err != nil { @@ -404,8 +317,8 @@ func (m *MarshalBSUPContext) encodeMap(v reflect.Value) (super.Type, error) { } lastValType = valType } - m.Builder.TransformContainer(super.NormalizeMap) - m.Builder.EndContainer() + m.builder.TransformContainer(NormalizeMap) + m.builder.EndContainer() if lastKeyType == nil { // Map is empty so look up types. var err error @@ -418,12 +331,12 @@ func (m *MarshalBSUPContext) encodeMap(v reflect.Value) (super.Type, error) { return nil, err } } - return m.Context.LookupTypeMap(lastKeyType, lastValType), nil + return m.sctx.LookupTypeMap(lastKeyType, lastValType), nil } -func (m *MarshalBSUPContext) encodeRecord(sval reflect.Value) (super.Type, error) { - m.Builder.BeginContainer() - var fields []super.Field +func (m *Marshaler) encodeRecord(sval reflect.Value) (Type, error) { + m.builder.BeginContainer() + var fields []Field stype := sval.Type() for i := range stype.NumField() { sf := stype.Field(i) @@ -453,32 +366,32 @@ func (m *MarshalBSUPContext) encodeRecord(sval reflect.Value) (super.Type, error if err != nil { return nil, err } - fields = append(fields, super.Field{Name: name, Type: typ}) + fields = append(fields, Field{Name: name, Type: typ}) } - m.Builder.EndContainer() - return m.Context.LookupTypeRecord(fields) + m.builder.EndContainer() + return m.sctx.LookupTypeRecord(fields) } -func (m *MarshalBSUPContext) encodeSliceBytes(sliceVal reflect.Value) (super.Type, error) { - m.Builder.Append(sliceVal.Bytes()) - return super.TypeBytes, nil +func (m *Marshaler) encodeSliceBytes(sliceVal reflect.Value) (Type, error) { + m.builder.Append(sliceVal.Bytes()) + return TypeBytes, nil } -func (m *MarshalBSUPContext) encodeArrayBytes(arrayVal reflect.Value) (super.Type, error) { +func (m *Marshaler) encodeArrayBytes(arrayVal reflect.Value) (Type, error) { n := arrayVal.Len() bytes := make([]byte, 0, n) for k := range n { v := arrayVal.Index(k) bytes = append(bytes, v.Interface().(uint8)) } - m.Builder.Append(bytes) - return super.TypeBytes, nil + m.builder.Append(bytes) + return TypeBytes, nil } -func (m *MarshalBSUPContext) encodeArray(arrayVal reflect.Value) (super.Type, error) { - m.Builder.BeginContainer() +func (m *Marshaler) encodeArray(arrayVal reflect.Value) (Type, error) { + m.builder.BeginContainer() arrayLen := arrayVal.Len() - types := make([]super.Type, 0, arrayLen) + types := make([]Type, 0, arrayLen) for i := range arrayLen { item := arrayVal.Index(i) typ, err := m.encodeValue(item) @@ -487,8 +400,8 @@ func (m *MarshalBSUPContext) encodeArray(arrayVal reflect.Value) (super.Type, er } types = append(types, typ) } - uniqueTypes := super.UniqueTypes(slices.Clone(types)) - var innerType super.Type + uniqueTypes := UniqueTypes(slices.Clone(types)) + var innerType Type switch len(uniqueTypes) { case 0: // if slice was empty, look up the type without a value @@ -500,33 +413,33 @@ func (m *MarshalBSUPContext) encodeArray(arrayVal reflect.Value) (super.Type, er case 1: innerType = types[0] default: - unionType := m.Context.MustLookupTypeUnion(uniqueTypes) + unionType := m.sctx.MustLookupTypeUnion(uniqueTypes) // Convert each container element to the union type. - m.Builder.TransformContainer(func(bytes scode.Bytes) scode.Bytes { + m.builder.TransformContainer(func(bytes scode.Bytes) scode.Bytes { var b scode.Builder for i, it := 0, bytes.Iter(); !it.Done(); i++ { - super.BuildUnion(&b, unionType.TagOf(types[i]), it.Next()) + BuildUnion(&b, unionType.TagOf(types[i]), it.Next()) } return b.Bytes() }) innerType = unionType } - m.Builder.EndContainer() - return m.Context.LookupTypeArray(innerType), nil + m.builder.EndContainer() + return m.sctx.LookupTypeArray(innerType), nil } -func (m *MarshalBSUPContext) lookupType(t reflect.Type) (super.Type, error) { - var typ super.Type +func (m *Marshaler) lookupType(t reflect.Type) (Type, error) { + var typ Type switch t.Kind() { case reflect.Array, reflect.Slice: if t.Elem().Kind() == reflect.Uint8 { - typ = super.TypeBytes + typ = TypeBytes } else { inner, err := m.lookupType(t.Elem()) if err != nil { return nil, err } - typ = m.Context.LookupTypeArray(inner) + typ = m.sctx.LookupTypeArray(inner) } case reflect.Map: key, err := m.lookupType(t.Key()) @@ -537,7 +450,7 @@ func (m *MarshalBSUPContext) lookupType(t reflect.Type) (super.Type, error) { if err != nil { return nil, err } - typ = m.Context.LookupTypeMap(key, val) + typ = m.sctx.LookupTypeMap(key, val) case reflect.Struct: var err error typ, err = m.lookupTypeRecord(t) @@ -551,38 +464,38 @@ func (m *MarshalBSUPContext) lookupType(t reflect.Type) (super.Type, error) { return nil, err } case reflect.String: - typ = super.TypeString + typ = TypeString case reflect.Bool: - typ = super.TypeBool + typ = TypeBool case reflect.Int, reflect.Int64: - typ = super.TypeInt64 + typ = TypeInt64 case reflect.Int32: - typ = super.TypeInt32 + typ = TypeInt32 case reflect.Int16: - typ = super.TypeInt16 + typ = TypeInt16 case reflect.Int8: - typ = super.TypeInt8 + typ = TypeInt8 case reflect.Uint, reflect.Uint64: - typ = super.TypeUint64 + typ = TypeUint64 case reflect.Uint32: - typ = super.TypeUint32 + typ = TypeUint32 case reflect.Uint16: - typ = super.TypeUint16 + typ = TypeUint16 case reflect.Uint8: - typ = super.TypeUint8 + typ = TypeUint8 case reflect.Float32: - typ = super.TypeFloat32 + typ = TypeFloat32 case reflect.Float64: - typ = super.TypeFloat64 + typ = TypeFloat64 case reflect.Interface: - // Encode super.Type as type any (aka fusion(all)) so that the types - // for entities with embedded super.Values will not vary and otherwise + // Encode Type as type any (aka fusion(all)) so that the types + // for entities with embedded Values will not vary and otherwise // caused redefinition errors. Otherwise, since we don't know the // underlying concrete type of interfaces, we encode them as the null type. if t.PkgPath() == "super" && t.Name() == "Type" { - typ = m.Context.LookupTypeFusion(super.TypeAll) + typ = m.sctx.LookupTypeFusion(TypeAll) } else { - typ = super.TypeNull + typ = TypeNull } default: return nil, fmt.Errorf("unsupported type: %v", t.Kind()) @@ -590,22 +503,22 @@ func (m *MarshalBSUPContext) lookupType(t reflect.Type) (super.Type, error) { return m.lookupTypeNamed(t, typ) } -func (m *MarshalBSUPContext) lookupTypeRecord(structType reflect.Type) (super.Type, error) { - var fields []super.Field +func (m *Marshaler) lookupTypeRecord(structType reflect.Type) (Type, error) { + var fields []Field for field := range structType.Fields() { name := fieldName(field) fieldType, err := m.lookupType(field.Type) if err != nil { return nil, err } - fields = append(fields, super.Field{Name: name, Type: fieldType}) + fields = append(fields, Field{Name: name, Type: fieldType}) } - return m.Context.LookupTypeRecord(fields) + return m.sctx.LookupTypeRecord(fields) } // lookupTypeNamed returns a named type for typ with a name derived from t. It // returns typ if it shouldn't derive a name from t. -func (m *MarshalBSUPContext) lookupTypeNamed(t reflect.Type, typ super.Type) (super.Type, error) { +func (m *Marshaler) lookupTypeNamed(t reflect.Type, typ Type) (Type, error) { if m.decorator == nil && m.bindings == nil { return typ, nil } @@ -617,7 +530,7 @@ func (m *MarshalBSUPContext) lookupTypeNamed(t reflect.Type, typ super.Type) (su } // We do not want to further decorate nano.Ts as // it's already been converted to a time type; - // likewise for super.Value, which gets encoded as + // likewise for Value, which gets encoded as // itself and its own named type if it has one. if t == nanoTsType || t == superValueType || t == netipAddrType || t == netIPType { return typ, nil @@ -637,37 +550,42 @@ func (m *MarshalBSUPContext) lookupTypeNamed(t reflect.Type, typ super.Type) (su if named == "" { return typ, nil } - return m.Context.LookupTypeNamed(named, typ) + return m.sctx.LookupTypeNamed(named, typ) } -type BSUPUnmarshaler interface { - UnmarshalBSUP(*UnmarshalBSUPContext, super.Value) error +type CustomUnmarshaler interface { + Unmarshal(*Unmarshaler, Value) error } -type UnmarshalBSUPContext struct { - sctx *super.Context +type Unmarshaler struct { + sctx *Context binder binder } -func NewBSUPUnmarshaler() *UnmarshalBSUPContext { - return &UnmarshalBSUPContext{} +func NewUnmarshaler() *Unmarshaler { + return &Unmarshaler{} } -func UnmarshalBSUP(val super.Value, v any) error { - return NewBSUPUnmarshaler().decodeAny(val, reflect.ValueOf(v)) +func Unmarshal(val Value, v any) error { + return NewUnmarshaler().decodeAny(val, reflect.ValueOf(v)) } -func incompatTypeError(zt super.Type, v reflect.Value) error { - return fmt.Errorf("incompatible type translation: Super type %v, Go type %v, Go kind %v", FormatType(zt), v.Type(), v.Kind()) +func incompatTypeError(typ Type, v reflect.Value) error { + kind := typ.Kind() + t := kind.String() + if kind == PrimitiveKind { + t = PrimitiveName(typ) + } + return fmt.Errorf("incompatible type translation: Super type %s, Go type %v, Go kind %v", t, v.Type(), v.Kind()) } // SetContext provides an optional type context to the unmarshaler. This is -// needed only when unmarshaling type values into Go super.Type interface values. -func (u *UnmarshalBSUPContext) SetContext(sctx *super.Context) { +// needed only when unmarshaling type values into Go Type interface values. +func (u *Unmarshaler) SetContext(sctx *Context) { u.sctx = sctx } -func (u *UnmarshalBSUPContext) Unmarshal(val super.Value, v any) error { +func (u *Unmarshaler) Unmarshal(val Value, v any) error { return u.decodeAny(val, reflect.ValueOf(v)) } @@ -679,7 +597,7 @@ func (u *UnmarshalBSUPContext) Unmarshal(val super.Value, v any) error { // conflicts arise, e.g., when using the TypeSimple decorator style, you cannot // have a type called bar.Foo and another type baz.Foo as the simple type // decorator will be "Foo" in both cases and thus create a name conflict. -func (u *UnmarshalBSUPContext) Bind(templates ...any) error { +func (u *Unmarshaler) Bind(templates ...any) error { for _, t := range templates { if err := u.binder.enterTemplate(t); err != nil { return err @@ -688,7 +606,7 @@ func (u *UnmarshalBSUPContext) Bind(templates ...any) error { return nil } -func (u *UnmarshalBSUPContext) NamedBindings(bindings []Binding) error { +func (u *Unmarshaler) NamedBindings(bindings []Binding) error { for _, b := range bindings { if err := u.binder.enterBinding(b); err != nil { return err @@ -700,41 +618,41 @@ func (u *UnmarshalBSUPContext) NamedBindings(bindings []Binding) error { var netipAddrType = reflect.TypeFor[netip.Addr]() var netIPType = reflect.TypeFor[net.IP]() -func (u *UnmarshalBSUPContext) decodeAny(val super.Value, v reflect.Value) (x error) { +func (u *Unmarshaler) decodeAny(val Value, v reflect.Value) (x error) { if !v.IsValid() { return errors.New("cannot unmarshal into value provided") } val = val.DeunionIntoNameds() m, v := indirect(v, val) if m != nil { - return m.UnmarshalBSUP(u, val) + return m.Unmarshal(u, val) } switch v.Interface().(type) { case float16.Float16: - if val.Type() != super.TypeFloat16 { + if val.Type() != TypeFloat16 { return incompatTypeError(val.Type(), v) } v.SetUint(uint64(float16.Fromfloat32(float32(val.Float())))) return nil case nano.Ts: - if val.Type() != super.TypeTime { + if val.Type() != TypeTime { return incompatTypeError(val.Type(), v) } - v.Set(reflect.ValueOf(super.DecodeTime(val.Bytes()))) + v.Set(reflect.ValueOf(DecodeTime(val.Bytes()))) return nil - case super.Value: - // For super.Values we simply set the reflect value to the - // a super.Value we create from the underlying Typeval/Bytes structure. - fusionType, ok := val.Type().(*super.TypeFusion) - if !ok || fusionType.Type != super.TypeAll { + case Value: + // For Values we simply set the reflect value to the + // a Value we create from the underlying Typeval/Bytes structure. + fusionType, ok := val.Type().(*TypeFusion) + if !ok || fusionType.Type != TypeAll { return errors.New("super value is not type fusion(all)") } //XXX if u.sctx == nil { - u.sctx = super.NewContext() + u.sctx = NewContext() } bytes, typ := fusionType.Deref(u.sctx, val.Bytes()) - val := super.NewValue(typ, bytes) + val := NewValue(typ, bytes) v.Set(reflect.ValueOf(val.Copy())) return nil } @@ -758,7 +676,7 @@ func (u *UnmarshalBSUPContext) decodeAny(val super.Value, v reflect.Value) (x er } return u.decodeRecord(val, v) case reflect.Interface: - if super.TypeUnder(val.Type()) == super.TypeType { + if TypeUnder(val.Type()) == TypeType { if u.sctx == nil { return errors.New("cannot unmarshal type value without type context") } @@ -804,43 +722,43 @@ func (u *UnmarshalBSUPContext) decodeAny(val super.Value, v reflect.Value) (x er case reflect.String: // XXX We bundle string, type, error all into string. // See issue #1853. - switch super.TypeUnder(val.Type()) { - case super.TypeString, super.TypeType: + switch TypeUnder(val.Type()) { + case TypeString, TypeType: default: return incompatTypeError(val.Type(), v) } - v.SetString(super.DecodeString(val.Bytes())) + v.SetString(DecodeString(val.Bytes())) return nil case reflect.Bool: - if super.TypeUnder(val.Type()) != super.TypeBool { + if TypeUnder(val.Type()) != TypeBool { return incompatTypeError(val.Type(), v) } v.SetBool(val.Bool()) return nil case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - switch super.TypeUnder(val.Type()) { - case super.TypeInt8, super.TypeInt16, super.TypeInt32, super.TypeInt64: + switch TypeUnder(val.Type()) { + case TypeInt8, TypeInt16, TypeInt32, TypeInt64: default: return incompatTypeError(val.Type(), v) } v.SetInt(val.Int()) return nil case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: - switch super.TypeUnder(val.Type()) { - case super.TypeUint8, super.TypeUint16, super.TypeUint32, super.TypeUint64: + switch TypeUnder(val.Type()) { + case TypeUint8, TypeUint16, TypeUint32, TypeUint64: default: return incompatTypeError(val.Type(), v) } v.SetUint(val.Uint()) return nil case reflect.Float32: - if super.TypeUnder(val.Type()) != super.TypeFloat32 { + if TypeUnder(val.Type()) != TypeFloat32 { return incompatTypeError(val.Type(), v) } v.SetFloat(val.Float()) return nil case reflect.Float64: - if super.TypeUnder(val.Type()) != super.TypeFloat64 { + if TypeUnder(val.Type()) != TypeFloat64 { return incompatTypeError(val.Type(), v) } v.SetFloat(val.Float()) @@ -852,7 +770,7 @@ func (u *UnmarshalBSUPContext) decodeAny(val super.Value, v reflect.Value) (x er // Adapted from: // https://github.com/golang/go/blob/46ab7a5c4f80d912f25b6b3e1044282a2a79df8b/src/encoding/json/decode.go#L426 -func indirect(v reflect.Value, val super.Value) (BSUPUnmarshaler, reflect.Value) { +func indirect(v reflect.Value, val Value) (CustomUnmarshaler, reflect.Value) { // If v is a named type and is addressable, // start with its address, so that if the type has pointer methods, // we find them. @@ -863,7 +781,7 @@ func indirect(v reflect.Value, val super.Value) (BSUPUnmarshaler, reflect.Value) for v.Kind() == reflect.Pointer { if v.CanSet() && val.IsNull() { // If the reflect value can be set and the value is nil we want - // to store this pointer since if destination is not a super.Value the + // to store this pointer since if destination is not a Value the // pointer will be set to nil. nilptr = v } @@ -871,40 +789,40 @@ func indirect(v reflect.Value, val super.Value) (BSUPUnmarshaler, reflect.Value) v.Set(reflect.New(v.Type().Elem())) } if v.Type().NumMethod() > 0 && v.CanInterface() { - if u, ok := v.Interface().(BSUPUnmarshaler); ok { + if u, ok := v.Interface().(CustomUnmarshaler); ok { return u, reflect.Value{} } } v = v.Elem() } - if _, ok := v.Interface().(super.Value); !ok && nilptr.IsValid() { + if _, ok := v.Interface().(Value); !ok && nilptr.IsValid() { return nil, nilptr } return nil, v } -func (u *UnmarshalBSUPContext) decodeNetipAddr(val super.Value, v reflect.Value) error { - if super.TypeUnder(val.Type()) != super.TypeIP { +func (u *Unmarshaler) decodeNetipAddr(val Value, v reflect.Value) error { + if TypeUnder(val.Type()) != TypeIP { return incompatTypeError(val.Type(), v) } - v.Set(reflect.ValueOf(super.DecodeIP(val.Bytes()))) + v.Set(reflect.ValueOf(DecodeIP(val.Bytes()))) return nil } -func (u *UnmarshalBSUPContext) decodeNetIP(val super.Value, v reflect.Value) error { - if super.TypeUnder(val.Type()) != super.TypeIP { +func (u *Unmarshaler) decodeNetIP(val Value, v reflect.Value) error { + if TypeUnder(val.Type()) != TypeIP { return incompatTypeError(val.Type(), v) } - v.Set(reflect.ValueOf(net.ParseIP(super.DecodeIP(val.Bytes()).String()))) + v.Set(reflect.ValueOf(net.ParseIP(DecodeIP(val.Bytes()).String()))) return nil } -func (u *UnmarshalBSUPContext) decodeMap(val super.Value, mapVal reflect.Value) error { +func (u *Unmarshaler) decodeMap(val Value, mapVal reflect.Value) error { if val.IsNull() { mapVal.Set(reflect.Zero(mapVal.Type())) return nil } - typ, ok := super.TypeUnder(val.Type()).(*super.TypeMap) + typ, ok := TypeUnder(val.Type()).(*TypeMap) if !ok { return errors.New("not a map") } @@ -915,11 +833,11 @@ func (u *UnmarshalBSUPContext) decodeMap(val super.Value, mapVal reflect.Value) valType := mapVal.Type().Elem() for it := val.ContainerIter(); !it.Done(); { key := reflect.New(keyType).Elem() - if err := u.decodeAny(super.NewValue(typ.KeyType, it.Next()), key); err != nil { + if err := u.decodeAny(NewValue(typ.KeyType, it.Next()), key); err != nil { return err } val := reflect.New(valType).Elem() - if err := u.decodeAny(super.NewValue(typ.ValType, it.Next()), val); err != nil { + if err := u.decodeAny(NewValue(typ.ValType, it.Next()), val); err != nil { return err } mapVal.SetMapIndex(key, val) @@ -927,10 +845,10 @@ func (u *UnmarshalBSUPContext) decodeMap(val super.Value, mapVal reflect.Value) return nil } -func (u *UnmarshalBSUPContext) decodeRecord(val super.Value, sval reflect.Value) error { - recType, ok := super.TypeUnder(val.Type()).(*super.TypeRecord) +func (u *Unmarshaler) decodeRecord(val Value, sval reflect.Value) error { + recType, ok := TypeUnder(val.Type()).(*TypeRecord) if !ok { - return fmt.Errorf("cannot unmarshal value %q into Go struct", String(val)) + return fmt.Errorf("cannot unmarshal value into Go struct") } nameToField := make(map[string]int) stype := sval.Type() @@ -947,7 +865,7 @@ func (u *UnmarshalBSUPContext) decodeRecord(val super.Value, sval reflect.Value) name := recType.Fields[i].Name if fieldIdx, ok := nameToField[name]; ok { typ := recType.Fields[i].Type - if err := u.decodeAny(super.NewValue(typ, itzv), sval.Field(fieldIdx)); err != nil { + if err := u.decodeAny(NewValue(typ, itzv), sval.Field(fieldIdx)); err != nil { return err } } @@ -955,13 +873,13 @@ func (u *UnmarshalBSUPContext) decodeRecord(val super.Value, sval reflect.Value) return nil } -func (u *UnmarshalBSUPContext) decodeArray(val super.Value, arrVal reflect.Value) error { +func (u *Unmarshaler) decodeArray(val Value, arrVal reflect.Value) error { if val.IsNull() { arrVal.Set(reflect.Zero(arrVal.Type())) return nil } - typ := super.TypeUnder(val.Type()) - if typ == super.TypeBytes && arrVal.Type().Elem().Kind() == reflect.Uint8 { + typ := TypeUnder(val.Type()) + if typ == TypeBytes && arrVal.Type().Elem().Kind() == reflect.Uint8 { if arrVal.Kind() == reflect.Array { return u.decodeArrayBytes(val, arrVal) } @@ -969,9 +887,9 @@ func (u *UnmarshalBSUPContext) decodeArray(val super.Value, arrVal reflect.Value arrVal.SetBytes(val.Bytes()) return nil } - arrType, ok := typ.(*super.TypeArray) + arrType, ok := typ.(*TypeArray) if !ok { - return fmt.Errorf("unmarshaling type %q: not an array", String(typ)) + return fmt.Errorf("not an array") } i := 0 for it := val.ContainerIter(); !it.Done(); i++ { @@ -985,7 +903,7 @@ func (u *UnmarshalBSUPContext) decodeArray(val super.Value, arrVal reflect.Value if i >= arrVal.Len() { arrVal.SetLen(i + 1) } - if err := u.decodeAny(super.NewValue(arrType.Type, itzv), arrVal.Index(i)); err != nil { + if err := u.decodeAny(NewValue(arrType.Type, itzv), arrVal.Index(i)); err != nil { return err } } @@ -998,7 +916,7 @@ func (u *UnmarshalBSUPContext) decodeArray(val super.Value, arrVal reflect.Value return nil } -func (u *UnmarshalBSUPContext) decodeArrayBytes(val super.Value, arrayVal reflect.Value) error { +func (u *Unmarshaler) decodeArrayBytes(val Value, arrayVal reflect.Value) error { if len(val.Bytes()) != arrayVal.Len() { return errors.New("BSUP bytes value length differs from Go array") } @@ -1097,9 +1015,9 @@ func typeNameOfValue(value any) (string, error) { // This process requires // a value rather than a super type as it must determine the types of union elements // from their tags. -func (u *UnmarshalBSUPContext) lookupGoType(typ super.Type, bytes scode.Bytes) (reflect.Type, error) { +func (u *Unmarshaler) lookupGoType(typ Type, bytes scode.Bytes) (reflect.Type, error) { switch typ := typ.(type) { - case *super.TypeNamed: + case *TypeNamed: if template := u.binder.lookup(typ.Name); template != nil { return template, nil } @@ -1110,9 +1028,9 @@ func (u *UnmarshalBSUPContext) lookupGoType(typ super.Type, bytes scode.Bytes) ( // value and the concrete value doesn't implement the // interface. return u.lookupGoType(typ.Type, bytes) - case *super.TypeRecord: + case *TypeRecord: return nil, errors.New("unmarshaling records into interface value requires type binding") - case *super.TypeArray: + case *TypeArray: // If we got here, we know the array type wasn't named and // therefore cannot have mixed-type elements. So we don't need // to traverse the array and can just take the first element @@ -1140,7 +1058,7 @@ func (u *UnmarshalBSUPContext) lookupGoType(typ super.Type, bytes scode.Bytes) ( return nil, err } return reflect.SliceOf(elemType), nil - case *super.TypeSet: + case *TypeSet: // See comment above for TypeArray as it applies here. it := bytes.Iter() if it.Done() { @@ -1153,25 +1071,25 @@ func (u *UnmarshalBSUPContext) lookupGoType(typ super.Type, bytes scode.Bytes) ( return nil, err } return reflect.SliceOf(elemType), nil - case *super.TypeUnion: + case *TypeUnion: return u.lookupGoType(typ.Untag(bytes)) - case *super.TypeEnum: + case *TypeEnum: // For now just return nil here. The layer above will flag // a type error. At some point, we can create Go-native data structures // in package super for representing a union or enum as a standalone // entity. See issue #1853. return nil, nil - case *super.TypeMap: + case *TypeMap: it := bytes.Iter() if it.Done() { - return nil, fmt.Errorf("corrupt map value in super unmarshal: type %q", String(typ)) + return nil, fmt.Errorf("corrupt map value in super unmarshal") } keyType, err := u.lookupGoType(typ.KeyType, it.Next()) if err != nil { return nil, err } if it.Done() { - return nil, fmt.Errorf("corrupt map value in super unmarshal: type %q", String(typ)) + return nil, fmt.Errorf("corrupt map value in super unmarshal") } valType, err := u.lookupGoType(typ.ValType, it.Next()) if err != nil { @@ -1183,47 +1101,47 @@ func (u *UnmarshalBSUPContext) lookupGoType(typ super.Type, bytes scode.Bytes) ( } } -func (u *UnmarshalBSUPContext) lookupPrimitiveType(typ super.Type) (reflect.Type, error) { +func (u *Unmarshaler) lookupPrimitiveType(typ Type) (reflect.Type, error) { var v any switch typ := typ.(type) { // XXX We should have counterparts for error and type type. // See issue #1853. // XXX udpate issue? - case *super.TypeOfString, *super.TypeOfType: + case *TypeOfString, *TypeOfType: v = "" - case *super.TypeOfBool: + case *TypeOfBool: v = false - case *super.TypeOfUint8: + case *TypeOfUint8: v = uint8(0) - case *super.TypeOfUint16: + case *TypeOfUint16: v = uint16(0) - case *super.TypeOfUint32: + case *TypeOfUint32: v = uint32(0) - case *super.TypeOfUint64: + case *TypeOfUint64: v = uint64(0) - case *super.TypeOfInt8: + case *TypeOfInt8: v = int8(0) - case *super.TypeOfInt16: + case *TypeOfInt16: v = int16(0) - case *super.TypeOfInt32: + case *TypeOfInt32: v = int32(0) - case *super.TypeOfInt64: + case *TypeOfInt64: v = int64(0) - case *super.TypeOfFloat16: + case *TypeOfFloat16: v = float16.Fromfloat32(0) - case *super.TypeOfFloat32: + case *TypeOfFloat32: v = float32(0) - case *super.TypeOfFloat64: + case *TypeOfFloat64: v = float64(0) - case *super.TypeOfIP: + case *TypeOfIP: v = netip.Addr{} - case *super.TypeOfNet: + case *TypeOfNet: v = net.IPNet{} - case *super.TypeOfTime: + case *TypeOfTime: v = time.Time{} - case *super.TypeOfDuration: + case *TypeOfDuration: v = time.Duration(0) - case *super.TypeOfNull: + case *TypeOfNull: return nil, nil default: return nil, fmt.Errorf("unknown Super type: %v", typ) diff --git a/order/direction.go b/order/direction.go index 27ef657c7a..d58c86d417 100644 --- a/order/direction.go +++ b/order/direction.go @@ -6,11 +6,13 @@ import ( "strings" "github.com/brimdata/super" - "github.com/brimdata/super/sup" ) type Direction int +var _ super.CustomMarshaler = (*Direction)(nil) +var _ super.CustomUnmarshaler = (*Direction)(nil) + const ( Down Direction = -1 Up Direction = 1 @@ -83,11 +85,11 @@ func (d Direction) MarshalJSON() ([]byte, error) { return json.Marshal(d.String()) } -func (d Direction) MarshalBSUP(m *sup.MarshalBSUPContext) (super.Type, error) { +func (d Direction) Marshal(m *super.Marshaler) (super.Type, error) { return m.MarshalValue(d.String()) } -func (d *Direction) UnmarshalBSUP(u *sup.UnmarshalBSUPContext, val super.Value) error { +func (d *Direction) Unmarshal(u *super.Unmarshaler, val super.Value) error { dir, err := ParseDirection(string(val.Bytes())) if err != nil { return err diff --git a/order/nulls.go b/order/nulls.go index 9fa2b42eaa..7f0f75dd05 100644 --- a/order/nulls.go +++ b/order/nulls.go @@ -11,6 +11,9 @@ import ( // Nulls represents the position of nulls in an ordering of values. type Nulls bool +var _ super.CustomMarshaler = (*Nulls)(nil) +var _ super.CustomUnmarshaler = (*Nulls)(nil) + const ( NullsLast Nulls = false NullsFirst Nulls = true @@ -39,11 +42,11 @@ func (n *Nulls) UnmarshalText(b []byte) error { return nil } -func (n Nulls) MarshalBSUP(m *sup.MarshalBSUPContext) (super.Type, error) { +func (n Nulls) Marshal(m *super.Marshaler) (super.Type, error) { return m.MarshalValue(n.String()) } -func (n *Nulls) UnmarshalBSUP(u *sup.UnmarshalBSUPContext, val super.Value) error { +func (n *Nulls) Unmarshal(u *super.Unmarshaler, val super.Value) error { if val.Type().ID() != super.IDString { return fmt.Errorf("cannot unmarshal %q into order.Nulls", sup.FormatValue(val)) } diff --git a/pkg/storage/uri.go b/pkg/storage/uri.go index 9883d6b34e..3a323ae777 100644 --- a/pkg/storage/uri.go +++ b/pkg/storage/uri.go @@ -5,11 +5,12 @@ import ( "strings" "github.com/brimdata/super" - "github.com/brimdata/super/sup" ) type URI url.URL +var _ super.CustomMarshaler = (*URI)(nil) + // ParseURI parses the path using `url.Parse`. If the provided uri does not // contain a scheme, the scheme is set to file. Relative paths are // treated as files and resolved as absolute paths using filepath.Abs. @@ -69,6 +70,6 @@ func (u *URI) UnmarshalText(b []byte) error { return nil } -func (u *URI) MarshalBSUP(mc *sup.MarshalBSUPContext) (super.Type, error) { +func (u *URI) Marshal(mc *super.Marshaler) (super.Type, error) { return mc.MarshalValue(u.String()) } diff --git a/runtime/sam/expr/function/parse.go b/runtime/sam/expr/function/parse.go index 614d043818..30b22a7493 100644 --- a/runtime/sam/expr/function/parse.go +++ b/runtime/sam/expr/function/parse.go @@ -6,16 +6,15 @@ import ( "strconv" "github.com/brimdata/super" - "github.com/brimdata/super/sup" ) type ParseURI struct { sctx *super.Context - marshaler *sup.MarshalBSUPContext + marshaler *super.Marshaler } func NewParseURI(sctx *super.Context) *ParseURI { - return &ParseURI{sctx, sup.NewBSUPMarshalerWithContext(sctx)} + return &ParseURI{sctx, super.NewMarshaler(sctx)} } func (p *ParseURI) Call(args []super.Value) super.Value { diff --git a/runtime/sam/op/meta/deleter.go b/runtime/sam/op/meta/deleter.go index 2a7910f9f9..f2286b32eb 100644 --- a/runtime/sam/op/meta/deleter.go +++ b/runtime/sam/op/meta/deleter.go @@ -9,7 +9,6 @@ import ( "github.com/brimdata/super/runtime" "github.com/brimdata/super/runtime/sam/expr" "github.com/brimdata/super/sbuf" - "github.com/brimdata/super/sup" "github.com/brimdata/super/vector/vio" "github.com/segmentio/ksuid" ) @@ -22,7 +21,7 @@ type Deleter struct { rctx *runtime.Context pool *db.Pool progress *vio.Progress - unmarshaler *sup.UnmarshalBSUPContext + unmarshaler *super.Unmarshaler done bool err error deletes *sync.Map @@ -36,7 +35,7 @@ func NewDeleter(rctx *runtime.Context, parent sbuf.Puller, pool *db.Pool, pushdo rctx: rctx, pool: pool, progress: progress, - unmarshaler: sup.NewBSUPUnmarshaler(), + unmarshaler: super.NewUnmarshaler(), deletes: deletes, } } diff --git a/runtime/sam/op/meta/lister.go b/runtime/sam/op/meta/lister.go index 82d8d323b4..9a10c3a519 100644 --- a/runtime/sam/op/meta/lister.go +++ b/runtime/sam/op/meta/lister.go @@ -13,7 +13,6 @@ import ( "github.com/brimdata/super/order" "github.com/brimdata/super/runtime/sam/expr" "github.com/brimdata/super/sbuf" - "github.com/brimdata/super/sup" "github.com/segmentio/ksuid" "golang.org/x/sync/errgroup" ) @@ -28,7 +27,7 @@ type Lister struct { snap commits.View pruner *pruner group *errgroup.Group - marshaler *sup.MarshalBSUPContext + marshaler *super.Marshaler mu sync.Mutex objects []*data.Object err error @@ -45,8 +44,8 @@ func NewSortedLister(ctx context.Context, sctx *super.Context, pool *db.Pool, co } func NewSortedListerFromSnap(ctx context.Context, sctx *super.Context, pool *db.Pool, snap commits.View, pruner expr.Evaluator) *Lister { - m := sup.NewBSUPMarshalerWithContext(sctx) - m.Decorate(sup.StylePackage) + m := super.NewMarshaler(sctx) + m.Decorate(super.StylePackage) l := &Lister{ ctx: ctx, pool: pool, diff --git a/runtime/sam/op/meta/scanner.go b/runtime/sam/op/meta/scanner.go index 05d4adaca4..27a299f194 100644 --- a/runtime/sam/op/meta/scanner.go +++ b/runtime/sam/op/meta/scanner.go @@ -11,7 +11,6 @@ import ( "github.com/brimdata/super/runtime/sam/expr" "github.com/brimdata/super/sbuf" "github.com/brimdata/super/sio" - "github.com/brimdata/super/sup" "github.com/segmentio/ksuid" ) @@ -40,8 +39,8 @@ func NewPoolMetaScanner(ctx context.Context, sctx *super.Context, r *db.Root, po var vals []super.Value switch meta { case "branches": - m := sup.NewBSUPMarshalerWithContext(sctx) - m.Decorate(sup.StylePackage) + m := super.NewMarshaler(sctx) + m.Decorate(super.StylePackage) vals, err = p.BatchifyBranches(ctx, sctx, nil, m, nil) if err != nil { return nil, err @@ -113,8 +112,8 @@ func NewCommitMetaScanner(ctx context.Context, sctx *super.Context, r *db.Root, func objectReader(sctx *super.Context, snap commits.View, order order.Which) (sio.Reader, error) { objects := snap.Select(nil, order) - m := sup.NewBSUPMarshalerWithContext(sctx) - m.Decorate(sup.StylePackage) + m := super.NewMarshaler(sctx) + m.Decorate(super.StylePackage) return readerFunc(func() (*super.Value, error) { if len(objects) == 0 { return nil, nil diff --git a/runtime/sam/op/meta/sequence.go b/runtime/sam/op/meta/sequence.go index 1d45e90a6d..0fcde84d7a 100644 --- a/runtime/sam/op/meta/sequence.go +++ b/runtime/sam/op/meta/sequence.go @@ -13,7 +13,6 @@ import ( "github.com/brimdata/super/runtime/sam/op/merge" "github.com/brimdata/super/sbuf" "github.com/brimdata/super/sio/bsupio" - "github.com/brimdata/super/sup" "github.com/brimdata/super/vector" "github.com/brimdata/super/vector/vio" ) @@ -28,7 +27,7 @@ type SequenceScanner struct { rctx *runtime.Context pool *db.Pool progress *vio.Progress - unmarshaler *sup.UnmarshalBSUPContext + unmarshaler *super.Unmarshaler done bool err error } @@ -41,7 +40,7 @@ func NewSequenceScanner(rctx *runtime.Context, parent sbuf.Puller, pool *db.Pool pruner: pruner, pool: pool, progress: progress, - unmarshaler: sup.NewBSUPUnmarshaler(), + unmarshaler: super.NewUnmarshaler(), } } @@ -148,7 +147,7 @@ func (s *SearchScanner) Pull(done bool) (sbuf.Batch, error) { } } -func newScanner(ctx context.Context, sctx *super.Context, pool *db.Pool, u *sup.UnmarshalBSUPContext, pruner expr.Evaluator, pushdown sbuf.Pushdown, progress *vio.Progress, val super.Value) (sbuf.Puller, *data.Object, error) { +func newScanner(ctx context.Context, sctx *super.Context, pool *db.Pool, u *super.Unmarshaler, pruner expr.Evaluator, pushdown sbuf.Pushdown, progress *vio.Progress, val super.Value) (sbuf.Puller, *data.Object, error) { named, ok := val.Type().(*super.TypeNamed) if !ok { return nil, nil, errors.New("system error: SequenceScanner encountered unnamed object") diff --git a/runtime/sam/op/meta/slicer.go b/runtime/sam/op/meta/slicer.go index b19b33692a..efc0b75f9f 100644 --- a/runtime/sam/op/meta/slicer.go +++ b/runtime/sam/op/meta/slicer.go @@ -19,8 +19,8 @@ import ( // non-overlapping Partitions. type Slicer struct { parent sbuf.Puller - marshaler *sup.MarshalBSUPContext - unmarshaler *sup.UnmarshalBSUPContext + marshaler *super.Marshaler + unmarshaler *super.Unmarshaler objects []*data.Object cmp expr.CompareFn min *super.Value @@ -29,12 +29,12 @@ type Slicer struct { } func NewSlicer(parent sbuf.Puller, sctx *super.Context) *Slicer { - m := sup.NewBSUPMarshalerWithContext(sctx) - m.Decorate(sup.StylePackage) + m := super.NewMarshaler(sctx) + m.Decorate(super.StylePackage) return &Slicer{ parent: parent, marshaler: m, - unmarshaler: sup.NewBSUPUnmarshaler(), + unmarshaler: super.NewUnmarshaler(), //XXX check that nulls position is consistent for both dirs in database ops cmp: expr.NewValueCompareFn(order.Asc, order.NullsLast), } diff --git a/runtime/vam/op/scan.go b/runtime/vam/op/scan.go index df0e87252a..ed15bb1cdd 100644 --- a/runtime/vam/op/scan.go +++ b/runtime/vam/op/scan.go @@ -124,13 +124,13 @@ type result struct { type objectPuller struct { parent sbuf.Puller - unmarshaler *sup.UnmarshalBSUPContext + unmarshaler *super.Unmarshaler } func newObjectPuller(parent sbuf.Puller) *objectPuller { return &objectPuller{ parent: parent, - unmarshaler: sup.NewBSUPUnmarshaler(), + unmarshaler: super.NewUnmarshaler(), } } diff --git a/service/client_test.go b/service/client_test.go index a249bae7c2..a47354a66b 100644 --- a/service/client_test.go +++ b/service/client_test.go @@ -17,7 +17,6 @@ import ( "github.com/brimdata/super/sio" "github.com/brimdata/super/sio/bsupio" "github.com/brimdata/super/sio/supio" - "github.com/brimdata/super/sup" "github.com/segmentio/ksuid" "github.com/stretchr/testify/require" ) @@ -61,7 +60,7 @@ func (c *testClient) TestPoolList() []pools.Config { return confs } var pool pools.Config - err = sup.UnmarshalBSUP(*rec, &pool) + err = super.Unmarshal(*rec, &pool) require.NoError(c, err) confs = append(confs, pool) } diff --git a/service/core.go b/service/core.go index 32db9410d5..5f147deb29 100644 --- a/service/core.go +++ b/service/core.go @@ -12,12 +12,12 @@ import ( "sync" "time" + "github.com/brimdata/super" "github.com/brimdata/super/api" "github.com/brimdata/super/compiler" "github.com/brimdata/super/db" "github.com/brimdata/super/pkg/storage" "github.com/brimdata/super/runtime" - "github.com/brimdata/super/sup" "github.com/gorilla/mux" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/collectors" @@ -221,8 +221,8 @@ func (c *Core) ServeHTTP(w http.ResponseWriter, r *http.Request) { } func (c *Core) publishEvent(w *ResponseWriter, name string, data any) { - marshaler := sup.NewBSUPMarshaler() - marshaler.Decorate(sup.StyleSimple) + marshaler := super.NewMarshaler(super.NewContext()) + marshaler.Decorate(super.StyleSimple) zv, err := marshaler.Marshal(data) if err != nil { w.Logger.Error("Error marshaling published event", zap.Error(err)) diff --git a/service/request.go b/service/request.go index ced927d08f..02b6deb71a 100644 --- a/service/request.go +++ b/service/request.go @@ -24,7 +24,6 @@ import ( "github.com/brimdata/super/service/srverr" "github.com/brimdata/super/sio" "github.com/brimdata/super/sio/anyio" - "github.com/brimdata/super/sup" "github.com/brimdata/super/vector/vio" "github.com/gorilla/mux" "github.com/segmentio/ksuid" @@ -39,8 +38,8 @@ type Request struct { func newRequest(w http.ResponseWriter, r *http.Request, c *Core) (*ResponseWriter, *Request, bool) { req := &Request{Request: r} req.Logger = c.logger.With(zap.String("request_id", req.ID())) - m := sup.NewBSUPMarshaler() - m.Decorate(sup.StylePackage) + m := super.NewMarshaler(super.NewContext()) + m.Decorate(super.StylePackage) res := &ResponseWriter{ ResponseWriter: w, Logger: req.Logger, @@ -190,7 +189,7 @@ func (r *Request) Unmarshal(w *ResponseWriter, body any, templates ...any) bool if zv == nil { return true } - m := sup.NewBSUPUnmarshaler() + m := super.NewUnmarshaler() m.Bind(templates...) if err := m.Unmarshal(*zv, body); err != nil { w.Error(srverr.ErrInvalid(err)) @@ -220,7 +219,7 @@ type ResponseWriter struct { Format string Logger *zap.Logger zw vio.PushCloser - marshaler *sup.MarshalBSUPContext + marshaler *super.Marshaler request *Request written atomic.Int32 } diff --git a/sio/csvio/reader.go b/sio/csvio/reader.go index 2d8e898a8b..a52863efeb 100644 --- a/sio/csvio/reader.go +++ b/sio/csvio/reader.go @@ -10,12 +10,11 @@ import ( "unicode/utf8" "github.com/brimdata/super" - "github.com/brimdata/super/sup" ) type Reader struct { reader *csv.Reader - marshaler *sup.MarshalBSUPContext + marshaler *super.Marshaler strings bool valid bool hdr []string @@ -50,7 +49,7 @@ func NewReader(sctx *super.Context, r io.Reader, opts ReaderOpts) *Reader { reader.ReuseRecord = true return &Reader{ reader: reader, - marshaler: sup.NewBSUPMarshalerWithContext(sctx), + marshaler: super.NewMarshaler(sctx), } } diff --git a/sio/dbio/unmarshal.go b/sio/dbio/unmarshal.go index a796cf32b7..4fb4f4ba54 100644 --- a/sio/dbio/unmarshal.go +++ b/sio/dbio/unmarshal.go @@ -1,19 +1,19 @@ package dbio import ( + "github.com/brimdata/super" "github.com/brimdata/super/db" "github.com/brimdata/super/db/commits" "github.com/brimdata/super/db/data" "github.com/brimdata/super/db/pools" "github.com/brimdata/super/pkg/field" "github.com/brimdata/super/runtime/sam/op/meta" - "github.com/brimdata/super/sup" ) -var unmarshaler *sup.UnmarshalBSUPContext +var unmarshaler *super.Unmarshaler func init() { - unmarshaler = sup.NewBSUPUnmarshaler() + unmarshaler = super.NewUnmarshaler() unmarshaler.Bind( commits.Add{}, commits.Commit{}, diff --git a/sio/parquetio/metadata.go b/sio/parquetio/metadata.go index 5a8da4bbc1..3f1ea4bd62 100644 --- a/sio/parquetio/metadata.go +++ b/sio/parquetio/metadata.go @@ -10,14 +10,13 @@ import ( "github.com/apache/arrow-go/v18/parquet/pqarrow" "github.com/brimdata/super" "github.com/brimdata/super/pkg/field" - "github.com/brimdata/super/sup" "github.com/x448/float16" ) func buildMetadataValue(sctx *super.Context, rgmd *metadata.RowGroupMetaData, colIndexes []int, colIndexToField map[int]*pqarrow.SchemaField) super.Value { var paths field.List var vals []super.Value - m := sup.NewBSUPMarshaler() + m := super.NewMarshaler(sctx) for _, i := range colIndexes { min, max, path, ok := columnChunkStats(rgmd, i, colIndexToField[i].Field.Type) if !ok { diff --git a/sup/marshal_bsup_test.go b/sup/marshal_bsup_test.go index 7b7323d4fe..190271acd1 100644 --- a/sup/marshal_bsup_test.go +++ b/sup/marshal_bsup_test.go @@ -19,19 +19,19 @@ import ( ) func boomerang(t *testing.T, in any, out any) { - rec, err := sup.NewBSUPMarshaler().Marshal(in) + sctx := super.NewContext() + rec, err := super.NewMarshaler(sctx).Marshal(in) require.NoError(t, err) var buf bytes.Buffer zw := bsupio.NewRowWriter(sio.NopCloser(&buf)) err = zw.Write(rec) require.NoError(t, err) require.NoError(t, zw.Close()) - sctx := super.NewContext() zr := bsupio.NewRowReader(sctx, &buf) defer zr.Close() val, err := zr.Read() require.NoError(t, err) - err = sup.UnmarshalBSUP(*val, out) + err = super.Unmarshal(*val, out) require.NoError(t, err) } @@ -45,7 +45,7 @@ func TestMarshalBSUP(t *testing.T) { Sub1 S2 PField1 *bool } - rec, err := sup.NewBSUPMarshaler().Marshal(S1{ + rec, err := super.NewMarshaler(super.NewContext()).Marshal(S1{ Field1: "value1", Sub1: S2{ Field2: "value2", @@ -87,8 +87,8 @@ type BSUPThings struct { func TestMarshalSlice(t *testing.T) { t.Skip() // skipping until we fix marshal to use named types for interfaces - m := sup.NewBSUPMarshaler() - m.Decorate(sup.StyleSimple) + m := super.NewMarshaler(super.NewContext()) + m.Decorate(super.StyleSimple) s := []BSUPThing{{"hello", 123}, {"world", 0}} r := BSUPThings{s} @@ -151,14 +151,14 @@ type TestIP struct { func TestIPType(t *testing.T) { s := TestIP{Addr: netip.MustParseAddr("192.168.1.1")} sctx := super.NewContext() - m := sup.NewBSUPMarshalerWithContext(sctx) + m := super.NewMarshaler(sctx) rec, err := m.Marshal(s) require.NoError(t, err) require.NotNil(t, rec) assert.Equal(t, "{Addr:192.168.1.1}", sup.FormatValue(rec)) var tip TestIP - err = sup.UnmarshalBSUP(rec, &tip) + err = super.Unmarshal(rec, &tip) require.NoError(t, err) require.Equal(t, s, tip) } @@ -178,16 +178,17 @@ func TestUnmarshalRecord(t *testing.T) { v1 := T1{ T1f1: &T2{T2f1: T3{T3f1: 1, T3f2: 1.0}, T2f2: "t2f2-string1"}, } - rec, err := sup.NewBSUPMarshaler().Marshal(v1) + sctx := super.NewContext() + rec, err := super.NewMarshaler(sctx).Marshal(v1) require.NoError(t, err) require.NotNil(t, rec) const expected = `{top:{T2f1:{T3f1:1::int32,T3f2:1.::float32},T2f2:"t2f2-string1"}}` require.Equal(t, expected, sup.FormatValue(rec)) - val := sup.MustParseValue(super.NewContext(), expected) + val := sup.MustParseValue(sctx, expected) var v2 T1 - err = sup.UnmarshalBSUP(val, &v2) + err = super.Unmarshal(val, &v2) require.NoError(t, err) require.Equal(t, v1, v2) @@ -195,7 +196,7 @@ func TestUnmarshalRecord(t *testing.T) { T4f1 *T2 `super:"top"` } var v3 *T4 - err = sup.UnmarshalBSUP(rec, &v3) + err = super.Unmarshal(rec, &v3) require.NoError(t, err) require.NotNil(t, v3) require.NotNil(t, v3.T4f1) @@ -205,34 +206,34 @@ func TestUnmarshalRecord(t *testing.T) { func TestUnmarshalNull(t *testing.T) { t.Run("slice", func(t *testing.T) { slice := []int{1} - require.NoError(t, sup.UnmarshalBSUP(super.Null, &slice)) + require.NoError(t, super.Unmarshal(super.Null, &slice)) assert.Nil(t, slice) slice = []int{1} val := sup.MustParseValue(super.NewContext(), "null::(null|[int64])") - require.NoError(t, sup.UnmarshalBSUP(val, &slice)) + require.NoError(t, super.Unmarshal(val, &slice)) assert.Nil(t, slice) buf := []byte("testing") - require.NoError(t, sup.UnmarshalBSUP(super.Null, &buf)) + require.NoError(t, super.Unmarshal(super.Null, &buf)) assert.Nil(t, buf) buf = []byte("testing") val = sup.MustParseValue(super.NewContext(), "null::(null|bytes)") - require.NoError(t, sup.UnmarshalBSUP(val, &buf)) + require.NoError(t, super.Unmarshal(val, &buf)) assert.Nil(t, buf) }) t.Run("primitive", func(t *testing.T) { integer := -1 - assert.EqualError(t, sup.UnmarshalBSUP(super.Null, &integer), "incompatible type translation: Super type null, Go type int, Go kind int") + assert.EqualError(t, super.Unmarshal(super.Null, &integer), "incompatible type translation: Super type null, Go type int, Go kind int") intptr := &integer - assert.NoError(t, sup.UnmarshalBSUP(super.Null, &intptr)) + assert.NoError(t, super.Unmarshal(super.Null, &intptr)) assert.Nil(t, intptr) }) t.Run("map", func(t *testing.T) { m := map[string]string{"key": "value"} - require.NoError(t, sup.UnmarshalBSUP(super.Null, &m)) + require.NoError(t, super.Unmarshal(super.Null, &m)) assert.Nil(t, m) m = map[string]string{"key": "value"} val := sup.MustParseValue(super.NewContext(), "null::(null|map{string:string})") - require.NoError(t, sup.UnmarshalBSUP(val, &m)) + require.NoError(t, super.Unmarshal(val, &m)) assert.Nil(t, m) }) t.Run("struct", func(t *testing.T) { @@ -243,14 +244,14 @@ func TestUnmarshalNull(t *testing.T) { Test *testobj `super:"test"` } val := sup.MustParseValue(super.NewContext(), "{test:null::(null|{Val:int64})}") - require.NoError(t, sup.UnmarshalBSUP(val, &obj)) + require.NoError(t, super.Unmarshal(val, &obj)) require.Nil(t, obj.Test) var slice struct { Test []string `super:"test"` } slice.Test = []string{"1"} val = sup.MustParseValue(super.NewContext(), "{test:null}") - require.NoError(t, sup.UnmarshalBSUP(val, &slice)) + require.NoError(t, super.Unmarshal(val, &slice)) require.Nil(t, slice.Test) }) } @@ -263,12 +264,12 @@ func TestUnmarshalSlice(t *testing.T) { T1f1: []bool{true, false, true}, } sctx := super.NewContext() - rec, err := sup.NewBSUPMarshalerWithContext(sctx).Marshal(v1) + rec, err := super.NewMarshaler(sctx).Marshal(v1) require.NoError(t, err) require.NotNil(t, rec) var v2 T1 - err = sup.UnmarshalBSUP(rec, &v2) + err = super.Unmarshal(rec, &v2) require.NoError(t, err) require.Equal(t, v1, v2) @@ -280,23 +281,23 @@ func TestUnmarshalSlice(t *testing.T) { Field1: []*int{intp(1), intp(2)}, } sctx = super.NewContext() - rec, err = sup.NewBSUPMarshalerWithContext(sctx).Marshal(v3) + rec, err = super.NewMarshaler(sctx).Marshal(v3) require.NoError(t, err) require.NotNil(t, rec) var v4 T2 - err = sup.UnmarshalBSUP(rec, &v4) + err = super.Unmarshal(rec, &v4) require.NoError(t, err) require.Equal(t, v1, v2) } type testMarshaler string -func (m testMarshaler) MarshalBSUP(mc *sup.MarshalBSUPContext) (super.Type, error) { +func (m testMarshaler) Marshal(mc *super.Marshaler) (super.Type, error) { return mc.MarshalValue("marshal-" + string(m)) } -func (m *testMarshaler) UnmarshalBSUP(mc *sup.UnmarshalBSUPContext, val super.Value) error { +func (m *testMarshaler) Unmarshal(mc *super.Unmarshaler, val super.Value) error { var s string if err := mc.Unmarshal(val, &s); err != nil { return err @@ -316,13 +317,13 @@ func TestMarshalInterface(t *testing.T) { } m1 := testMarshaler("m1") r1 := rectype{M1: &m1, M2: testMarshaler("m2")} - rec, err := sup.NewBSUPMarshaler().Marshal(r1) + rec, err := super.NewMarshaler(super.NewContext()).Marshal(r1) require.NoError(t, err) require.NotNil(t, rec) assert.Equal(t, `{M1:"marshal-m1",M2:"marshal-m2"}`, sup.FormatValue(rec)) var r2 rectype - err = sup.UnmarshalBSUP(rec, &r2) + err = super.Unmarshal(rec, &r2) require.NoError(t, err) assert.Equal(t, "m1", string(*r2.M1)) assert.Equal(t, "m2", string(r2.M2)) @@ -336,14 +337,14 @@ func TestMarshalArray(t *testing.T) { } a2 := &[2]string{"foo", "bar"} r1 := rectype{A1: [2]int8{1, 2}, A2: a2} // A3 left as nil - rec, err := sup.NewBSUPMarshaler().Marshal(r1) + rec, err := super.NewMarshaler(super.NewContext()).Marshal(r1) require.NoError(t, err) require.NotNil(t, rec) const expected = `{A1:[1::int8,2::int8],A2:["foo","bar"],A3:[]::[bytes]}` assert.Equal(t, expected, sup.FormatValue(rec)) var r2 rectype - err = sup.UnmarshalBSUP(rec, &r2) + err = super.Unmarshal(rec, &r2) require.NoError(t, err) assert.Equal(t, r1.A1, r2.A1) assert.Equal(t, *r2.A2, *r2.A2) @@ -381,14 +382,14 @@ func TestNumbers(t *testing.T) { F32: math.MaxFloat32, F64: math.MaxFloat64, } - rec, err := sup.NewBSUPMarshaler().Marshal(r1) + rec, err := super.NewMarshaler(super.NewContext()).Marshal(r1) require.NoError(t, err) require.NotNil(t, rec) const expected = "{I:-9223372036854775808,I8:-128::int8,I16:-32768::int16,I32:-2147483648::int32,I64:-9223372036854775808,U:18446744073709551615::uint64,UI8:255::uint8,UI16:65535::uint16,UI32:4294967295::uint32,UI64:18446744073709551615::uint64,F16:65504.::float16,F32:3.4028235e+38::float32,F64:1.7976931348623157e+308}" assert.Equal(t, expected, sup.FormatValue(rec)) var r2 rectype - err = sup.UnmarshalBSUP(rec, &r2) + err = super.Unmarshal(rec, &r2) require.NoError(t, err) assert.Equal(t, r1, r2) } @@ -398,7 +399,7 @@ func TestCustomRecord(t *testing.T) { BSUPThing{"hello", 123}, 99, } - m := sup.NewBSUPMarshaler() + m := super.NewMarshaler(super.NewContext()) rec, err := m.MarshalCustom([]string{"foo", "bar"}, vals) require.NoError(t, err) assert.Equal(t, `{foo:{a:"hello",B:123},bar:99}`, sup.FormatValue(rec)) @@ -437,19 +438,19 @@ type Rolls []int func TestInterfaceBSUPMarshal(t *testing.T) { t1 := Make(2) - m := sup.NewBSUPMarshaler() - m.Decorate(sup.StylePackage) + m := super.NewMarshaler(super.NewContext()) + m.Decorate(super.StylePackage) zv, err := m.Marshal(t1) require.NoError(t, err) assert.Equal(t, `"sup_test.ThingTwo"`, sup.String(zv.Type())) - m.Decorate(sup.StyleSimple) + m.Decorate(super.StyleSimple) rolls := Rolls{1, 2, 3} zv, err = m.Marshal(rolls) require.NoError(t, err) assert.Equal(t, "Rolls", sup.String(zv.Type())) - m.Decorate(sup.StyleFull) + m.Decorate(super.StyleFull) zv, err = m.Marshal(rolls) require.NoError(t, err) assert.Equal(t, `"github.com/brimdata/super/sup_test.Rolls"`, sup.String(zv.Type())) @@ -462,13 +463,13 @@ func TestInterfaceBSUPMarshal(t *testing.T) { func TestInterfaceUnmarshal(t *testing.T) { t1 := Make(1) - m := sup.NewBSUPMarshaler() - m.Decorate(sup.StylePackage) + m := super.NewMarshaler(super.NewContext()) + m.Decorate(super.StylePackage) zv, err := m.Marshal(t1) require.NoError(t, err) assert.Equal(t, `"sup_test.BSUPThing"`, sup.String(zv.Type())) - u := sup.NewBSUPUnmarshaler() + u := super.NewUnmarshaler() u.Bind(BSUPThing{}, ThingTwo{}) var thing ThingaMaBob require.NoError(t, err) @@ -483,7 +484,7 @@ func TestInterfaceUnmarshal(t *testing.T) { assert.Equal(t, true, ok) assert.Equal(t, t1, actualThing) - u2 := sup.NewBSUPUnmarshaler() + u2 := super.NewUnmarshaler() var genericThing any err = u2.Unmarshal(zv, &genericThing) require.Error(t, err) @@ -492,19 +493,19 @@ func TestInterfaceUnmarshal(t *testing.T) { func TestBindings(t *testing.T) { t1 := Make(1) - m := sup.NewBSUPMarshaler() - m.NamedBindings([]sup.Binding{ - {"SpecialThingOne", &BSUPThing{}}, - {"SpecialThingTwo", &ThingTwo{}}, + m := super.NewMarshaler(super.NewContext()) + m.NamedBindings([]super.Binding{ + {Name: "SpecialThingOne", Template: &BSUPThing{}}, + {Name: "SpecialThingTwo", Template: &ThingTwo{}}, }) zv, err := m.Marshal(t1) require.NoError(t, err) assert.Equal(t, "SpecialThingOne", sup.String(zv.Type())) - u := sup.NewBSUPUnmarshaler() - u.NamedBindings([]sup.Binding{ - {"SpecialThingOne", &BSUPThing{}}, - {"SpecialThingTwo", &ThingTwo{}}, + u := super.NewUnmarshaler() + u.NamedBindings([]super.Binding{ + {Name: "SpecialThingOne", Template: &BSUPThing{}}, + {Name: "SpecialThingTwo", Template: &ThingTwo{}}, }) var thing ThingaMaBob require.NoError(t, err) @@ -514,19 +515,19 @@ func TestBindings(t *testing.T) { } func TestEmptyInterface(t *testing.T) { - zv, err := sup.MarshalBSUP(int8(123)) + zv, err := super.Marshal(super.NewContext(), int8(123)) require.NoError(t, err) assert.Equal(t, "int8", sup.String(zv.Type())) var v any - err = sup.UnmarshalBSUP(zv, &v) + err = super.Unmarshal(zv, &v) require.NoError(t, err) i, ok := v.(int8) assert.Equal(t, true, ok) assert.Equal(t, int8(123), i) var actual int8 - err = sup.UnmarshalBSUP(zv, &actual) + err = super.Unmarshal(zv, &actual) require.NoError(t, err) assert.Equal(t, int8(123), actual) } @@ -535,15 +536,15 @@ type CustomInt8 int8 func TestNamedNormal(t *testing.T) { t1 := CustomInt8(88) - m := sup.NewBSUPMarshaler() - m.Decorate(sup.StyleSimple) + m := super.NewMarshaler(super.NewContext()) + m.Decorate(super.StyleSimple) zv, err := m.Marshal(t1) require.NoError(t, err) assert.Equal(t, "CustomInt8", sup.String(zv.Type())) var actual CustomInt8 - u := sup.NewBSUPUnmarshaler() + u := super.NewUnmarshaler() u.Bind(CustomInt8(0)) err = u.Unmarshal(zv, &actual) require.NoError(t, err) @@ -569,13 +570,13 @@ func TestEmbeddedInterface(t *testing.T) { t1 := &EmbeddedA{ A: Make(1), } - m := sup.NewBSUPMarshaler() - m.Decorate(sup.StyleSimple) + m := super.NewMarshaler(super.NewContext()) + m.Decorate(super.StyleSimple) zv, err := m.Marshal(t1) require.NoError(t, err) assert.Equal(t, "EmbeddedA", sup.String(zv.Type())) - u := sup.NewBSUPUnmarshaler() + u := super.NewUnmarshaler() u.Bind(BSUPThing{}, ThingTwo{}) var actual EmbeddedA require.NoError(t, err) @@ -595,7 +596,7 @@ func TestEmbeddedInterface(t *testing.T) { func TestMultipleSuperValues(t *testing.T) { t.Skip() bytes := []byte("foo") - u := sup.NewBSUPUnmarshaler() + u := super.NewUnmarshaler() var foo super.Value err := u.Unmarshal(super.NewValue(super.TypeString, bytes), &foo) require.NoError(t, err) @@ -613,9 +614,9 @@ func TestSuperValues(t *testing.T) { test := func(t *testing.T, name, s string, v any) { t.Run(name, func(t *testing.T) { val := sup.MustParseValue(super.NewContext(), s) - err := sup.UnmarshalBSUP(val, v) + err := super.Unmarshal(val, v) require.NoError(t, err) - val, err = sup.MarshalBSUP(v) + val, err = super.Marshal(super.NewContext(), v) require.NoError(t, err) assert.Equal(t, s, sup.FormatValue(val)) }) diff --git a/sup/marshal_test.go b/sup/marshal_test.go index 5d0260c2d9..42ed7e489e 100644 --- a/sup/marshal_test.go +++ b/sup/marshal_test.go @@ -32,28 +32,28 @@ type Animal struct { func (a *Animal) Color() string { return a.MyColor } func TestInterfaceMarshal(t *testing.T) { - m := sup.NewMarshaler() - m.Decorate(sup.StyleSimple) + m := super.NewMarshaler(super.NewContext()) + m.Decorate(super.StyleSimple) - supRose, err := m.Marshal(Thing(&Plant{"red"})) + roseVal, err := m.Marshal(Thing(&Plant{"red"})) require.NoError(t, err) assert.Equal(t, `type Plant={MyColor:string} -{MyColor:"red"}::Plant`, supRose) +{MyColor:"red"}::Plant`, sup.FormatValueWithTypes(roseVal)) - supFlamingo, err := m.Marshal(Thing(&Animal{"pink"})) + flamingoVal, err := m.Marshal(Thing(&Animal{"pink"})) require.NoError(t, err) assert.Equal(t, `type Animal={MyColor:string} -{MyColor:"pink"}::Animal`, supFlamingo) +{MyColor:"pink"}::Animal`, sup.FormatValueWithTypes(flamingoVal)) - u := sup.NewUnmarshaler() + u := super.NewUnmarshaler() u.Bind(Plant{}, Animal{}) var thing Thing - err = u.Unmarshal(supRose, &thing) + err = u.Unmarshal(roseVal, &thing) require.NoError(t, err) assert.Equal(t, "red", thing.Color()) - err = u.Unmarshal(supFlamingo, &thing) + err = u.Unmarshal(flamingoVal, &thing) require.NoError(t, err) assert.Equal(t, "pink", thing.Color()) } @@ -61,27 +61,27 @@ func TestInterfaceMarshal(t *testing.T) { type Roll bool func TestMarshal(t *testing.T) { - z, err := sup.Marshal("hello, world") + z, err := super.Marshal(super.NewContext(), "hello, world") require.NoError(t, err) - assert.Equal(t, `"hello, world"`, z) + assert.Equal(t, `"hello, world"`, sup.FormatValue(z)) aIn := []int8{1, 2, 3} - z, err = sup.Marshal(aIn) + z, err = super.Marshal(super.NewContext(), aIn) require.NoError(t, err) - assert.Equal(t, `[1::int8,2::int8,3::int8]`, z) + assert.Equal(t, `[1::int8,2::int8,3::int8]`, sup.FormatValue(z)) var v any - err = sup.Unmarshal(z, &v) + err = super.Unmarshal(z, &v) require.NoError(t, err) aOut, ok := v.([]int8) assert.Equal(t, ok, true) assert.Equal(t, aIn, aOut) - m := sup.NewMarshaler() - m.Decorate(sup.StyleSimple) + m := super.NewMarshaler(super.NewContext()) + m.Decorate(super.StyleSimple) z, err = m.Marshal(Roll(true)) require.NoError(t, err) - assert.Equal(t, "type Roll=bool\ntrue::Roll", z) + assert.Equal(t, "type Roll=bool\ntrue::Roll", sup.FormatValueWithTypes(z)) } type BytesRecord struct { @@ -106,7 +106,7 @@ type SliceRecord struct { } func TestBytes(t *testing.T) { - m := sup.NewBSUPMarshaler() + m := super.NewMarshaler(super.NewContext()) rec, err := m.Marshal(BytesRecord{B: []byte{1, 2, 3}}) require.NoError(t, err) require.NotNil(t, rec) @@ -118,8 +118,8 @@ func TestBytes(t *testing.T) { assert.Equal(t, "{A:0x040506}", sup.FormatValue(rec)) id := IDRecord{A: ID{0, 1, 2, 3}, B: ID{4, 5, 6, 7}} - m = sup.NewBSUPMarshaler() - m.Decorate(sup.StyleSimple) + m = super.NewMarshaler(super.NewContext()) + m.Decorate(super.StyleSimple) rec, err = m.Marshal(id) require.NoError(t, err) require.NotNil(t, rec) @@ -128,21 +128,21 @@ type IDRecord={A:ID,B:ID} {A:0x00010203,B:0x04050607}::IDRecord`, sup.FormatValueWithTypes(rec)) var id2 IDRecord - u := sup.NewBSUPUnmarshaler() + u := super.NewUnmarshaler() u.Bind(IDRecord{}, ID{}) - err = sup.UnmarshalBSUP(rec, &id2) + err = super.Unmarshal(rec, &id2) require.NoError(t, err) assert.Equal(t, id, id2) b2 := BytesRecord{B: nil} - m = sup.NewBSUPMarshaler() + m = super.NewMarshaler(super.NewContext()) rec, err = m.Marshal(b2) require.NoError(t, err) require.NotNil(t, rec) assert.Equal(t, "{B:0x}", sup.FormatValue(rec)) s := SliceRecord{S: nil} - m = sup.NewBSUPMarshaler() + m = super.NewMarshaler(super.NewContext()) rec, err = m.Marshal(s) require.NoError(t, err) require.NotNil(t, rec) @@ -163,8 +163,8 @@ func TestMixedTypeArrayInsideRecord(t *testing.T) { &Animal{"blue"}, }, } - m := sup.NewBSUPMarshaler() - m.Decorate(sup.StyleSimple) + m := super.NewMarshaler(super.NewContext()) + m.Decorate(super.StyleSimple) zv, err := m.Marshal(x) require.NoError(t, err) @@ -184,10 +184,12 @@ func TestMixedTypeArrayInsideRecord(t *testing.T) { // Double check that all the proper typing made it into the implied union. assert.Equal(t, `{X:"hello",S:[[{MyColor:"red"}::=Plant,{MyColor:"blue"}::=Animal]]}:=RecordWithInterfaceSlice`, actual) - u := sup.NewUnmarshaler() + u := super.NewUnmarshaler() u.Bind(Animal{}, Plant{}, RecordWithInterfaceSlice{}) var out RecordWithInterfaceSlice - err = u.Unmarshal(actual, &out) + val, err := sup.ParseValue(super.NewContext(), actual) + require.NoError(t, err) + err = u.Unmarshal(val, &out) require.NoError(t, err) assert.Equal(t, *x, out) } @@ -202,10 +204,12 @@ func TestMixedTypeUnmarshal(t *testing.T) { type Plant={MyColor:string} {S:[{MyColor:"red"}::Plant,{MyColor:"blue"}::Animal]} ` - u := sup.NewUnmarshaler() + u := super.NewUnmarshaler() u.Bind(Animal{}, Plant{}, ArrayOfThings{}) var out ArrayOfThings - err := u.Unmarshal(in, &out) + val, err := sup.ParseValue(super.NewContext(), in) + require.NoError(t, err) + err = u.Unmarshal(val, &out) require.NoError(t, err) assert.Equal(t, ArrayOfThings{S: []Thing{&Plant{"red"}, &Animal{"blue"}}}, out) } @@ -227,8 +231,8 @@ func TestMixedTypeArrayOfStructWithInterface(t *testing.T) { Thing: &Animal{"blue"}, }, } - m := sup.NewBSUPMarshaler() - m.Decorate(sup.StyleSimple) + m := super.NewMarshaler(super.NewContext()) + m.Decorate(super.StyleSimple) zv, err := m.Marshal(input) require.NoError(t, err) @@ -249,10 +253,12 @@ func TestMixedTypeArrayOfStructWithInterface(t *testing.T) { // Double check that all the proper typing made it into the implied union. assert.Equal(t, `[{Message:"hello",Thing:{MyColor:"red"}::=Plant}::=MessageThing,{Message:"world",Thing:{MyColor:"blue"}::=Animal}::=MessageThing]`, actual) - u := sup.NewUnmarshaler() + u := super.NewUnmarshaler() u.Bind(Plant{}, Animal{}, MessageThing{}) var out RecordWithInterfaceSlice - err = u.Unmarshal(actual, &out) + val, err := sup.ParseValue(super.NewContext(), actual) + require.NoError(t, err) + err = u.Unmarshal(val, &out) require.NoError(t, err) assert.Equal(t, input, out) } @@ -264,7 +270,7 @@ type Foo struct { func TestUnexported(t *testing.T) { f := &Foo{1, 2} - m := sup.NewBSUPMarshaler() + m := super.NewMarshaler(super.NewContext()) _, err := m.Marshal(f) require.NoError(t, err) } @@ -280,13 +286,13 @@ func TestBSUPValueField(t *testing.T) { Name: "test1", Field: super.NewInt64(123), } - m := sup.NewBSUPMarshaler() - m.Decorate(sup.StyleSimple) + m := super.NewMarshaler(super.NewContext()) + m.Decorate(super.StyleSimple) zv, err := m.Marshal(bsupValueField) require.NoError(t, err) assert.Equal(t, `type BSUPValueField={Name:string,field:any} {Name:"test1",field:123::any}::BSUPValueField`, sup.FormatValueWithTypes(zv)) - u := sup.NewBSUPUnmarshaler() + u := super.NewUnmarshaler() var out BSUPValueField err = u.Unmarshal(zv, &out) require.NoError(t, err) @@ -299,13 +305,13 @@ func TestBSUPValueField(t *testing.T) { Name: "test2", Field: zv2, } - m2 := sup.NewBSUPMarshaler() - m2.Decorate(sup.StyleSimple) + m2 := super.NewMarshaler(super.NewContext()) + m2.Decorate(super.StyleSimple) zv3, err := m2.Marshal(bsupValueField2) require.NoError(t, err) assert.Equal(t, `type BSUPValueField={Name:string,field:any} {Name:"test2",field:{s:"foo",a:[1,2,3]}::any}::BSUPValueField`, sup.FormatValueWithTypes(zv3)) - u2 := sup.NewBSUPUnmarshaler() + u2 := super.NewUnmarshaler() var out2 BSUPValueField err = u2.Unmarshal(zv3, &out2) require.NoError(t, err) @@ -316,11 +322,11 @@ func TestJSONFieldTag(t *testing.T) { type jsonTag struct { Value string `json:"value"` } - s, err := sup.Marshal(jsonTag{Value: "test"}) + s, err := super.Marshal(super.NewContext(), jsonTag{Value: "test"}) require.NoError(t, err) - assert.Equal(t, `{value:"test"}`, s) + assert.Equal(t, `{value:"test"}`, sup.FormatValue(s)) var j jsonTag - require.NoError(t, sup.Unmarshal(s, &j)) + require.NoError(t, super.Unmarshal(s, &j)) assert.Equal(t, jsonTag{Value: "test"}, j) } @@ -329,54 +335,54 @@ func TestIgnoreField(t *testing.T) { Value string `super:"value"` Ignore func() error `super:"-"` } - b, err := sup.Marshal(s{Value: "test"}) + b, err := super.Marshal(super.NewContext(), s{Value: "test"}) require.NoError(t, err) - assert.Equal(t, `{value:"test"}`, b) + assert.Equal(t, `{value:"test"}`, sup.FormatValue(b)) var v s - require.NoError(t, sup.Unmarshal(b, &v)) + require.NoError(t, super.Unmarshal(b, &v)) assert.Equal(t, s{Value: "test"}, v) } func TestMarshalNetIP(t *testing.T) { before := net.ParseIP("10.0.0.1") - b, err := sup.Marshal(before) + b, err := super.Marshal(super.NewContext(), before) require.NoError(t, err) - assert.Equal(t, `10.0.0.1`, b) + assert.Equal(t, `10.0.0.1`, sup.FormatValue(b)) var after net.IP - err = sup.Unmarshal(b, &after) + err = super.Unmarshal(b, &after) require.NoError(t, err) assert.Equal(t, before, after) } func TestMarshalNetipAddr(t *testing.T) { before := netip.MustParseAddr("10.0.0.1") - b, err := sup.Marshal(before) + b, err := super.Marshal(super.NewContext(), before) require.NoError(t, err) - assert.Equal(t, `10.0.0.1`, b) + assert.Equal(t, `10.0.0.1`, sup.FormatValue(b)) var after netip.Addr - err = sup.Unmarshal(b, &after) + err = super.Unmarshal(b, &after) require.NoError(t, err) assert.Equal(t, before, after) } func TestMarshalDecoratedIPs(t *testing.T) { - m := sup.NewMarshaler() + m := super.NewMarshaler(super.NewContext()) // Make sure IPs don't get decorated with Go type and just // appear as native super-structured IPs. - m.Decorate(sup.StyleSimple) + m.Decorate(super.StyleSimple) b, err := m.Marshal(net.ParseIP("142.250.72.142")) require.NoError(t, err) - assert.Equal(t, `142.250.72.142`, b) + assert.Equal(t, `142.250.72.142`, sup.FormatValue(b)) b, err = m.Marshal(netip.MustParseAddr("142.250.72.142")) require.NoError(t, err) - assert.Equal(t, `142.250.72.142`, b) + assert.Equal(t, `142.250.72.142`, sup.FormatValue(b)) } func TestMarshalGoTime(t *testing.T) { tm, _ := time.Parse(time.RFC3339, "2006-01-02T15:04:05.123Z") - b, err := sup.Marshal(tm) + b, err := super.Marshal(super.NewContext(), tm) require.NoError(t, err) - assert.Equal(t, `2006-01-02T15:04:05.123Z`, b) + assert.Equal(t, `2006-01-02T15:04:05.123Z`, sup.FormatValue(b)) } type Metadata interface { @@ -432,11 +438,11 @@ func TestRecordWithMixedTypeNamedArrayElems(t *testing.T) { }, }, } - m := sup.NewBSUPMarshaler() - m.Decorate(sup.StyleSimple) + m := super.NewMarshaler(super.NewContext()) + m.Decorate(super.StyleSimple) val, err := m.Marshal(in) require.NoError(t, err) - u := sup.NewBSUPUnmarshaler() + u := super.NewUnmarshaler() u.Bind(Record{}, Array{}, Primitive{}) var out Metadata err = u.Unmarshal(val, &out) @@ -445,26 +451,32 @@ func TestRecordWithMixedTypeNamedArrayElems(t *testing.T) { } func TestInterfaceWithConcreteEmptyValue(t *testing.T) { - u := sup.NewUnmarshaler() + u := super.NewUnmarshaler() // This case doesn't need a binding because we set the // interface value to an empty underlying value. out := Metadata(&Primitive{}) - err := u.Unmarshal(`type Primitive={Foo:string} {Foo:"foo"}::Primitive`, &out) + val, err := sup.ParseValue(super.NewContext(), `type Primitive={Foo:string} {Foo:"foo"}::Primitive`) + require.NoError(t, err) + err = u.Unmarshal(val, &out) require.NoError(t, err) assert.Equal(t, &Primitive{Foo: "foo"}, out) } func TestSuperType(t *testing.T) { sctx := super.NewContext() - u := sup.NewUnmarshaler() + u := super.NewUnmarshaler() var typ super.Type - err := u.Unmarshal(``, &typ) + val, err := sup.ParseValue(super.NewContext(), ``) + require.NoError(t, err) + err = u.Unmarshal(val, &typ) assert.EqualError(t, err, `cannot unmarshal type value without type context`) u.SetContext(sctx) - err = u.Unmarshal(``, &typ) + err = u.Unmarshal(val, &typ) require.NoError(t, err) assert.Equal(t, super.TypeString, typ) - err = u.Unmarshal(``, &typ) + val, err = sup.ParseValue(super.NewContext(), ``) + require.NoError(t, err) + err = u.Unmarshal(val, &typ) require.NoError(t, err) assert.Equal(t, super.TypeInt64, typ) } @@ -472,7 +484,9 @@ func TestSuperType(t *testing.T) { func TestSimpleUnionUnmarshal(t *testing.T) { t.Skip("see issue #4012") var i int64 - err := sup.Unmarshal(`1::int64|string`, &i) + val, err := sup.ParseValue(super.NewContext(), `1::int64|string`) + require.NoError(t, err) + err = super.Unmarshal(val, &i) require.NoError(t, err) assert.Equal(t, 1, i) } @@ -481,7 +495,7 @@ func TestEmbeddedNilInterface(t *testing.T) { in := &Record{ Fields: nil, } - val, err := sup.Marshal(in) + val, err := super.Marshal(super.NewContext(), in) require.NoError(t, err) - assert.Equal(t, `{Fields:[]::[{Name:string,Values:null}]}`, val) + assert.Equal(t, `{Fields:[]::[{Name:string,Values:null}]}`, sup.FormatValue(val)) } From f3fddbd4e61aa858479fa6c56c81efa62693f07d Mon Sep 17 00:00:00 2001 From: Steven McCanne Date: Mon, 28 Sep 2026 05:39:27 -0700 Subject: [PATCH 2/4] re-enable terminal-output-format test --- cmd/super/ztests/terminal-output-format.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/cmd/super/ztests/terminal-output-format.yaml b/cmd/super/ztests/terminal-output-format.yaml index b09e5d6d03..3749252b5f 100644 --- a/cmd/super/ztests/terminal-output-format.yaml +++ b/cmd/super/ztests/terminal-output-format.yaml @@ -1,5 +1,3 @@ -skip: true - script: | pty() { python3 -c 'import pty, sys; pty.spawn(sys.argv[1:])' "$@" From afa7232fd0f4b3db0062f37d116632b2930f116f Mon Sep 17 00:00:00 2001 From: Steven McCanne Date: Mon, 28 Sep 2026 10:59:52 -0700 Subject: [PATCH 3/4] address PR feedback --- db/pools/config.go | 4 ++-- marshal.go | 8 ++++---- order/direction.go | 4 ++-- order/nulls.go | 4 ++-- pkg/storage/uri.go | 2 +- sup/marshal_bsup_test.go | 6 ++++-- 6 files changed, 15 insertions(+), 13 deletions(-) diff --git a/db/pools/config.go b/db/pools/config.go index 61138d0cae..8dd4ca41f5 100644 --- a/db/pools/config.go +++ b/db/pools/config.go @@ -68,7 +68,7 @@ var hackedBindings = []super.Binding{ {Name: "pools.Config", Template: marshalConfig{}}, } -func (p Config) Marshal(marshaler *super.Marshaler) (super.Type, error) { +func (p Config) MarshalSuper(marshaler *super.Marshaler) (super.Type, error) { marshaler.NamedBindings(hackedBindings) m := marshalConfig{ Ts: p.Ts, @@ -86,7 +86,7 @@ func (p Config) Marshal(marshaler *super.Marshaler) (super.Type, error) { return typ, err } -func (p *Config) Unmarshal(unmarshaler *super.Unmarshaler, val super.Value) error { +func (p *Config) UnmarshalSuper(unmarshaler *super.Unmarshaler, val super.Value) error { unmarshaler.NamedBindings(hackedBindings) var m marshalConfig if err := unmarshaler.Unmarshal(val, &m); err != nil { diff --git a/marshal.go b/marshal.go index b79672d6b0..25cf341045 100644 --- a/marshal.go +++ b/marshal.go @@ -18,7 +18,7 @@ import ( //XXX handle new TypeError => marshal as a SUP string? type CustomMarshaler interface { - Marshal(*Marshaler) (Type, error) + MarshalSuper(*Marshaler) (Type, error) } func Marshal(sctx *Context, v any) (Value, error) { @@ -192,7 +192,7 @@ func (m *Marshaler) encodeAny(v reflect.Value) (Type, error) { } switch v := v.Interface().(type) { case CustomMarshaler: - return v.Marshal(m) + return v.MarshalSuper(m) case float16.Float16: m.builder.Append(EncodeFloat16(v.Float32())) return TypeFloat16, nil @@ -554,7 +554,7 @@ func (m *Marshaler) lookupTypeNamed(t reflect.Type, typ Type) (Type, error) { } type CustomUnmarshaler interface { - Unmarshal(*Unmarshaler, Value) error + UnmarshalSuper(*Unmarshaler, Value) error } type Unmarshaler struct { @@ -625,7 +625,7 @@ func (u *Unmarshaler) decodeAny(val Value, v reflect.Value) (x error) { val = val.DeunionIntoNameds() m, v := indirect(v, val) if m != nil { - return m.Unmarshal(u, val) + return m.UnmarshalSuper(u, val) } switch v.Interface().(type) { case float16.Float16: diff --git a/order/direction.go b/order/direction.go index d58c86d417..79b9173767 100644 --- a/order/direction.go +++ b/order/direction.go @@ -85,11 +85,11 @@ func (d Direction) MarshalJSON() ([]byte, error) { return json.Marshal(d.String()) } -func (d Direction) Marshal(m *super.Marshaler) (super.Type, error) { +func (d Direction) MarshalSuper(m *super.Marshaler) (super.Type, error) { return m.MarshalValue(d.String()) } -func (d *Direction) Unmarshal(u *super.Unmarshaler, val super.Value) error { +func (d *Direction) UnmarshalSuper(u *super.Unmarshaler, val super.Value) error { dir, err := ParseDirection(string(val.Bytes())) if err != nil { return err diff --git a/order/nulls.go b/order/nulls.go index 7f0f75dd05..d1ce625aab 100644 --- a/order/nulls.go +++ b/order/nulls.go @@ -42,11 +42,11 @@ func (n *Nulls) UnmarshalText(b []byte) error { return nil } -func (n Nulls) Marshal(m *super.Marshaler) (super.Type, error) { +func (n Nulls) MarshalSuper(m *super.Marshaler) (super.Type, error) { return m.MarshalValue(n.String()) } -func (n *Nulls) Unmarshal(u *super.Unmarshaler, val super.Value) error { +func (n *Nulls) UnmarshalSuper(u *super.Unmarshaler, val super.Value) error { if val.Type().ID() != super.IDString { return fmt.Errorf("cannot unmarshal %q into order.Nulls", sup.FormatValue(val)) } diff --git a/pkg/storage/uri.go b/pkg/storage/uri.go index 3a323ae777..f63ac1c164 100644 --- a/pkg/storage/uri.go +++ b/pkg/storage/uri.go @@ -70,6 +70,6 @@ func (u *URI) UnmarshalText(b []byte) error { return nil } -func (u *URI) Marshal(mc *super.Marshaler) (super.Type, error) { +func (u *URI) MarshalSuper(mc *super.Marshaler) (super.Type, error) { return mc.MarshalValue(u.String()) } diff --git a/sup/marshal_bsup_test.go b/sup/marshal_bsup_test.go index 190271acd1..9eeffa1b31 100644 --- a/sup/marshal_bsup_test.go +++ b/sup/marshal_bsup_test.go @@ -293,11 +293,13 @@ func TestUnmarshalSlice(t *testing.T) { type testMarshaler string -func (m testMarshaler) Marshal(mc *super.Marshaler) (super.Type, error) { +var _ super.CustomMarshaler = (*testMarshaler)(nil) + +func (m testMarshaler) MarshalSuper(mc *super.Marshaler) (super.Type, error) { return mc.MarshalValue("marshal-" + string(m)) } -func (m *testMarshaler) Unmarshal(mc *super.Unmarshaler, val super.Value) error { +func (m *testMarshaler) UnmarshalSuper(mc *super.Unmarshaler, val super.Value) error { var s string if err := mc.Unmarshal(val, &s); err != nil { return err From 26b512832a8fc83e65ec8a5fd75663031cb16129 Mon Sep 17 00:00:00 2001 From: Steven McCanne Date: Mon, 28 Sep 2026 11:08:35 -0700 Subject: [PATCH 4/4] fix rebase --- cmd/super/dev/bsup/command.go | 9 ++++----- db/journal/store.go | 6 ------ 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/cmd/super/dev/bsup/command.go b/cmd/super/dev/bsup/command.go index b42c658349..26179f7da1 100644 --- a/cmd/super/dev/bsup/command.go +++ b/cmd/super/dev/bsup/command.go @@ -16,7 +16,6 @@ import ( "github.com/brimdata/super/pkg/charm" "github.com/brimdata/super/pkg/storage" - "github.com/brimdata/super/sup" ) var spec = &charm.Spec{ @@ -73,7 +72,7 @@ func (c *Command) Run(args []string) error { } var vals []super.Value sctx := super.NewContext() - marshaler := sup.NewBSUPMarshalerWithContext(sctx) + marshaler := super.NewMarshaler(sctx) for { hdr, err := readHeader(r) if err != nil { @@ -134,7 +133,7 @@ func readHeader(r io.Reader) (bsup.Header, error) { return hdr, err } -func readObject(sctx *super.Context, marshaler *sup.MarshalBSUPContext, r io.Reader, vals []super.Value) ([]super.Value, error) { +func readObject(sctx *super.Context, marshaler *super.Marshaler, r io.Reader, vals []super.Value) ([]super.Value, error) { var bytes [bsup.DataHeaderSize]byte if _, err := io.ReadFull(r, bytes[:]); err != nil { return vals, err @@ -191,7 +190,7 @@ func readObject(sctx *super.Context, marshaler *sup.MarshalBSUPContext, r io.Rea return vals, err } -func readFooter(sctx *super.Context, marshaler *sup.MarshalBSUPContext, r io.Reader, vals []super.Value) ([]super.Value, error) { +func readFooter(sctx *super.Context, marshaler *super.Marshaler, r io.Reader, vals []super.Value) ([]super.Value, error) { var bytes [bsup.FooterSize]byte if _, err := io.ReadFull(r, bytes[:]); err != nil { return vals, err @@ -227,7 +226,7 @@ func readFooter(sctx *super.Context, marshaler *sup.MarshalBSUPContext, r io.Rea return vals, nil } -func marshalTypeDefs(marshaler *sup.MarshalBSUPContext, vals []super.Value, bytes []byte) ([]super.Value, error) { +func marshalTypeDefs(marshaler *super.Marshaler, vals []super.Value, bytes []byte) ([]super.Value, error) { id := uint32(super.IDTypeComplex) for len(bytes) > 0 { var desc any diff --git a/db/journal/store.go b/db/journal/store.go index 85949efd94..faaedf09b4 100644 --- a/db/journal/store.go +++ b/db/journal/store.go @@ -215,15 +215,9 @@ func (s *Store) putSnapshot(ctx context.Context, at ID, table map[string]Entry) return s.writeTable(zw, table) } -<<<<<<< HEAD func (s *Store) writeTable(w *bsupio.RowWriter, table map[string]Entry) error { - marshaler := sup.NewBSUPMarshaler() - marshaler.Decorate(sup.StylePackage) -======= -func (s *Store) writeTable(w *bsupio.Writer, table map[string]Entry) error { marshaler := super.NewMarshaler(super.NewContext()) marshaler.Decorate(super.StylePackage) ->>>>>>> d0eb21c33 (move the marshaler to package super and simplify naming) for _, entry := range table { val, err := marshaler.Marshal(entry) if err != nil {