mirror/protobuf-go
1
0
Fork 0
mirror of https://github.com/protocolbuffers/protobuf-go.git synced 2025-01-29 18:32:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

encoding: add MarshalAppend to protojson and prototext

Browse Source 
Adds MarshalAppend methods to allow for byte slices to be reused.
Copies signature from the binary encoding.

Small changes to internal json and text libraries to use strconv
AppendInt and AppendUint for number encoding.

Change-Id: Ife7c8979c1c153a0a0bf9b70b296b8158d38dffc
Reviewed-on: https://go-review.googlesource.com/c/protobuf/+/489615
Reviewed-by: Edward McFarlane <emcfarlane000@gmail.com>
Reviewed-by: Joseph Tsai <joetsai@digital-static.net>
Reviewed-by: Lasse Folger <lassefolger@google.com>
Reviewed-by: Damien Neil <dneil@google.com>
This commit is contained in:
Edward McFarlane 2023-04-27 11:07:10 +01:00 committed by Damien Neil
parent 1bca6d9b7d
commit 05cbe34333
8 changed files with 122 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
encoding/protojson/encode.go
View File

@ -106,13 +106,19 @@ func (o MarshalOptions) Format(m proto.Message) string {
// MarshalOptions. Do not depend on the output being stable. It may change over
// time across different versions of the program.
func (o MarshalOptions) Marshal(m proto.Message) ([]byte, error) {
return o.marshal(m)
return o.marshal(nil, m)
}
// MarshalAppend appends the JSON format encoding of m to b,
// returning the result.
func (o MarshalOptions) MarshalAppend(b []byte, m proto.Message) ([]byte, error) {
return o.marshal(b, m)
}
// marshal is a centralized function that all marshal operations go through.
// For profiling purposes, avoid changing the name of this function or
// introducing other code paths for marshal that do not go through this.
func (o MarshalOptions) marshal(m proto.Message) ([]byte, error) {
func (o MarshalOptions) marshal(b []byte, m proto.Message) ([]byte, error) {
if o.Multiline && o.Indent == "" {
o.Indent = defaultIndent
}
@ -120,7 +126,7 @@ func (o MarshalOptions) marshal(m proto.Message) ([]byte, error) {
o.Resolver = protoregistry.GlobalTypes
}
internalEnc, err := json.NewEncoder(o.Indent)
internalEnc, err := json.NewEncoder(b, o.Indent)
if err != nil {
return nil, err
}
@ -128,7 +134,7 @@ func (o MarshalOptions) marshal(m proto.Message) ([]byte, error) {
// Treat nil message interface as an empty message,
// in which case the output in an empty JSON object.
if m == nil {
return []byte("{}"), nil
return append(b, '{', '}'), nil
}
enc := encoder{internalEnc, o}

41
encoding/protojson/encode_test.go
View File

@ -2310,3 +2310,44 @@ func TestMarshal(t *testing.T) {
})
}
}
func TestEncodeAppend(t *testing.T) {
want := []byte("prefix")
got := append([]byte(nil), want...)
got, err := protojson.MarshalOptions{}.MarshalAppend(got, &pb3.Scalars{
SString: "value",
})
if err != nil {
t.Fatal(err)
}
if !bytes.HasPrefix(got, want) {
t.Fatalf("MarshalAppend modified prefix: got %v, want prefix %v", got, want)
}
}
func TestMarshalAppendAllocations(t *testing.T) {
m := &pb3.Scalars{SInt32: 1}
const count = 1000
size := 12
b := make([]byte, size)
// AllocsPerRun returns an integral value.
marshalAllocs := testing.AllocsPerRun(count, func() {
_, err := protojson.MarshalOptions{}.MarshalAppend(b[:0], m)
if err != nil {
t.Fatal(err)
}
})
b = nil
marshalAppendAllocs := testing.AllocsPerRun(count, func() {
var err error
b, err = protojson.MarshalOptions{}.MarshalAppend(b, m)
if err != nil {
t.Fatal(err)
}
})
if marshalAllocs != marshalAppendAllocs {
t.Errorf("%v allocs/op when writing to a preallocated buffer", marshalAllocs)
t.Errorf("%v allocs/op when repeatedly appending to a slice", marshalAppendAllocs)
t.Errorf("expect amortized allocs/op to be identical")
}
}

14
encoding/prototext/encode.go
View File

@ -101,13 +101,19 @@ func (o MarshalOptions) Format(m proto.Message) string {
// MarshalOptions object. Do not depend on the output being stable. It may
// change over time across different versions of the program.
func (o MarshalOptions) Marshal(m proto.Message) ([]byte, error) {
return o.marshal(m)
return o.marshal(nil, m)
}
// MarshalAppend appends the textproto format encoding of m to b,
// returning the result.
func (o MarshalOptions) MarshalAppend(b []byte, m proto.Message) ([]byte, error) {
return o.marshal(b, m)
}
// marshal is a centralized function that all marshal operations go through.
// For profiling purposes, avoid changing the name of this function or
// introducing other code paths for marshal that do not go through this.
func (o MarshalOptions) marshal(m proto.Message) ([]byte, error) {
func (o MarshalOptions) marshal(b []byte, m proto.Message) ([]byte, error) {
var delims = [2]byte{'{', '}'}
if o.Multiline && o.Indent == "" {
@ -117,7 +123,7 @@ func (o MarshalOptions) marshal(m proto.Message) ([]byte, error) {
o.Resolver = protoregistry.GlobalTypes
}
internalEnc, err := text.NewEncoder(o.Indent, delims, o.EmitASCII)
internalEnc, err := text.NewEncoder(b, o.Indent, delims, o.EmitASCII)
if err != nil {
return nil, err
}
@ -125,7 +131,7 @@ func (o MarshalOptions) marshal(m proto.Message) ([]byte, error) {
// Treat nil message interface as an empty message,
// in which case there is nothing to output.
if m == nil {
return []byte{}, nil
return b, nil
}
enc := encoder{internalEnc, o}

42
encoding/prototext/encode_test.go
View File

@ -5,6 +5,7 @@
package prototext_test
import (
"bytes"
"math"
"testing"
@ -1435,3 +1436,44 @@ value: "\x80"
})
}
}
func TestEncodeAppend(t *testing.T) {
want := []byte("prefix")
got := append([]byte(nil), want...)
got, err := prototext.MarshalOptions{}.MarshalAppend(got, &pb3.Scalars{
SString: "value",
})
if err != nil {
t.Fatal(err)
}
if !bytes.HasPrefix(got, want) {
t.Fatalf("MarshalAppend modified prefix: got %v, want prefix %v", got, want)
}
}
func TestMarshalAppendAllocations(t *testing.T) {
m := &pb3.Scalars{SInt32: 1}
const count = 1000
size := 9
b := make([]byte, size)
// AllocsPerRun returns an integral value.
marshalAllocs := testing.AllocsPerRun(count, func() {
_, err := prototext.MarshalOptions{}.MarshalAppend(b[:0], m)
if err != nil {
t.Fatal(err)
}
})
b = nil
marshalAppendAllocs := testing.AllocsPerRun(count, func() {
var err error
b, err = prototext.MarshalOptions{}.MarshalAppend(b, m)
if err != nil {
t.Fatal(err)
}
})
if marshalAllocs != marshalAppendAllocs {
t.Errorf("%v allocs/op when writing to a preallocated buffer", marshalAllocs)
t.Errorf("%v allocs/op when repeatedly appending to a slice", marshalAppendAllocs)
t.Errorf("expect amortized allocs/op to be identical")
}
}

10
internal/encoding/json/encode.go
View File

@ -41,8 +41,10 @@ type Encoder struct {
//
// If indent is a non-empty string, it causes every entry for an Array or Object
// to be preceded by the indent and trailed by a newline.
func NewEncoder(indent string) (*Encoder, error) {
e := &Encoder{}
func NewEncoder(buf []byte, indent string) (*Encoder, error) {
e := &Encoder{
out: buf,
}
if len(indent) > 0 {
if strings.Trim(indent, " \t") != "" {
return nil, errors.New("indent may only be composed of space or tab characters")
@ -176,13 +178,13 @@ func appendFloat(out []byte, n float64, bitSize int) []byte {
// WriteInt writes out the given signed integer in JSON number value.
func (e *Encoder) WriteInt(n int64) {
e.prepareNext(scalar)
e.out = append(e.out, strconv.FormatInt(n, 10)...)
e.out = strconv.AppendInt(e.out, n, 10)
}
// WriteUint writes out the given unsigned integer in JSON number value.
func (e *Encoder) WriteUint(n uint64) {
e.prepareNext(scalar)
e.out = append(e.out, strconv.FormatUint(n, 10)...)
e.out = strconv.AppendUint(e.out, n, 10)
}
// StartObject writes out the '{' symbol.

6
internal/encoding/json/encode_test.go
View File

@ -356,7 +356,7 @@ func TestEncoder(t *testing.T) {
for _, tc := range tests {
t.Run(tc.desc, func(t *testing.T) {
if tc.wantOut != "" {
enc, err := json.NewEncoder("")
enc, err := json.NewEncoder(nil, "")
if err != nil {
t.Fatalf("NewEncoder() returned error: %v", err)
}
@ -367,7 +367,7 @@ func TestEncoder(t *testing.T) {
}
}
if tc.wantOutIndent != "" {
enc, err := json.NewEncoder("\t")
enc, err := json.NewEncoder(nil, "\t")
if err != nil {
t.Fatalf("NewEncoder() returned error: %v", err)
}
@ -387,7 +387,7 @@ func TestWriteStringError(t *testing.T) {
for _, in := range tests {
t.Run(in, func(t *testing.T) {
enc, err := json.NewEncoder("")
enc, err := json.NewEncoder(nil, "")
if err != nil {
t.Fatalf("NewEncoder() returned error: %v", err)
}

10
internal/encoding/text/encode.go
View File

@ -53,8 +53,10 @@ type encoderState struct {
// If outputASCII is true, strings will be serialized in such a way that
// multi-byte UTF-8 sequences are escaped. This property ensures that the
// overall output is ASCII (as opposed to UTF-8).
func NewEncoder(indent string, delims [2]byte, outputASCII bool) (*Encoder, error) {
e := &Encoder{}
func NewEncoder(buf []byte, indent string, delims [2]byte, outputASCII bool) (*Encoder, error) {
e := &Encoder{
encoderState: encoderState{out: buf},
}
if len(indent) > 0 {
if strings.Trim(indent, " \t") != "" {
return nil, errors.New("indent may only be composed of space and tab characters")
@ -195,13 +197,13 @@ func appendFloat(out []byte, n float64, bitSize int) []byte {
// WriteInt writes out the given signed integer value.
func (e *Encoder) WriteInt(n int64) {
e.prepareNext(scalar)
e.out = append(e.out, strconv.FormatInt(n, 10)...)
e.out = strconv.AppendInt(e.out, n, 10)
}
// WriteUint writes out the given unsigned integer value.
func (e *Encoder) WriteUint(n uint64) {
e.prepareNext(scalar)
e.out = append(e.out, strconv.FormatUint(n, 10)...)
e.out = strconv.AppendUint(e.out, n, 10)
}
// WriteLiteral writes out the given string as a literal value without quotes.

8
internal/encoding/text/encode_test.go
View File

@ -341,7 +341,7 @@ func runEncoderTest(t *testing.T, tc encoderTestCase, delims [2]byte) {
t.Helper()
if tc.wantOut != "" {
enc, err := text.NewEncoder("", delims, false)
enc, err := text.NewEncoder(nil, "", delims, false)
if err != nil {
t.Fatalf("NewEncoder returned error: %v", err)
}
@ -352,7 +352,7 @@ func runEncoderTest(t *testing.T, tc encoderTestCase, delims [2]byte) {
}
}
if tc.wantOutIndent != "" {
enc, err := text.NewEncoder("\t", delims, false)
enc, err := text.NewEncoder(nil, "\t", delims, false)
if err != nil {
t.Fatalf("NewEncoder returned error: %v", err)
}
@ -520,7 +520,7 @@ func runEncodeStringsTest(t *testing.T, in string, want string, outputASCII bool
charType = "ASCII"
}
enc, err := text.NewEncoder("", [2]byte{}, outputASCII)
enc, err := text.NewEncoder(nil, "", [2]byte{}, outputASCII)
if err != nil {
t.Fatalf("[%s] NewEncoder returned error: %v", charType, err)
}
@ -532,7 +532,7 @@ func runEncodeStringsTest(t *testing.T, in string, want string, outputASCII bool
}
func TestReset(t *testing.T) {
enc, err := text.NewEncoder("\t", [2]byte{}, false)
enc, err := text.NewEncoder(nil, "\t", [2]byte{}, false)
if err != nil {
t.Fatalf("NewEncoder returned error: %v", err)
}