Herbie Ong 9b3d97c473 encoding/prototext: rewrite of internal/encoding/text
* Fixes golang/protobuf#842. Unmarshal can now parse singular or
  repeated message fields without the field separator.
* Fixes golang/protobuf#1011. Handles negative 0 properly.
* For unknown fields with fixed 32-bit and 64-bit wire types, output is
  now in hex format with 0x prefix similar to C++ lib output. Previous
  Go implementation simply outputs these as decimal numbers %d.
* All parsing errors, except for unexpected EOF should now contain line
  and column number info.
* Fixed following conformance-related features:
  * Parse nan,inf,-inf,infinity,-infinity as case-insensitive.
  * Interpret float32 overflows as inf or -inf.
  * Parse large int-like number as proto float.
* Discard unknown map field if DiscardUnknown=true.
* Allow whitespaces/comments in Any type URL and extension field names per spec.
* Improves performance and memory usage. It is now as fast and efficient as
  protojson, if not better on most benchmarks.

name                                     old time/op    new time/op    delta
Text/Unmarshal/google_message1_proto2-4    14.1µs ±43%     8.7µs ±12%  -38.27%  (p=0.000 n=10+10)
Text/Unmarshal/google_message1_proto3-4    11.6µs ±18%     7.7µs ± 9%  -33.69%  (p=0.000 n=10+10)
Text/Unmarshal/google_message2-4           6.20ms ±27%    4.10ms ± 5%  -33.95%  (p=0.000 n=10+10)
Text/Marshal/google_message1_proto2-4      12.8µs ± 6%    10.3µs ±23%  -19.54%  (p=0.000 n=9+10)
Text/Marshal/google_message1_proto3-4      11.9µs ±16%     8.6µs ±10%  -27.45%  (p=0.000 n=10+10)
Text/Marshal/google_message2-4             5.59ms ± 5%    5.30ms ±22%     ~     (p=0.356 n=9+10)
JSON/Unmarshal/google_message1_proto2-4    12.3µs ±61%    13.9µs ±26%     ~     (p=0.190 n=10+10)
JSON/Unmarshal/google_message1_proto3-4    7.51µs ± 6%    7.86µs ± 1%   +4.66%  (p=0.010 n=10+9)
JSON/Unmarshal/google_message2-4           3.74ms ± 2%    3.94ms ± 2%   +5.32%  (p=0.000 n=10+10)
JSON/Marshal/google_message1_proto2-4      9.90µs ±12%    9.95µs ± 4%     ~     (p=0.315 n=9+10)
JSON/Marshal/google_message1_proto3-4      7.55µs ± 4%    7.93µs ± 3%   +4.98%  (p=0.000 n=10+10)
JSON/Marshal/google_message2-4             4.29ms ± 5%    4.49ms ± 2%   +4.53%  (p=0.001 n=10+10)

name                                     old alloc/op   new alloc/op   delta
Text/Unmarshal/google_message1_proto2-4    12.5kB ± 0%     2.0kB ± 0%  -83.87%  (p=0.000 n=10+10)
Text/Unmarshal/google_message1_proto3-4    12.2kB ± 0%     1.8kB ± 0%  -85.33%  (p=0.000 n=10+10)
Text/Unmarshal/google_message2-4           5.35MB ± 0%    0.89MB ± 0%  -83.28%  (p=0.000 n=10+9)
Text/Marshal/google_message1_proto2-4      12.0kB ± 0%     1.4kB ± 0%  -88.15%  (p=0.000 n=10+10)
Text/Marshal/google_message1_proto3-4      12.4kB ± 0%     1.9kB ± 0%  -84.91%  (p=0.000 n=10+10)
Text/Marshal/google_message2-4             5.64MB ± 0%    1.02MB ± 0%  -81.85%  (p=0.000 n=10+9)
JSON/Unmarshal/google_message1_proto2-4    2.29kB ± 0%    2.29kB ± 0%     ~     (all equal)
JSON/Unmarshal/google_message1_proto3-4    2.08kB ± 0%    2.08kB ± 0%     ~     (all equal)
JSON/Unmarshal/google_message2-4            899kB ± 0%     899kB ± 0%     ~     (p=1.000 n=10+10)
JSON/Marshal/google_message1_proto2-4      1.46kB ± 0%    1.46kB ± 0%     ~     (all equal)
JSON/Marshal/google_message1_proto3-4      1.36kB ± 0%    1.36kB ± 0%     ~     (all equal)
JSON/Marshal/google_message2-4             1.19MB ± 0%    1.19MB ± 0%     ~     (p=0.197 n=10+10)

name                                     old allocs/op  new allocs/op  delta
Text/Unmarshal/google_message1_proto2-4       133 ± 0%        89 ± 0%  -33.08%  (p=0.000 n=10+10)
Text/Unmarshal/google_message1_proto3-4       108 ± 0%        67 ± 0%  -37.96%  (p=0.000 n=10+10)
Text/Unmarshal/google_message2-4            60.0k ± 0%     38.7k ± 0%  -35.52%  (p=0.000 n=10+10)
Text/Marshal/google_message1_proto2-4        65.0 ± 0%      25.0 ± 0%  -61.54%  (p=0.000 n=10+10)
Text/Marshal/google_message1_proto3-4        59.0 ± 0%      22.0 ± 0%  -62.71%  (p=0.000 n=10+10)
Text/Marshal/google_message2-4              27.4k ± 0%      7.3k ± 0%  -73.39%  (p=0.000 n=10+10)
JSON/Unmarshal/google_message1_proto2-4      95.0 ± 0%      95.0 ± 0%     ~     (all equal)
JSON/Unmarshal/google_message1_proto3-4      74.0 ± 0%      74.0 ± 0%     ~     (all equal)
JSON/Unmarshal/google_message2-4            36.3k ± 0%     36.3k ± 0%     ~     (all equal)
JSON/Marshal/google_message1_proto2-4        27.0 ± 0%      27.0 ± 0%     ~     (all equal)
JSON/Marshal/google_message1_proto3-4        30.0 ± 0%      30.0 ± 0%     ~     (all equal)
JSON/Marshal/google_message2-4              11.3k ± 0%     11.3k ± 0%     ~     (p=1.000 n=10+10)

Change-Id: I377925facde5535f06333b6f25e9c9b358dc062f
Reviewed-on: https://go-review.googlesource.com/c/protobuf/+/204602
Reviewed-by: Joe Tsai <thebrokentoaster@gmail.com>
2020-02-05 02:11:08 +00:00

214 lines
5.9 KiB
Go

// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package defval marshals and unmarshals textual forms of default values.
//
// This package handles both the form historically used in Go struct field tags
// and also the form used by google.protobuf.FieldDescriptorProto.default_value
// since they differ in superficial ways.
package defval
import (
"fmt"
"math"
"strconv"
ptext "google.golang.org/protobuf/internal/encoding/text"
errors "google.golang.org/protobuf/internal/errors"
pref "google.golang.org/protobuf/reflect/protoreflect"
)
// Format is the serialization format used to represent the default value.
type Format int
const (
_ Format = iota
// Descriptor uses the serialization format that protoc uses with the
// google.protobuf.FieldDescriptorProto.default_value field.
Descriptor
// GoTag uses the historical serialization format in Go struct field tags.
GoTag
)
// Unmarshal deserializes the default string s according to the given kind k.
// When k is an enum, a list of enum value descriptors must be provided.
func Unmarshal(s string, k pref.Kind, evs pref.EnumValueDescriptors, f Format) (pref.Value, pref.EnumValueDescriptor, error) {
switch k {
case pref.BoolKind:
if f == GoTag {
switch s {
case "1":
return pref.ValueOfBool(true), nil, nil
case "0":
return pref.ValueOfBool(false), nil, nil
}
} else {
switch s {
case "true":
return pref.ValueOfBool(true), nil, nil
case "false":
return pref.ValueOfBool(false), nil, nil
}
}
case pref.EnumKind:
if f == GoTag {
// Go tags use the numeric form of the enum value.
if n, err := strconv.ParseInt(s, 10, 32); err == nil {
if ev := evs.ByNumber(pref.EnumNumber(n)); ev != nil {
return pref.ValueOfEnum(ev.Number()), ev, nil
}
}
} else {
// Descriptor default_value use the enum identifier.
ev := evs.ByName(pref.Name(s))
if ev != nil {
return pref.ValueOfEnum(ev.Number()), ev, nil
}
}
case pref.Int32Kind, pref.Sint32Kind, pref.Sfixed32Kind:
if v, err := strconv.ParseInt(s, 10, 32); err == nil {
return pref.ValueOfInt32(int32(v)), nil, nil
}
case pref.Int64Kind, pref.Sint64Kind, pref.Sfixed64Kind:
if v, err := strconv.ParseInt(s, 10, 64); err == nil {
return pref.ValueOfInt64(int64(v)), nil, nil
}
case pref.Uint32Kind, pref.Fixed32Kind:
if v, err := strconv.ParseUint(s, 10, 32); err == nil {
return pref.ValueOfUint32(uint32(v)), nil, nil
}
case pref.Uint64Kind, pref.Fixed64Kind:
if v, err := strconv.ParseUint(s, 10, 64); err == nil {
return pref.ValueOfUint64(uint64(v)), nil, nil
}
case pref.FloatKind, pref.DoubleKind:
var v float64
var err error
switch s {
case "-inf":
v = math.Inf(-1)
case "inf":
v = math.Inf(+1)
case "nan":
v = math.NaN()
default:
v, err = strconv.ParseFloat(s, 64)
}
if err == nil {
if k == pref.FloatKind {
return pref.ValueOfFloat32(float32(v)), nil, nil
} else {
return pref.ValueOfFloat64(float64(v)), nil, nil
}
}
case pref.StringKind:
// String values are already unescaped and can be used as is.
return pref.ValueOfString(s), nil, nil
case pref.BytesKind:
if b, ok := unmarshalBytes(s); ok {
return pref.ValueOfBytes(b), nil, nil
}
}
return pref.Value{}, nil, errors.New("could not parse value for %v: %q", k, s)
}
// Marshal serializes v as the default string according to the given kind k.
// When specifying the Descriptor format for an enum kind, the associated
// enum value descriptor must be provided.
func Marshal(v pref.Value, ev pref.EnumValueDescriptor, k pref.Kind, f Format) (string, error) {
switch k {
case pref.BoolKind:
if f == GoTag {
if v.Bool() {
return "1", nil
} else {
return "0", nil
}
} else {
if v.Bool() {
return "true", nil
} else {
return "false", nil
}
}
case pref.EnumKind:
if f == GoTag {
return strconv.FormatInt(int64(v.Enum()), 10), nil
} else {
return string(ev.Name()), nil
}
case pref.Int32Kind, pref.Sint32Kind, pref.Sfixed32Kind, pref.Int64Kind, pref.Sint64Kind, pref.Sfixed64Kind:
return strconv.FormatInt(v.Int(), 10), nil
case pref.Uint32Kind, pref.Fixed32Kind, pref.Uint64Kind, pref.Fixed64Kind:
return strconv.FormatUint(v.Uint(), 10), nil
case pref.FloatKind, pref.DoubleKind:
f := v.Float()
switch {
case math.IsInf(f, -1):
return "-inf", nil
case math.IsInf(f, +1):
return "inf", nil
case math.IsNaN(f):
return "nan", nil
default:
if k == pref.FloatKind {
return strconv.FormatFloat(f, 'g', -1, 32), nil
} else {
return strconv.FormatFloat(f, 'g', -1, 64), nil
}
}
case pref.StringKind:
// String values are serialized as is without any escaping.
return v.String(), nil
case pref.BytesKind:
if s, ok := marshalBytes(v.Bytes()); ok {
return s, nil
}
}
return "", errors.New("could not format value for %v: %v", k, v)
}
// unmarshalBytes deserializes bytes by applying C unescaping.
func unmarshalBytes(s string) ([]byte, bool) {
// Bytes values use the same escaping as the text format,
// however they lack the surrounding double quotes.
v, err := ptext.UnmarshalString(`"` + s + `"`)
if err != nil {
return nil, false
}
return []byte(v), true
}
// marshalBytes serializes bytes by using C escaping.
// To match the exact output of protoc, this is identical to the
// CEscape function in strutil.cc of the protoc source code.
func marshalBytes(b []byte) (string, bool) {
var s []byte
for _, c := range b {
switch c {
case '\n':
s = append(s, `\n`...)
case '\r':
s = append(s, `\r`...)
case '\t':
s = append(s, `\t`...)
case '"':
s = append(s, `\"`...)
case '\'':
s = append(s, `\'`...)
case '\\':
s = append(s, `\\`...)
default:
if printableASCII := c >= 0x20 && c <= 0x7e; printableASCII {
s = append(s, c)
} else {
s = append(s, fmt.Sprintf(`\%03o`, c)...)
}
}
}
return string(s), true
}