mirror of
https://github.com/protocolbuffers/protobuf-go.git
synced 2025-01-10 07:05:11 +00:00
9b3d97c473
* Fixes golang/protobuf#842. Unmarshal can now parse singular or repeated message fields without the field separator. * Fixes golang/protobuf#1011. Handles negative 0 properly. * For unknown fields with fixed 32-bit and 64-bit wire types, output is now in hex format with 0x prefix similar to C++ lib output. Previous Go implementation simply outputs these as decimal numbers %d. * All parsing errors, except for unexpected EOF should now contain line and column number info. * Fixed following conformance-related features: * Parse nan,inf,-inf,infinity,-infinity as case-insensitive. * Interpret float32 overflows as inf or -inf. * Parse large int-like number as proto float. * Discard unknown map field if DiscardUnknown=true. * Allow whitespaces/comments in Any type URL and extension field names per spec. * Improves performance and memory usage. It is now as fast and efficient as protojson, if not better on most benchmarks. name old time/op new time/op delta Text/Unmarshal/google_message1_proto2-4 14.1µs ±43% 8.7µs ±12% -38.27% (p=0.000 n=10+10) Text/Unmarshal/google_message1_proto3-4 11.6µs ±18% 7.7µs ± 9% -33.69% (p=0.000 n=10+10) Text/Unmarshal/google_message2-4 6.20ms ±27% 4.10ms ± 5% -33.95% (p=0.000 n=10+10) Text/Marshal/google_message1_proto2-4 12.8µs ± 6% 10.3µs ±23% -19.54% (p=0.000 n=9+10) Text/Marshal/google_message1_proto3-4 11.9µs ±16% 8.6µs ±10% -27.45% (p=0.000 n=10+10) Text/Marshal/google_message2-4 5.59ms ± 5% 5.30ms ±22% ~ (p=0.356 n=9+10) JSON/Unmarshal/google_message1_proto2-4 12.3µs ±61% 13.9µs ±26% ~ (p=0.190 n=10+10) JSON/Unmarshal/google_message1_proto3-4 7.51µs ± 6% 7.86µs ± 1% +4.66% (p=0.010 n=10+9) JSON/Unmarshal/google_message2-4 3.74ms ± 2% 3.94ms ± 2% +5.32% (p=0.000 n=10+10) JSON/Marshal/google_message1_proto2-4 9.90µs ±12% 9.95µs ± 4% ~ (p=0.315 n=9+10) JSON/Marshal/google_message1_proto3-4 7.55µs ± 4% 7.93µs ± 3% +4.98% (p=0.000 n=10+10) JSON/Marshal/google_message2-4 4.29ms ± 5% 4.49ms ± 2% +4.53% (p=0.001 n=10+10) name old alloc/op new alloc/op delta Text/Unmarshal/google_message1_proto2-4 12.5kB ± 0% 2.0kB ± 0% -83.87% (p=0.000 n=10+10) Text/Unmarshal/google_message1_proto3-4 12.2kB ± 0% 1.8kB ± 0% -85.33% (p=0.000 n=10+10) Text/Unmarshal/google_message2-4 5.35MB ± 0% 0.89MB ± 0% -83.28% (p=0.000 n=10+9) Text/Marshal/google_message1_proto2-4 12.0kB ± 0% 1.4kB ± 0% -88.15% (p=0.000 n=10+10) Text/Marshal/google_message1_proto3-4 12.4kB ± 0% 1.9kB ± 0% -84.91% (p=0.000 n=10+10) Text/Marshal/google_message2-4 5.64MB ± 0% 1.02MB ± 0% -81.85% (p=0.000 n=10+9) JSON/Unmarshal/google_message1_proto2-4 2.29kB ± 0% 2.29kB ± 0% ~ (all equal) JSON/Unmarshal/google_message1_proto3-4 2.08kB ± 0% 2.08kB ± 0% ~ (all equal) JSON/Unmarshal/google_message2-4 899kB ± 0% 899kB ± 0% ~ (p=1.000 n=10+10) JSON/Marshal/google_message1_proto2-4 1.46kB ± 0% 1.46kB ± 0% ~ (all equal) JSON/Marshal/google_message1_proto3-4 1.36kB ± 0% 1.36kB ± 0% ~ (all equal) JSON/Marshal/google_message2-4 1.19MB ± 0% 1.19MB ± 0% ~ (p=0.197 n=10+10) name old allocs/op new allocs/op delta Text/Unmarshal/google_message1_proto2-4 133 ± 0% 89 ± 0% -33.08% (p=0.000 n=10+10) Text/Unmarshal/google_message1_proto3-4 108 ± 0% 67 ± 0% -37.96% (p=0.000 n=10+10) Text/Unmarshal/google_message2-4 60.0k ± 0% 38.7k ± 0% -35.52% (p=0.000 n=10+10) Text/Marshal/google_message1_proto2-4 65.0 ± 0% 25.0 ± 0% -61.54% (p=0.000 n=10+10) Text/Marshal/google_message1_proto3-4 59.0 ± 0% 22.0 ± 0% -62.71% (p=0.000 n=10+10) Text/Marshal/google_message2-4 27.4k ± 0% 7.3k ± 0% -73.39% (p=0.000 n=10+10) JSON/Unmarshal/google_message1_proto2-4 95.0 ± 0% 95.0 ± 0% ~ (all equal) JSON/Unmarshal/google_message1_proto3-4 74.0 ± 0% 74.0 ± 0% ~ (all equal) JSON/Unmarshal/google_message2-4 36.3k ± 0% 36.3k ± 0% ~ (all equal) JSON/Marshal/google_message1_proto2-4 27.0 ± 0% 27.0 ± 0% ~ (all equal) JSON/Marshal/google_message1_proto3-4 30.0 ± 0% 30.0 ± 0% ~ (all equal) JSON/Marshal/google_message2-4 11.3k ± 0% 11.3k ± 0% ~ (p=1.000 n=10+10) Change-Id: I377925facde5535f06333b6f25e9c9b358dc062f Reviewed-on: https://go-review.googlesource.com/c/protobuf/+/204602 Reviewed-by: Joe Tsai <thebrokentoaster@gmail.com>
191 lines
3.9 KiB
Go
191 lines
3.9 KiB
Go
// Copyright 2018 The Go Authors. All rights reserved.
|
|
// Use of this source code is governed by a BSD-style
|
|
// license that can be found in the LICENSE file.
|
|
|
|
package text
|
|
|
|
// parseNumberValue parses a number from the input and returns a Token object.
|
|
func (d *Decoder) parseNumberValue() (Token, bool) {
|
|
in := d.in
|
|
num := parseNumber(in)
|
|
if num.size == 0 {
|
|
return Token{}, false
|
|
}
|
|
numAttrs := num.kind
|
|
if num.neg {
|
|
numAttrs |= isNegative
|
|
}
|
|
strSize := num.size
|
|
last := num.size - 1
|
|
if num.kind == numFloat && (d.in[last] == 'f' || d.in[last] == 'F') {
|
|
strSize = last
|
|
}
|
|
tok := Token{
|
|
kind: Scalar,
|
|
attrs: numberValue,
|
|
pos: len(d.orig) - len(d.in),
|
|
raw: d.in[:num.size],
|
|
str: string(d.in[:strSize]),
|
|
numAttrs: numAttrs,
|
|
}
|
|
d.consume(num.size)
|
|
return tok, true
|
|
}
|
|
|
|
const (
|
|
numDec uint8 = (1 << iota) / 2
|
|
numHex
|
|
numOct
|
|
numFloat
|
|
)
|
|
|
|
// number is the result of parsing out a valid number from parseNumber. It
|
|
// contains data for doing float or integer conversion via the strconv package
|
|
// in conjunction with the input bytes.
|
|
type number struct {
|
|
kind uint8
|
|
neg bool
|
|
size int
|
|
}
|
|
|
|
// parseNumber constructs a number object from given input. It allows for the
|
|
// following patterns:
|
|
// integer: ^-?([1-9][0-9]*|0[xX][0-9a-fA-F]+|0[0-7]*)
|
|
// float: ^-?((0|[1-9][0-9]*)?([.][0-9]*)?([eE][+-]?[0-9]+)?[fF]?)
|
|
// It also returns the number of parsed bytes for the given number, 0 if it is
|
|
// not a number.
|
|
func parseNumber(input []byte) number {
|
|
kind := numDec
|
|
var size int
|
|
var neg bool
|
|
|
|
s := input
|
|
if len(s) == 0 {
|
|
return number{}
|
|
}
|
|
|
|
// Optional -
|
|
if s[0] == '-' {
|
|
neg = true
|
|
s = s[1:]
|
|
size++
|
|
if len(s) == 0 {
|
|
return number{}
|
|
}
|
|
}
|
|
|
|
// C++ allows for whitespace and comments in between the negative sign and
|
|
// the rest of the number. This logic currently does not but is consistent
|
|
// with v1.
|
|
|
|
switch {
|
|
case s[0] == '0':
|
|
if len(s) > 1 {
|
|
switch {
|
|
case s[1] == 'x' || s[1] == 'X':
|
|
// Parse as hex number.
|
|
kind = numHex
|
|
n := 2
|
|
s = s[2:]
|
|
for len(s) > 0 && (('0' <= s[0] && s[0] <= '9') ||
|
|
('a' <= s[0] && s[0] <= 'f') ||
|
|
('A' <= s[0] && s[0] <= 'F')) {
|
|
s = s[1:]
|
|
n++
|
|
}
|
|
if n == 2 {
|
|
return number{}
|
|
}
|
|
size += n
|
|
|
|
case '0' <= s[1] && s[1] <= '7':
|
|
// Parse as octal number.
|
|
kind = numOct
|
|
n := 2
|
|
s = s[2:]
|
|
for len(s) > 0 && '0' <= s[0] && s[0] <= '7' {
|
|
s = s[1:]
|
|
n++
|
|
}
|
|
size += n
|
|
}
|
|
|
|
if kind&(numHex|numOct) > 0 {
|
|
if len(s) > 0 && !isDelim(s[0]) {
|
|
return number{}
|
|
}
|
|
return number{kind: kind, neg: neg, size: size}
|
|
}
|
|
}
|
|
s = s[1:]
|
|
size++
|
|
|
|
case '1' <= s[0] && s[0] <= '9':
|
|
n := 1
|
|
s = s[1:]
|
|
for len(s) > 0 && '0' <= s[0] && s[0] <= '9' {
|
|
s = s[1:]
|
|
n++
|
|
}
|
|
size += n
|
|
|
|
case s[0] == '.':
|
|
// Set kind to numFloat to signify the intent to parse as float. And
|
|
// that it needs to have other digits after '.'.
|
|
kind = numFloat
|
|
|
|
default:
|
|
return number{}
|
|
}
|
|
|
|
// . followed by 0 or more digits.
|
|
if len(s) > 0 && s[0] == '.' {
|
|
n := 1
|
|
s = s[1:]
|
|
// If decimal point was before any digits, it should be followed by
|
|
// other digits.
|
|
if len(s) == 0 && kind == numFloat {
|
|
return number{}
|
|
}
|
|
for len(s) > 0 && '0' <= s[0] && s[0] <= '9' {
|
|
s = s[1:]
|
|
n++
|
|
}
|
|
size += n
|
|
kind = numFloat
|
|
}
|
|
|
|
// e or E followed by an optional - or + and 1 or more digits.
|
|
if len(s) >= 2 && (s[0] == 'e' || s[0] == 'E') {
|
|
kind = numFloat
|
|
s = s[1:]
|
|
n := 1
|
|
if s[0] == '+' || s[0] == '-' {
|
|
s = s[1:]
|
|
n++
|
|
if len(s) == 0 {
|
|
return number{}
|
|
}
|
|
}
|
|
for len(s) > 0 && '0' <= s[0] && s[0] <= '9' {
|
|
s = s[1:]
|
|
n++
|
|
}
|
|
size += n
|
|
}
|
|
|
|
// Optional suffix f or F for floats.
|
|
if len(s) > 0 && (s[0] == 'f' || s[0] == 'F') {
|
|
kind = numFloat
|
|
s = s[1:]
|
|
size++
|
|
}
|
|
|
|
// Check that next byte is a delimiter or it is at the end.
|
|
if len(s) > 0 && !isDelim(s[0]) {
|
|
return number{}
|
|
}
|
|
|
|
return number{kind: kind, neg: neg, size: size}
|
|
}
|