protobuf-go/internal/encoding/text/decode_number.go
Herbie Ong 9b3d97c473 encoding/prototext: rewrite of internal/encoding/text
* Fixes golang/protobuf#842. Unmarshal can now parse singular or
  repeated message fields without the field separator.
* Fixes golang/protobuf#1011. Handles negative 0 properly.
* For unknown fields with fixed 32-bit and 64-bit wire types, output is
  now in hex format with 0x prefix similar to C++ lib output. Previous
  Go implementation simply outputs these as decimal numbers %d.
* All parsing errors, except for unexpected EOF should now contain line
  and column number info.
* Fixed following conformance-related features:
  * Parse nan,inf,-inf,infinity,-infinity as case-insensitive.
  * Interpret float32 overflows as inf or -inf.
  * Parse large int-like number as proto float.
* Discard unknown map field if DiscardUnknown=true.
* Allow whitespaces/comments in Any type URL and extension field names per spec.
* Improves performance and memory usage. It is now as fast and efficient as
  protojson, if not better on most benchmarks.

name                                     old time/op    new time/op    delta
Text/Unmarshal/google_message1_proto2-4    14.1µs ±43%     8.7µs ±12%  -38.27%  (p=0.000 n=10+10)
Text/Unmarshal/google_message1_proto3-4    11.6µs ±18%     7.7µs ± 9%  -33.69%  (p=0.000 n=10+10)
Text/Unmarshal/google_message2-4           6.20ms ±27%    4.10ms ± 5%  -33.95%  (p=0.000 n=10+10)
Text/Marshal/google_message1_proto2-4      12.8µs ± 6%    10.3µs ±23%  -19.54%  (p=0.000 n=9+10)
Text/Marshal/google_message1_proto3-4      11.9µs ±16%     8.6µs ±10%  -27.45%  (p=0.000 n=10+10)
Text/Marshal/google_message2-4             5.59ms ± 5%    5.30ms ±22%     ~     (p=0.356 n=9+10)
JSON/Unmarshal/google_message1_proto2-4    12.3µs ±61%    13.9µs ±26%     ~     (p=0.190 n=10+10)
JSON/Unmarshal/google_message1_proto3-4    7.51µs ± 6%    7.86µs ± 1%   +4.66%  (p=0.010 n=10+9)
JSON/Unmarshal/google_message2-4           3.74ms ± 2%    3.94ms ± 2%   +5.32%  (p=0.000 n=10+10)
JSON/Marshal/google_message1_proto2-4      9.90µs ±12%    9.95µs ± 4%     ~     (p=0.315 n=9+10)
JSON/Marshal/google_message1_proto3-4      7.55µs ± 4%    7.93µs ± 3%   +4.98%  (p=0.000 n=10+10)
JSON/Marshal/google_message2-4             4.29ms ± 5%    4.49ms ± 2%   +4.53%  (p=0.001 n=10+10)

name                                     old alloc/op   new alloc/op   delta
Text/Unmarshal/google_message1_proto2-4    12.5kB ± 0%     2.0kB ± 0%  -83.87%  (p=0.000 n=10+10)
Text/Unmarshal/google_message1_proto3-4    12.2kB ± 0%     1.8kB ± 0%  -85.33%  (p=0.000 n=10+10)
Text/Unmarshal/google_message2-4           5.35MB ± 0%    0.89MB ± 0%  -83.28%  (p=0.000 n=10+9)
Text/Marshal/google_message1_proto2-4      12.0kB ± 0%     1.4kB ± 0%  -88.15%  (p=0.000 n=10+10)
Text/Marshal/google_message1_proto3-4      12.4kB ± 0%     1.9kB ± 0%  -84.91%  (p=0.000 n=10+10)
Text/Marshal/google_message2-4             5.64MB ± 0%    1.02MB ± 0%  -81.85%  (p=0.000 n=10+9)
JSON/Unmarshal/google_message1_proto2-4    2.29kB ± 0%    2.29kB ± 0%     ~     (all equal)
JSON/Unmarshal/google_message1_proto3-4    2.08kB ± 0%    2.08kB ± 0%     ~     (all equal)
JSON/Unmarshal/google_message2-4            899kB ± 0%     899kB ± 0%     ~     (p=1.000 n=10+10)
JSON/Marshal/google_message1_proto2-4      1.46kB ± 0%    1.46kB ± 0%     ~     (all equal)
JSON/Marshal/google_message1_proto3-4      1.36kB ± 0%    1.36kB ± 0%     ~     (all equal)
JSON/Marshal/google_message2-4             1.19MB ± 0%    1.19MB ± 0%     ~     (p=0.197 n=10+10)

name                                     old allocs/op  new allocs/op  delta
Text/Unmarshal/google_message1_proto2-4       133 ± 0%        89 ± 0%  -33.08%  (p=0.000 n=10+10)
Text/Unmarshal/google_message1_proto3-4       108 ± 0%        67 ± 0%  -37.96%  (p=0.000 n=10+10)
Text/Unmarshal/google_message2-4            60.0k ± 0%     38.7k ± 0%  -35.52%  (p=0.000 n=10+10)
Text/Marshal/google_message1_proto2-4        65.0 ± 0%      25.0 ± 0%  -61.54%  (p=0.000 n=10+10)
Text/Marshal/google_message1_proto3-4        59.0 ± 0%      22.0 ± 0%  -62.71%  (p=0.000 n=10+10)
Text/Marshal/google_message2-4              27.4k ± 0%      7.3k ± 0%  -73.39%  (p=0.000 n=10+10)
JSON/Unmarshal/google_message1_proto2-4      95.0 ± 0%      95.0 ± 0%     ~     (all equal)
JSON/Unmarshal/google_message1_proto3-4      74.0 ± 0%      74.0 ± 0%     ~     (all equal)
JSON/Unmarshal/google_message2-4            36.3k ± 0%     36.3k ± 0%     ~     (all equal)
JSON/Marshal/google_message1_proto2-4        27.0 ± 0%      27.0 ± 0%     ~     (all equal)
JSON/Marshal/google_message1_proto3-4        30.0 ± 0%      30.0 ± 0%     ~     (all equal)
JSON/Marshal/google_message2-4              11.3k ± 0%     11.3k ± 0%     ~     (p=1.000 n=10+10)

Change-Id: I377925facde5535f06333b6f25e9c9b358dc062f
Reviewed-on: https://go-review.googlesource.com/c/protobuf/+/204602
Reviewed-by: Joe Tsai <thebrokentoaster@gmail.com>
2020-02-05 02:11:08 +00:00

191 lines
3.9 KiB
Go

// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package text
// parseNumberValue parses a number from the input and returns a Token object.
func (d *Decoder) parseNumberValue() (Token, bool) {
in := d.in
num := parseNumber(in)
if num.size == 0 {
return Token{}, false
}
numAttrs := num.kind
if num.neg {
numAttrs |= isNegative
}
strSize := num.size
last := num.size - 1
if num.kind == numFloat && (d.in[last] == 'f' || d.in[last] == 'F') {
strSize = last
}
tok := Token{
kind: Scalar,
attrs: numberValue,
pos: len(d.orig) - len(d.in),
raw: d.in[:num.size],
str: string(d.in[:strSize]),
numAttrs: numAttrs,
}
d.consume(num.size)
return tok, true
}
const (
numDec uint8 = (1 << iota) / 2
numHex
numOct
numFloat
)
// number is the result of parsing out a valid number from parseNumber. It
// contains data for doing float or integer conversion via the strconv package
// in conjunction with the input bytes.
type number struct {
kind uint8
neg bool
size int
}
// parseNumber constructs a number object from given input. It allows for the
// following patterns:
// integer: ^-?([1-9][0-9]*|0[xX][0-9a-fA-F]+|0[0-7]*)
// float: ^-?((0|[1-9][0-9]*)?([.][0-9]*)?([eE][+-]?[0-9]+)?[fF]?)
// It also returns the number of parsed bytes for the given number, 0 if it is
// not a number.
func parseNumber(input []byte) number {
kind := numDec
var size int
var neg bool
s := input
if len(s) == 0 {
return number{}
}
// Optional -
if s[0] == '-' {
neg = true
s = s[1:]
size++
if len(s) == 0 {
return number{}
}
}
// C++ allows for whitespace and comments in between the negative sign and
// the rest of the number. This logic currently does not but is consistent
// with v1.
switch {
case s[0] == '0':
if len(s) > 1 {
switch {
case s[1] == 'x' || s[1] == 'X':
// Parse as hex number.
kind = numHex
n := 2
s = s[2:]
for len(s) > 0 && (('0' <= s[0] && s[0] <= '9') ||
('a' <= s[0] && s[0] <= 'f') ||
('A' <= s[0] && s[0] <= 'F')) {
s = s[1:]
n++
}
if n == 2 {
return number{}
}
size += n
case '0' <= s[1] && s[1] <= '7':
// Parse as octal number.
kind = numOct
n := 2
s = s[2:]
for len(s) > 0 && '0' <= s[0] && s[0] <= '7' {
s = s[1:]
n++
}
size += n
}
if kind&(numHex|numOct) > 0 {
if len(s) > 0 && !isDelim(s[0]) {
return number{}
}
return number{kind: kind, neg: neg, size: size}
}
}
s = s[1:]
size++
case '1' <= s[0] && s[0] <= '9':
n := 1
s = s[1:]
for len(s) > 0 && '0' <= s[0] && s[0] <= '9' {
s = s[1:]
n++
}
size += n
case s[0] == '.':
// Set kind to numFloat to signify the intent to parse as float. And
// that it needs to have other digits after '.'.
kind = numFloat
default:
return number{}
}
// . followed by 0 or more digits.
if len(s) > 0 && s[0] == '.' {
n := 1
s = s[1:]
// If decimal point was before any digits, it should be followed by
// other digits.
if len(s) == 0 && kind == numFloat {
return number{}
}
for len(s) > 0 && '0' <= s[0] && s[0] <= '9' {
s = s[1:]
n++
}
size += n
kind = numFloat
}
// e or E followed by an optional - or + and 1 or more digits.
if len(s) >= 2 && (s[0] == 'e' || s[0] == 'E') {
kind = numFloat
s = s[1:]
n := 1
if s[0] == '+' || s[0] == '-' {
s = s[1:]
n++
if len(s) == 0 {
return number{}
}
}
for len(s) > 0 && '0' <= s[0] && s[0] <= '9' {
s = s[1:]
n++
}
size += n
}
// Optional suffix f or F for floats.
if len(s) > 0 && (s[0] == 'f' || s[0] == 'F') {
kind = numFloat
s = s[1:]
size++
}
// Check that next byte is a delimiter or it is at the end.
if len(s) > 0 && !isDelim(s[0]) {
return number{}
}
return number{kind: kind, neg: neg, size: size}
}