From 97a87391b1eee9e69e20524a4fa47ed80ab0f67d Mon Sep 17 00:00:00 2001 From: Joe Tsai Date: Sun, 7 Jul 2019 01:49:59 -0700 Subject: [PATCH] internal/strs: unify string manipulation functionality Create a new internal/strs package that unifies common functionality: * Since protobuf itself pseudo-specifies at least 4 different camel-case and snake-case conversion functions, we define all variants in one place. * We move the internal/filedesc.nameBuilder function to this package. We simplify its implementation to not depend on a strings.Builder fork under the hood since the semantics we desire is simpler than what strings.Builder provides. * We use strs.Builder in reflect/protodesc in its construction of all the full names. This is perfect use case of strs.Builder since all full names within a file descriptor share the same lifetime. * Add an UnsafeString and UnsafeBytes cast function that will be useful in the near future for optimizing encoding/prototext and encoding/protojson. Change-Id: I2cf07cbaf6f72e5f9fd6ae3d37b0d46f6af2ad59 Reviewed-on: https://go-review.googlesource.com/c/protobuf/+/185198 Reviewed-by: Damien Neil --- encoding/protojson/well_known_types.go | 53 +---------- internal/filedesc/desc.go | 21 +---- internal/filedesc/desc_init.go | 69 +++++++++----- internal/filedesc/desc_lazy.go | 69 +++++++------- internal/filedesc/name_pure.go | 34 ------- internal/filedesc/name_unsafe.go | 124 ------------------------- internal/impl/legacy_message.go | 3 +- internal/strs/strings.go | 111 ++++++++++++++++++++++ internal/strs/strings_pure.go | 27 ++++++ internal/strs/strings_test.go | 108 +++++++++++++++++++++ internal/strs/strings_unsafe.go | 94 +++++++++++++++++++ reflect/protodesc/desc.go | 10 +- reflect/protodesc/desc_init.go | 53 +++++------ reflect/protodesc/desc_validate.go | 91 +----------------- reflect/protodesc/file_test.go | 55 ----------- 15 files changed, 468 insertions(+), 454 deletions(-) delete mode 100644 internal/filedesc/name_pure.go delete mode 100644 internal/filedesc/name_unsafe.go create mode 100644 internal/strs/strings.go create mode 100644 internal/strs/strings_pure.go create mode 100644 internal/strs/strings_test.go create mode 100644 internal/strs/strings_unsafe.go diff --git a/encoding/protojson/well_known_types.go b/encoding/protojson/well_known_types.go index c0291b6a..77833d84 100644 --- a/encoding/protojson/well_known_types.go +++ b/encoding/protojson/well_known_types.go @@ -14,6 +14,7 @@ import ( "google.golang.org/protobuf/internal/encoding/json" "google.golang.org/protobuf/internal/errors" "google.golang.org/protobuf/internal/fieldnum" + "google.golang.org/protobuf/internal/strs" "google.golang.org/protobuf/proto" pref "google.golang.org/protobuf/reflect/protoreflect" ) @@ -888,8 +889,8 @@ func (o MarshalOptions) marshalFieldMask(m pref.Message) error { for i := 0; i < list.Len(); i++ { s := list.Get(i).String() // Return error if conversion to camelCase is not reversible. - cc := camelCase(s) - if s != snakeCase(cc) { + cc := strs.JSONCamelCase(s) + if s != strs.JSONSnakeCase(cc) { return errors.New("%s.paths contains irreversible value %q", m.Descriptor().FullName(), s) } paths = append(paths, cc) @@ -920,53 +921,7 @@ func (o UnmarshalOptions) unmarshalFieldMask(m pref.Message) error { s = strings.TrimSpace(s) // Convert to snake_case. Unlike encoding, no validation is done because // it is not possible to know the original path names. - list.Append(pref.ValueOf(snakeCase(s))) + list.Append(pref.ValueOf(strs.JSONSnakeCase(s))) } return nil } - -// camelCase converts given string into camelCase where ASCII character after _ -// is turned into uppercase and _'s are removed. -func camelCase(s string) string { - var b []byte - var afterUnderscore bool - for i := 0; i < len(s); i++ { - c := s[i] - if afterUnderscore { - if isASCIILower(c) { - c -= 'a' - 'A' - } - } - if c == '_' { - afterUnderscore = true - continue - } - afterUnderscore = false - b = append(b, c) - } - return string(b) -} - -// snakeCase converts given string into snake_case where ASCII uppercase -// character is turned into _ + lowercase. -func snakeCase(s string) string { - var b []byte - for i := 0; i < len(s); i++ { - c := s[i] - if isASCIIUpper(c) { - c += 'a' - 'A' - b = append(b, '_', c) - } else { - b = append(b, c) - } - } - return string(b) -} - -func isASCIILower(c byte) bool { - return 'a' <= c && c <= 'z' -} - -func isASCIIUpper(c byte) bool { - return 'A' <= c && c <= 'Z' -} diff --git a/internal/filedesc/desc.go b/internal/filedesc/desc.go index 84c90fa4..20ffd83b 100644 --- a/internal/filedesc/desc.go +++ b/internal/filedesc/desc.go @@ -14,6 +14,7 @@ import ( "google.golang.org/protobuf/internal/descopts" "google.golang.org/protobuf/internal/encoding/defval" "google.golang.org/protobuf/internal/pragma" + "google.golang.org/protobuf/internal/strs" pref "google.golang.org/protobuf/reflect/protoreflect" ) @@ -465,30 +466,12 @@ type jsonName struct { func (js *jsonName) get(fd pref.FieldDescriptor) string { if !js.has { js.once.Do(func() { - js.name = makeJSONName(fd.Name()) + js.name = strs.JSONCamelCase(string(fd.Name())) }) } return js.name } -// makeJSONName creates a JSON name from the protobuf short name. -func makeJSONName(s pref.Name) string { - var b []byte - var wasUnderscore bool - for i := 0; i < len(s); i++ { // proto identifiers are always ASCII - c := s[i] - if c != '_' { - isLower := 'a' <= c && c <= 'z' - if wasUnderscore && isLower { - c -= 'a' - 'A' - } - b = append(b, c) - } - wasUnderscore = c == '_' - } - return string(b) -} - func DefaultValue(v pref.Value, ev pref.EnumValueDescriptor) defaultValue { dv := defaultValue{has: v.IsValid(), val: v, enum: ev} if b, ok := v.Interface().([]byte); ok { diff --git a/internal/filedesc/desc_init.go b/internal/filedesc/desc_init.go index a51a7eca..0bd7d380 100644 --- a/internal/filedesc/desc_init.go +++ b/internal/filedesc/desc_init.go @@ -5,8 +5,11 @@ package filedesc import ( + "sync" + "google.golang.org/protobuf/internal/encoding/wire" "google.golang.org/protobuf/internal/fieldnum" + "google.golang.org/protobuf/internal/strs" pref "google.golang.org/protobuf/reflect/protoreflect" ) @@ -89,8 +92,8 @@ func (fd *File) checkDecls() { } func (fd *File) unmarshalSeed(b []byte) { - nb := getNameBuilder() - defer putNameBuilder(nb) + sb := getBuilder() + defer putBuilder(sb) var prevField pref.FieldNumber var numEnums, numMessages, numExtensions, numServices int @@ -114,9 +117,9 @@ func (fd *File) unmarshalSeed(b []byte) { panic("invalid syntax") } case fieldnum.FileDescriptorProto_Name: - fd.L1.Path = nb.MakeString(v) + fd.L1.Path = sb.MakeString(v) case fieldnum.FileDescriptorProto_Package: - fd.L1.Package = pref.FullName(nb.MakeString(v)) + fd.L1.Package = pref.FullName(sb.MakeString(v)) case fieldnum.FileDescriptorProto_EnumType: if prevField != fieldnum.FileDescriptorProto_EnumType { if numEnums > 0 { @@ -183,7 +186,7 @@ func (fd *File) unmarshalSeed(b []byte) { for i := range fd.L1.Enums.List { _, n := wire.ConsumeVarint(b) v, m := wire.ConsumeBytes(b[n:]) - fd.L1.Enums.List[i].unmarshalSeed(v, nb, fd, fd, i) + fd.L1.Enums.List[i].unmarshalSeed(v, sb, fd, fd, i) b = b[n+m:] } } @@ -192,7 +195,7 @@ func (fd *File) unmarshalSeed(b []byte) { for i := range fd.L1.Messages.List { _, n := wire.ConsumeVarint(b) v, m := wire.ConsumeBytes(b[n:]) - fd.L1.Messages.List[i].unmarshalSeed(v, nb, fd, fd, i) + fd.L1.Messages.List[i].unmarshalSeed(v, sb, fd, fd, i) b = b[n+m:] } } @@ -201,7 +204,7 @@ func (fd *File) unmarshalSeed(b []byte) { for i := range fd.L1.Extensions.List { _, n := wire.ConsumeVarint(b) v, m := wire.ConsumeBytes(b[n:]) - fd.L1.Extensions.List[i].unmarshalSeed(v, nb, fd, fd, i) + fd.L1.Extensions.List[i].unmarshalSeed(v, sb, fd, fd, i) b = b[n+m:] } } @@ -210,13 +213,13 @@ func (fd *File) unmarshalSeed(b []byte) { for i := range fd.L1.Services.List { _, n := wire.ConsumeVarint(b) v, m := wire.ConsumeBytes(b[n:]) - fd.L1.Services.List[i].unmarshalSeed(v, nb, fd, fd, i) + fd.L1.Services.List[i].unmarshalSeed(v, sb, fd, fd, i) b = b[n+m:] } } } -func (ed *Enum) unmarshalSeed(b []byte, nb *nameBuilder, pf *File, pd pref.Descriptor, i int) { +func (ed *Enum) unmarshalSeed(b []byte, sb *strs.Builder, pf *File, pd pref.Descriptor, i int) { ed.L0.ParentFile = pf ed.L0.Parent = pd ed.L0.Index = i @@ -231,7 +234,7 @@ func (ed *Enum) unmarshalSeed(b []byte, nb *nameBuilder, pf *File, pd pref.Descr b = b[m:] switch num { case fieldnum.EnumDescriptorProto_Name: - ed.L0.FullName = nb.AppendFullName(pd.FullName(), v) + ed.L0.FullName = appendFullName(sb, pd.FullName(), v) case fieldnum.EnumDescriptorProto_Value: numValues++ } @@ -258,7 +261,7 @@ func (ed *Enum) unmarshalSeed(b []byte, nb *nameBuilder, pf *File, pd pref.Descr b = b[m:] switch num { case fieldnum.EnumDescriptorProto_Value: - ed.L2.Values.List[i].unmarshalFull(v, nb, pf, ed, i) + ed.L2.Values.List[i].unmarshalFull(v, sb, pf, ed, i) i++ } default: @@ -268,7 +271,7 @@ func (ed *Enum) unmarshalSeed(b []byte, nb *nameBuilder, pf *File, pd pref.Descr } } -func (md *Message) unmarshalSeed(b []byte, nb *nameBuilder, pf *File, pd pref.Descriptor, i int) { +func (md *Message) unmarshalSeed(b []byte, sb *strs.Builder, pf *File, pd pref.Descriptor, i int) { md.L0.ParentFile = pf md.L0.Parent = pd md.L0.Index = i @@ -286,7 +289,7 @@ func (md *Message) unmarshalSeed(b []byte, nb *nameBuilder, pf *File, pd pref.De b = b[m:] switch num { case fieldnum.DescriptorProto_Name: - md.L0.FullName = nb.AppendFullName(pd.FullName(), v) + md.L0.FullName = appendFullName(sb, pd.FullName(), v) case fieldnum.DescriptorProto_EnumType: if prevField != fieldnum.DescriptorProto_EnumType { if numEnums > 0 { @@ -337,7 +340,7 @@ func (md *Message) unmarshalSeed(b []byte, nb *nameBuilder, pf *File, pd pref.De for i := range md.L1.Enums.List { _, n := wire.ConsumeVarint(b) v, m := wire.ConsumeBytes(b[n:]) - md.L1.Enums.List[i].unmarshalSeed(v, nb, pf, md, i) + md.L1.Enums.List[i].unmarshalSeed(v, sb, pf, md, i) b = b[n+m:] } } @@ -346,7 +349,7 @@ func (md *Message) unmarshalSeed(b []byte, nb *nameBuilder, pf *File, pd pref.De for i := range md.L1.Messages.List { _, n := wire.ConsumeVarint(b) v, m := wire.ConsumeBytes(b[n:]) - md.L1.Messages.List[i].unmarshalSeed(v, nb, pf, md, i) + md.L1.Messages.List[i].unmarshalSeed(v, sb, pf, md, i) b = b[n+m:] } } @@ -355,13 +358,13 @@ func (md *Message) unmarshalSeed(b []byte, nb *nameBuilder, pf *File, pd pref.De for i := range md.L1.Extensions.List { _, n := wire.ConsumeVarint(b) v, m := wire.ConsumeBytes(b[n:]) - md.L1.Extensions.List[i].unmarshalSeed(v, nb, pf, md, i) + md.L1.Extensions.List[i].unmarshalSeed(v, sb, pf, md, i) b = b[n+m:] } } } -func (xd *Extension) unmarshalSeed(b []byte, nb *nameBuilder, pf *File, pd pref.Descriptor, i int) { +func (xd *Extension) unmarshalSeed(b []byte, sb *strs.Builder, pf *File, pd pref.Descriptor, i int) { xd.L0.ParentFile = pf xd.L0.Parent = pd xd.L0.Index = i @@ -384,9 +387,9 @@ func (xd *Extension) unmarshalSeed(b []byte, nb *nameBuilder, pf *File, pd pref. b = b[m:] switch num { case fieldnum.FieldDescriptorProto_Name: - xd.L0.FullName = nb.AppendFullName(pd.FullName(), v) + xd.L0.FullName = appendFullName(sb, pd.FullName(), v) case fieldnum.FieldDescriptorProto_Extendee: - xd.L1.Extendee = PlaceholderMessage(nb.MakeFullName(v)) + xd.L1.Extendee = PlaceholderMessage(makeFullName(sb, v)) } default: m := wire.ConsumeFieldValue(num, typ, b) @@ -395,7 +398,7 @@ func (xd *Extension) unmarshalSeed(b []byte, nb *nameBuilder, pf *File, pd pref. } } -func (sd *Service) unmarshalSeed(b []byte, nb *nameBuilder, pf *File, pd pref.Descriptor, i int) { +func (sd *Service) unmarshalSeed(b []byte, sb *strs.Builder, pf *File, pd pref.Descriptor, i int) { sd.L0.ParentFile = pf sd.L0.Parent = pd sd.L0.Index = i @@ -409,7 +412,7 @@ func (sd *Service) unmarshalSeed(b []byte, nb *nameBuilder, pf *File, pd pref.De b = b[m:] switch num { case fieldnum.ServiceDescriptorProto_Name: - sd.L0.FullName = nb.AppendFullName(pd.FullName(), v) + sd.L0.FullName = appendFullName(sb, pd.FullName(), v) } default: m := wire.ConsumeFieldValue(num, typ, b) @@ -417,3 +420,27 @@ func (sd *Service) unmarshalSeed(b []byte, nb *nameBuilder, pf *File, pd pref.De } } } + +var nameBuilderPool = sync.Pool{ + New: func() interface{} { return new(strs.Builder) }, +} + +func getBuilder() *strs.Builder { + return nameBuilderPool.Get().(*strs.Builder) +} +func putBuilder(b *strs.Builder) { + nameBuilderPool.Put(b) +} + +// makeFullName converts b to a protoreflect.FullName, +// where b must start with a leading dot. +func makeFullName(sb *strs.Builder, b []byte) pref.FullName { + if len(b) == 0 || b[0] != '.' { + panic("name reference must be fully qualified") + } + return pref.FullName(sb.MakeString(b[1:])) +} + +func appendFullName(sb *strs.Builder, prefix pref.FullName, suffix []byte) pref.FullName { + return sb.AppendFullName(prefix, pref.Name(strs.UnsafeString(suffix))) +} diff --git a/internal/filedesc/desc_lazy.go b/internal/filedesc/desc_lazy.go index 4d311096..55104ad5 100644 --- a/internal/filedesc/desc_lazy.go +++ b/internal/filedesc/desc_lazy.go @@ -11,6 +11,7 @@ import ( "google.golang.org/protobuf/internal/descopts" "google.golang.org/protobuf/internal/encoding/wire" "google.golang.org/protobuf/internal/fieldnum" + "google.golang.org/protobuf/internal/strs" "google.golang.org/protobuf/proto" pref "google.golang.org/protobuf/reflect/protoreflect" ) @@ -132,8 +133,8 @@ func (file *File) resolveMessageDependency(md pref.MessageDescriptor, i, j int32 } func (fd *File) unmarshalFull(b []byte) { - nb := getNameBuilder() - defer putNameBuilder(nb) + sb := getBuilder() + defer putBuilder(sb) var enumIdx, messageIdx, extensionIdx, serviceIdx int var rawOptions []byte @@ -156,23 +157,23 @@ func (fd *File) unmarshalFull(b []byte) { b = b[m:] switch num { case fieldnum.FileDescriptorProto_Dependency: - path := nb.MakeString(v) + path := sb.MakeString(v) imp, _ := fd.builder.FileRegistry.FindFileByPath(path) if imp == nil { imp = PlaceholderFile(path) } fd.L2.Imports = append(fd.L2.Imports, pref.FileImport{FileDescriptor: imp}) case fieldnum.FileDescriptorProto_EnumType: - fd.L1.Enums.List[enumIdx].unmarshalFull(v, nb) + fd.L1.Enums.List[enumIdx].unmarshalFull(v, sb) enumIdx++ case fieldnum.FileDescriptorProto_MessageType: - fd.L1.Messages.List[messageIdx].unmarshalFull(v, nb) + fd.L1.Messages.List[messageIdx].unmarshalFull(v, sb) messageIdx++ case fieldnum.FileDescriptorProto_Extension: - fd.L1.Extensions.List[extensionIdx].unmarshalFull(v, nb) + fd.L1.Extensions.List[extensionIdx].unmarshalFull(v, sb) extensionIdx++ case fieldnum.FileDescriptorProto_Service: - fd.L1.Services.List[serviceIdx].unmarshalFull(v, nb) + fd.L1.Services.List[serviceIdx].unmarshalFull(v, sb) serviceIdx++ case fieldnum.FileDescriptorProto_Options: rawOptions = appendOptions(rawOptions, v) @@ -185,7 +186,7 @@ func (fd *File) unmarshalFull(b []byte) { fd.L2.Options = fd.builder.optionsUnmarshaler(descopts.File, rawOptions) } -func (ed *Enum) unmarshalFull(b []byte, nb *nameBuilder) { +func (ed *Enum) unmarshalFull(b []byte, sb *strs.Builder) { var rawValues [][]byte var rawOptions []byte if !ed.L1.eagerValues { @@ -202,7 +203,7 @@ func (ed *Enum) unmarshalFull(b []byte, nb *nameBuilder) { case fieldnum.EnumDescriptorProto_Value: rawValues = append(rawValues, v) case fieldnum.EnumDescriptorProto_ReservedName: - ed.L2.ReservedNames.List = append(ed.L2.ReservedNames.List, pref.Name(nb.MakeString(v))) + ed.L2.ReservedNames.List = append(ed.L2.ReservedNames.List, pref.Name(sb.MakeString(v))) case fieldnum.EnumDescriptorProto_ReservedRange: ed.L2.ReservedRanges.List = append(ed.L2.ReservedRanges.List, unmarshalEnumReservedRange(v)) case fieldnum.EnumDescriptorProto_Options: @@ -216,7 +217,7 @@ func (ed *Enum) unmarshalFull(b []byte, nb *nameBuilder) { if !ed.L1.eagerValues && len(rawValues) > 0 { ed.L2.Values.List = make([]EnumValue, len(rawValues)) for i, b := range rawValues { - ed.L2.Values.List[i].unmarshalFull(b, nb, ed.L0.ParentFile, ed, i) + ed.L2.Values.List[i].unmarshalFull(b, sb, ed.L0.ParentFile, ed, i) } } ed.L2.Options = ed.L0.ParentFile.builder.optionsUnmarshaler(descopts.Enum, rawOptions) @@ -244,7 +245,7 @@ func unmarshalEnumReservedRange(b []byte) (r [2]pref.EnumNumber) { return r } -func (vd *EnumValue) unmarshalFull(b []byte, nb *nameBuilder, pf *File, pd pref.Descriptor, i int) { +func (vd *EnumValue) unmarshalFull(b []byte, sb *strs.Builder, pf *File, pd pref.Descriptor, i int) { vd.L0.ParentFile = pf vd.L0.Parent = pd vd.L0.Index = i @@ -267,7 +268,7 @@ func (vd *EnumValue) unmarshalFull(b []byte, nb *nameBuilder, pf *File, pd pref. switch num { case fieldnum.EnumValueDescriptorProto_Name: // NOTE: Enum values are in the same scope as the enum parent. - vd.L0.FullName = nb.AppendFullName(pd.Parent().FullName(), v) + vd.L0.FullName = appendFullName(sb, pd.Parent().FullName(), v) case fieldnum.EnumValueDescriptorProto_Options: rawOptions = appendOptions(rawOptions, v) } @@ -279,7 +280,7 @@ func (vd *EnumValue) unmarshalFull(b []byte, nb *nameBuilder, pf *File, pd pref. vd.L1.Options = pf.builder.optionsUnmarshaler(descopts.EnumValue, rawOptions) } -func (md *Message) unmarshalFull(b []byte, nb *nameBuilder) { +func (md *Message) unmarshalFull(b []byte, sb *strs.Builder) { var rawFields, rawOneofs [][]byte var enumIdx, messageIdx, extensionIdx int var rawOptions []byte @@ -297,7 +298,7 @@ func (md *Message) unmarshalFull(b []byte, nb *nameBuilder) { case fieldnum.DescriptorProto_OneofDecl: rawOneofs = append(rawOneofs, v) case fieldnum.DescriptorProto_ReservedName: - md.L2.ReservedNames.List = append(md.L2.ReservedNames.List, pref.Name(nb.MakeString(v))) + md.L2.ReservedNames.List = append(md.L2.ReservedNames.List, pref.Name(sb.MakeString(v))) case fieldnum.DescriptorProto_ReservedRange: md.L2.ReservedRanges.List = append(md.L2.ReservedRanges.List, unmarshalMessageReservedRange(v)) case fieldnum.DescriptorProto_ExtensionRange: @@ -306,13 +307,13 @@ func (md *Message) unmarshalFull(b []byte, nb *nameBuilder) { md.L2.ExtensionRanges.List = append(md.L2.ExtensionRanges.List, r) md.L2.ExtensionRangeOptions = append(md.L2.ExtensionRangeOptions, opts) case fieldnum.DescriptorProto_EnumType: - md.L1.Enums.List[enumIdx].unmarshalFull(v, nb) + md.L1.Enums.List[enumIdx].unmarshalFull(v, sb) enumIdx++ case fieldnum.DescriptorProto_NestedType: - md.L1.Messages.List[messageIdx].unmarshalFull(v, nb) + md.L1.Messages.List[messageIdx].unmarshalFull(v, sb) messageIdx++ case fieldnum.DescriptorProto_Extension: - md.L1.Extensions.List[extensionIdx].unmarshalFull(v, nb) + md.L1.Extensions.List[extensionIdx].unmarshalFull(v, sb) extensionIdx++ case fieldnum.DescriptorProto_Options: md.unmarshalOptions(v) @@ -328,14 +329,14 @@ func (md *Message) unmarshalFull(b []byte, nb *nameBuilder) { md.L2.Oneofs.List = make([]Oneof, len(rawOneofs)) for i, b := range rawFields { fd := &md.L2.Fields.List[i] - fd.unmarshalFull(b, nb, md.L0.ParentFile, md, i) + fd.unmarshalFull(b, sb, md.L0.ParentFile, md, i) if fd.L1.Cardinality == pref.Required { md.L2.RequiredNumbers.List = append(md.L2.RequiredNumbers.List, fd.L1.Number) } } for i, b := range rawOneofs { od := &md.L2.Oneofs.List[i] - od.unmarshalFull(b, nb, md.L0.ParentFile, md, i) + od.unmarshalFull(b, sb, md.L0.ParentFile, md, i) } } md.L2.Options = md.L0.ParentFile.builder.optionsUnmarshaler(descopts.Message, rawOptions) @@ -413,7 +414,7 @@ func unmarshalMessageExtensionRange(b []byte) (r [2]pref.FieldNumber, rawOptions return r, rawOptions } -func (fd *Field) unmarshalFull(b []byte, nb *nameBuilder, pf *File, pd pref.Descriptor, i int) { +func (fd *Field) unmarshalFull(b []byte, sb *strs.Builder, pf *File, pd pref.Descriptor, i int) { fd.L0.ParentFile = pf fd.L0.Parent = pd fd.L0.Index = i @@ -450,9 +451,9 @@ func (fd *Field) unmarshalFull(b []byte, nb *nameBuilder, pf *File, pd pref.Desc b = b[m:] switch num { case fieldnum.FieldDescriptorProto_Name: - fd.L0.FullName = nb.AppendFullName(pd.FullName(), v) + fd.L0.FullName = appendFullName(sb, pd.FullName(), v) case fieldnum.FieldDescriptorProto_JsonName: - fd.L1.JSONName = JSONName(nb.MakeString(v)) + fd.L1.JSONName = JSONName(sb.MakeString(v)) case fieldnum.FieldDescriptorProto_DefaultValue: fd.L1.Default.val = pref.ValueOf(v) // temporarily store as bytes; later resolved in resolveMessages case fieldnum.FieldDescriptorProto_TypeName: @@ -467,7 +468,7 @@ func (fd *Field) unmarshalFull(b []byte, nb *nameBuilder, pf *File, pd pref.Desc } } if rawTypeName != nil { - name := nb.MakeFullName(rawTypeName) + name := makeFullName(sb, rawTypeName) switch fd.L1.Kind { case pref.EnumKind: fd.L1.Enum = PlaceholderEnum(name) @@ -500,7 +501,7 @@ func (fd *Field) unmarshalOptions(b []byte) { } } -func (od *Oneof) unmarshalFull(b []byte, nb *nameBuilder, pf *File, pd pref.Descriptor, i int) { +func (od *Oneof) unmarshalFull(b []byte, sb *strs.Builder, pf *File, pd pref.Descriptor, i int) { od.L0.ParentFile = pf od.L0.Parent = pd od.L0.Index = i @@ -515,7 +516,7 @@ func (od *Oneof) unmarshalFull(b []byte, nb *nameBuilder, pf *File, pd pref.Desc b = b[m:] switch num { case fieldnum.OneofDescriptorProto_Name: - od.L0.FullName = nb.AppendFullName(pd.FullName(), v) + od.L0.FullName = appendFullName(sb, pd.FullName(), v) case fieldnum.OneofDescriptorProto_Options: rawOptions = appendOptions(rawOptions, v) } @@ -527,7 +528,7 @@ func (od *Oneof) unmarshalFull(b []byte, nb *nameBuilder, pf *File, pd pref.Desc od.L1.Options = pf.builder.optionsUnmarshaler(descopts.Oneof, rawOptions) } -func (xd *Extension) unmarshalFull(b []byte, nb *nameBuilder) { +func (xd *Extension) unmarshalFull(b []byte, sb *strs.Builder) { var rawTypeName []byte var rawOptions []byte xd.L2 = new(ExtensionL2) @@ -547,7 +548,7 @@ func (xd *Extension) unmarshalFull(b []byte, nb *nameBuilder) { b = b[m:] switch num { case fieldnum.FieldDescriptorProto_JsonName: - xd.L2.JSONName = JSONName(nb.MakeString(v)) + xd.L2.JSONName = JSONName(sb.MakeString(v)) case fieldnum.FieldDescriptorProto_DefaultValue: xd.L2.Default.val = pref.ValueOf(v) // temporarily store as bytes; later resolved in resolveExtensions case fieldnum.FieldDescriptorProto_TypeName: @@ -562,7 +563,7 @@ func (xd *Extension) unmarshalFull(b []byte, nb *nameBuilder) { } } if rawTypeName != nil { - name := nb.MakeFullName(rawTypeName) + name := makeFullName(sb, rawTypeName) switch xd.L1.Kind { case pref.EnumKind: xd.L2.Enum = PlaceholderEnum(name) @@ -592,7 +593,7 @@ func (xd *Extension) unmarshalOptions(b []byte) { } } -func (sd *Service) unmarshalFull(b []byte, nb *nameBuilder) { +func (sd *Service) unmarshalFull(b []byte, sb *strs.Builder) { var rawMethods [][]byte var rawOptions []byte sd.L2 = new(ServiceL2) @@ -617,13 +618,13 @@ func (sd *Service) unmarshalFull(b []byte, nb *nameBuilder) { if len(rawMethods) > 0 { sd.L2.Methods.List = make([]Method, len(rawMethods)) for i, b := range rawMethods { - sd.L2.Methods.List[i].unmarshalFull(b, nb, sd.L0.ParentFile, sd, i) + sd.L2.Methods.List[i].unmarshalFull(b, sb, sd.L0.ParentFile, sd, i) } } sd.L2.Options = sd.L0.ParentFile.builder.optionsUnmarshaler(descopts.Service, rawOptions) } -func (md *Method) unmarshalFull(b []byte, nb *nameBuilder, pf *File, pd pref.Descriptor, i int) { +func (md *Method) unmarshalFull(b []byte, sb *strs.Builder, pf *File, pd pref.Descriptor, i int) { md.L0.ParentFile = pf md.L0.Parent = pd md.L0.Index = i @@ -647,11 +648,11 @@ func (md *Method) unmarshalFull(b []byte, nb *nameBuilder, pf *File, pd pref.Des b = b[m:] switch num { case fieldnum.MethodDescriptorProto_Name: - md.L0.FullName = nb.AppendFullName(pd.FullName(), v) + md.L0.FullName = appendFullName(sb, pd.FullName(), v) case fieldnum.MethodDescriptorProto_InputType: - md.L1.Input = PlaceholderMessage(nb.MakeFullName(v)) + md.L1.Input = PlaceholderMessage(makeFullName(sb, v)) case fieldnum.MethodDescriptorProto_OutputType: - md.L1.Output = PlaceholderMessage(nb.MakeFullName(v)) + md.L1.Output = PlaceholderMessage(makeFullName(sb, v)) case fieldnum.MethodDescriptorProto_Options: rawOptions = appendOptions(rawOptions, v) } diff --git a/internal/filedesc/name_pure.go b/internal/filedesc/name_pure.go deleted file mode 100644 index 5626829b..00000000 --- a/internal/filedesc/name_pure.go +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build purego appengine - -package filedesc - -import pref "google.golang.org/protobuf/reflect/protoreflect" - -func getNameBuilder() *nameBuilder { return nil } -func putNameBuilder(*nameBuilder) {} - -type nameBuilder struct{} - -// MakeFullName converts b to a protoreflect.FullName, -// where b must start with a leading dot. -func (*nameBuilder) MakeFullName(b []byte) pref.FullName { - if len(b) == 0 || b[0] != '.' { - panic("name reference must be fully qualified") - } - return pref.FullName(b[1:]) -} - -// AppendFullName is equivalent to protoreflect.FullName.Append. -func (*nameBuilder) AppendFullName(prefix pref.FullName, name []byte) pref.FullName { - return prefix.Append(pref.Name(name)) -} - -// MakeString is equivalent to string(b), but optimized for large batches -// with a shared lifetime. -func (*nameBuilder) MakeString(b []byte) string { - return string(b) -} diff --git a/internal/filedesc/name_unsafe.go b/internal/filedesc/name_unsafe.go deleted file mode 100644 index 317114fe..00000000 --- a/internal/filedesc/name_unsafe.go +++ /dev/null @@ -1,124 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build !purego,!appengine - -package filedesc - -import ( - "sync" - "unsafe" - - pref "google.golang.org/protobuf/reflect/protoreflect" -) - -var nameBuilderPool = sync.Pool{ - New: func() interface{} { return new(nameBuilder) }, -} - -func getNameBuilder() *nameBuilder { - return nameBuilderPool.Get().(*nameBuilder) -} -func putNameBuilder(b *nameBuilder) { - nameBuilderPool.Put(b) -} - -type nameBuilder struct { - sb stringBuilder -} - -// MakeFullName converts b to a protoreflect.FullName, -// where b must start with a leading dot. -func (nb *nameBuilder) MakeFullName(b []byte) pref.FullName { - if len(b) == 0 || b[0] != '.' { - panic("name reference must be fully qualified") - } - return pref.FullName(nb.MakeString(b[1:])) -} - -// AppendFullName is equivalent to protoreflect.FullName.Append, -// but optimized for large batches where each name has a shared lifetime. -func (nb *nameBuilder) AppendFullName(prefix pref.FullName, name []byte) pref.FullName { - n := len(prefix) + len(".") + len(name) - if len(prefix) == 0 { - n -= len(".") - } - nb.grow(n) - nb.sb.WriteString(string(prefix)) - nb.sb.WriteByte('.') - nb.sb.Write(name) - return pref.FullName(nb.last(n)) -} - -// MakeString is equivalent to string(b), but optimized for large batches -// with a shared lifetime. -func (nb *nameBuilder) MakeString(b []byte) string { - nb.grow(len(b)) - nb.sb.Write(b) - return nb.last(len(b)) -} - -func (nb *nameBuilder) last(n int) string { - s := nb.sb.String() - return s[len(s)-n:] -} - -func (nb *nameBuilder) grow(n int) { - const batchSize = 1 << 16 - if nb.sb.Cap()-nb.sb.Len() < n { - nb.sb.Reset() - nb.sb.Grow(batchSize) - } -} - -// stringsBuilder is a simplified copy of the strings.Builder from Go1.12: -// * removed the shallow copy check -// * removed methods that we do not use (e.g. WriteRune) -// -// A forked version is used: -// * to enable Go1.9 support, but strings.Builder was added in Go1.10 -// * for the Cap method, which was missing until Go1.12 -// -// TODO: Remove this when Go1.12 is the minimally supported toolchain version. -type stringBuilder struct { - buf []byte -} - -func (b *stringBuilder) String() string { - return *(*string)(unsafe.Pointer(&b.buf)) -} -func (b *stringBuilder) Len() int { - return len(b.buf) -} -func (b *stringBuilder) Cap() int { - return cap(b.buf) -} -func (b *stringBuilder) Reset() { - b.buf = nil -} -func (b *stringBuilder) grow(n int) { - buf := make([]byte, len(b.buf), 2*cap(b.buf)+n) - copy(buf, b.buf) - b.buf = buf -} -func (b *stringBuilder) Grow(n int) { - if n < 0 { - panic("stringBuilder.Grow: negative count") - } - if cap(b.buf)-len(b.buf) < n { - b.grow(n) - } -} -func (b *stringBuilder) Write(p []byte) (int, error) { - b.buf = append(b.buf, p...) - return len(p), nil -} -func (b *stringBuilder) WriteByte(c byte) error { - b.buf = append(b.buf, c) - return nil -} -func (b *stringBuilder) WriteString(s string) (int, error) { - b.buf = append(b.buf, s...) - return len(s), nil -} diff --git a/internal/impl/legacy_message.go b/internal/impl/legacy_message.go index fd3732fb..ff8af03f 100644 --- a/internal/impl/legacy_message.go +++ b/internal/impl/legacy_message.go @@ -14,6 +14,7 @@ import ( "google.golang.org/protobuf/internal/descopts" ptag "google.golang.org/protobuf/internal/encoding/tag" "google.golang.org/protobuf/internal/filedesc" + "google.golang.org/protobuf/internal/strs" "google.golang.org/protobuf/reflect/protoreflect" pref "google.golang.org/protobuf/reflect/protoreflect" "google.golang.org/protobuf/reflect/prototype" @@ -248,7 +249,7 @@ func aberrantAppendField(md *filedesc.Message, goType reflect.Type, tag, tagKey, n := len(md.L1.Messages.List) md.L1.Messages.List = append(md.L1.Messages.List, filedesc.Message{L2: new(filedesc.MessageL2)}) md2 := &md.L1.Messages.List[n] - md2.L0.FullName = md.FullName().Append(aberrantMapEntryName(fd.Name())) + md2.L0.FullName = md.FullName().Append(pref.Name(strs.MapEntryName(string(fd.Name())))) md2.L0.ParentFile = md.L0.ParentFile md2.L0.Parent = md md2.L0.Index = n diff --git a/internal/strs/strings.go b/internal/strs/strings.go new file mode 100644 index 00000000..295bd296 --- /dev/null +++ b/internal/strs/strings.go @@ -0,0 +1,111 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package strs provides string manipulation functionality specific to protobuf. +package strs + +import ( + "strings" + "unicode" +) + +// JSONCamelCase converts a snake_case identifier to a camelCase identifier, +// according to the protobuf JSON specification. +func JSONCamelCase(s string) string { + var b []byte + var wasUnderscore bool + for i := 0; i < len(s); i++ { // proto identifiers are always ASCII + c := s[i] + if c != '_' { + isLower := 'a' <= c && c <= 'z' + if wasUnderscore && isLower { + c -= 'a' - 'A' // convert to uppercase + } + b = append(b, c) + } + wasUnderscore = c == '_' + } + return string(b) +} + +// JSONSnakeCase converts a camelCase identifier to a snake_case identifier, +// according to the protobuf JSON specification. +func JSONSnakeCase(s string) string { + var b []byte + for i := 0; i < len(s); i++ { // proto identifiers are always ASCII + c := s[i] + isUpper := 'A' <= c && c <= 'Z' + if isUpper { + b = append(b, '_') + c += 'a' - 'A' // convert to lowercase + } + b = append(b, c) + } + return string(b) +} + +// MapEntryName derives the name of the map entry message given the field name. +// See protoc v3.8.0: src/google/protobuf/descriptor.cc:254-276,6057 +func MapEntryName(s string) string { + var b []byte + upperNext := true + for _, c := range s { + switch { + case c == '_': + upperNext = true + case upperNext: + b = append(b, byte(unicode.ToUpper(c))) + upperNext = false + default: + b = append(b, byte(c)) + } + } + b = append(b, "Entry"...) + return string(b) +} + +// EnumValueName derives the camel-cased enum value name. +// See protoc v3.8.0: src/google/protobuf/descriptor.cc:297-313 +func EnumValueName(s string) string { + var b []byte + upperNext := true + for _, c := range s { + switch { + case c == '_': + upperNext = true + case upperNext: + b = append(b, byte(unicode.ToUpper(c))) + upperNext = false + default: + b = append(b, byte(unicode.ToLower(c))) + upperNext = false + } + } + return string(b) +} + +// TrimEnumPrefix trims the enum name prefix from an enum value name, +// where the prefix is all lowercase without underscores. +// See protoc v3.8.0: src/google/protobuf/descriptor.cc:330-375 +func TrimEnumPrefix(s, prefix string) string { + s0 := s // original input + for len(s) > 0 && len(prefix) > 0 { + if s[0] == '_' { + s = s[1:] + continue + } + if unicode.ToLower(rune(s[0])) != rune(prefix[0]) { + return s0 // no prefix match + } + s, prefix = s[1:], prefix[1:] + } + if len(prefix) > 0 { + return s0 // no prefix match + } + s = strings.TrimLeft(s, "_") + if len(s) == 0 { + return s0 // avoid returning empty string + } + return s +} diff --git a/internal/strs/strings_pure.go b/internal/strs/strings_pure.go new file mode 100644 index 00000000..85e074c9 --- /dev/null +++ b/internal/strs/strings_pure.go @@ -0,0 +1,27 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build purego appengine + +package strs + +import pref "google.golang.org/protobuf/reflect/protoreflect" + +func UnsafeString(b []byte) string { + return string(b) +} + +func UnsafeBytes(s string) []byte { + return []byte(s) +} + +type Builder struct{} + +func (*Builder) AppendFullName(prefix pref.FullName, name pref.Name) pref.FullName { + return prefix.Append(name) +} + +func (*Builder) MakeString(b []byte) string { + return string(b) +} diff --git a/internal/strs/strings_test.go b/internal/strs/strings_test.go new file mode 100644 index 00000000..2c4c2adf --- /dev/null +++ b/internal/strs/strings_test.go @@ -0,0 +1,108 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package strs + +import ( + "strconv" + "testing" +) + +func TestName(t *testing.T) { + tests := []struct { + in string + inEnumPrefix string + wantMapEntry string + wantEnumValue string + wantTrimValue string + wantJSONCamelCase string + wantJSONSnakeCase string + }{{ + in: "abc", + inEnumPrefix: "", + wantMapEntry: "AbcEntry", + wantEnumValue: "Abc", + wantTrimValue: "abc", + wantJSONCamelCase: "abc", + wantJSONSnakeCase: "abc", + }, { + in: "foo_baR_", + inEnumPrefix: "foo_bar", + wantMapEntry: "FooBaREntry", + wantEnumValue: "FooBar", + wantTrimValue: "foo_baR_", + wantJSONCamelCase: "fooBaR", + wantJSONSnakeCase: "foo_ba_r_", + }, { + in: "snake_caseCamelCase", + inEnumPrefix: "snakecasecamel", + wantMapEntry: "SnakeCaseCamelCaseEntry", + wantEnumValue: "SnakeCasecamelcase", + wantTrimValue: "Case", + wantJSONCamelCase: "snakeCaseCamelCase", + wantJSONSnakeCase: "snake_case_camel_case", + }, { + in: "FiZz_BuZz", + inEnumPrefix: "fizz", + wantMapEntry: "FiZzBuZzEntry", + wantEnumValue: "FizzBuzz", + wantTrimValue: "BuZz", + wantJSONCamelCase: "FiZzBuZz", + wantJSONSnakeCase: "_fi_zz__bu_zz", + }} + + for _, tt := range tests { + if got := MapEntryName(tt.in); got != tt.wantMapEntry { + t.Errorf("MapEntryName(%q) = %q, want %q", tt.in, got, tt.wantMapEntry) + } + if got := EnumValueName(tt.in); got != tt.wantEnumValue { + t.Errorf("EnumValueName(%q) = %q, want %q", tt.in, got, tt.wantEnumValue) + } + if got := TrimEnumPrefix(tt.in, tt.inEnumPrefix); got != tt.wantTrimValue { + t.Errorf("ErimEnumPrefix(%q, %q) = %q, want %q", tt.in, tt.inEnumPrefix, got, tt.wantTrimValue) + } + if got := JSONCamelCase(tt.in); got != tt.wantJSONCamelCase { + t.Errorf("JSONCamelCase(%q) = %q, want %q", tt.in, got, tt.wantJSONCamelCase) + } + if got := JSONSnakeCase(tt.in); got != tt.wantJSONSnakeCase { + t.Errorf("JSONSnakeCase(%q) = %q, want %q", tt.in, got, tt.wantJSONSnakeCase) + } + } +} + +var ( + srcString = "1234" + srcBytes = []byte(srcString) + dst uint64 +) + +func BenchmarkCast(b *testing.B) { + b.Run("Ideal", func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + dst, _ = strconv.ParseUint(srcString, 0, 64) + } + if dst != 1234 { + b.Errorf("got %d, want %s", dst, srcString) + } + }) + b.Run("Copy", func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + dst, _ = strconv.ParseUint(string(srcBytes), 0, 64) + } + if dst != 1234 { + b.Errorf("got %d, want %s", dst, srcString) + } + }) + b.Run("Cast", func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + dst, _ = strconv.ParseUint(UnsafeString(srcBytes), 0, 64) + } + if dst != 1234 { + b.Errorf("got %d, want %s", dst, srcString) + } + }) +} diff --git a/internal/strs/strings_unsafe.go b/internal/strs/strings_unsafe.go new file mode 100644 index 00000000..2160c701 --- /dev/null +++ b/internal/strs/strings_unsafe.go @@ -0,0 +1,94 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !purego,!appengine + +package strs + +import ( + "unsafe" + + pref "google.golang.org/protobuf/reflect/protoreflect" +) + +type ( + stringHeader struct { + Data unsafe.Pointer + Len int + } + sliceHeader struct { + Data unsafe.Pointer + Len int + Cap int + } +) + +// UnsafeString returns an unsafe string reference of b. +// The caller must treat the input slice as immutable. +// +// WARNING: Use carefully. The returned result must not leak to the end user +// unless the input slice is provably immutable. +func UnsafeString(b []byte) (s string) { + src := (*sliceHeader)(unsafe.Pointer(&b)) + dst := (*stringHeader)(unsafe.Pointer(&s)) + dst.Data = src.Data + dst.Len = src.Len + return s +} + +// UnsafeBytes returns an unsafe bytes slice reference of s. +// The caller must treat returned slice as immutable. +// +// WARNING: Use carefully. The returned result must not leak to the end user. +func UnsafeBytes(s string) (b []byte) { + src := (*stringHeader)(unsafe.Pointer(&s)) + dst := (*sliceHeader)(unsafe.Pointer(&b)) + dst.Data = src.Data + dst.Len = src.Len + dst.Cap = src.Len + return b +} + +// Builder builds a set of strings with shared lifetime. +// This differs from strings.Builder, which is for building a single string. +type Builder struct { + buf []byte +} + +// AppendFullName is equivalent to protoreflect.FullName.Append, +// but optimized for large batches where each name has a shared lifetime. +func (sb *Builder) AppendFullName(prefix pref.FullName, name pref.Name) pref.FullName { + n := len(prefix) + len(".") + len(name) + if len(prefix) == 0 { + n -= len(".") + } + sb.grow(n) + sb.buf = append(sb.buf, prefix...) + sb.buf = append(sb.buf, '.') + sb.buf = append(sb.buf, name...) + return pref.FullName(sb.last(n)) +} + +// MakeString is equivalent to string(b), but optimized for large batches +// with a shared lifetime. +func (sb *Builder) MakeString(b []byte) string { + sb.grow(len(b)) + sb.buf = append(sb.buf, b...) + return sb.last(len(b)) +} + +func (sb *Builder) grow(n int) { + if cap(sb.buf)-len(sb.buf) >= n { + return + } + + // Unlike strings.Builder, we do not need to copy over the contents + // of the old buffer since our builder provides no API for + // retrieving previously created strings. + sb.buf = make([]byte, 2*(cap(sb.buf)+n)) +} + +func (sb *Builder) last(n int) string { + return UnsafeString(sb.buf[len(sb.buf)-n:]) +} diff --git a/reflect/protodesc/desc.go b/reflect/protodesc/desc.go index f8a24beb..fd310c08 100644 --- a/reflect/protodesc/desc.go +++ b/reflect/protodesc/desc.go @@ -10,6 +10,7 @@ import ( "google.golang.org/protobuf/internal/errors" "google.golang.org/protobuf/internal/filedesc" "google.golang.org/protobuf/internal/pragma" + "google.golang.org/protobuf/internal/strs" "google.golang.org/protobuf/reflect/protoreflect" "google.golang.org/protobuf/reflect/protoregistry" @@ -144,17 +145,18 @@ func newFile(fd *descriptorpb.FileDescriptorProto, r Resolver, opts ...option) ( // google.protobuf.MethodDescriptorProto.input // google.protobuf.MethodDescriptorProto.output var err error + sb := new(strs.Builder) r1 := make(descsByName) - if f.L1.Enums.List, err = r1.initEnumDeclarations(fd.GetEnumType(), f); err != nil { + if f.L1.Enums.List, err = r1.initEnumDeclarations(fd.GetEnumType(), f, sb); err != nil { return nil, err } - if f.L1.Messages.List, err = r1.initMessagesDeclarations(fd.GetMessageType(), f); err != nil { + if f.L1.Messages.List, err = r1.initMessagesDeclarations(fd.GetMessageType(), f, sb); err != nil { return nil, err } - if f.L1.Extensions.List, err = r1.initExtensionDeclarations(fd.GetExtension(), f); err != nil { + if f.L1.Extensions.List, err = r1.initExtensionDeclarations(fd.GetExtension(), f, sb); err != nil { return nil, err } - if f.L1.Services.List, err = r1.initServiceDeclarations(fd.GetService(), f); err != nil { + if f.L1.Services.List, err = r1.initServiceDeclarations(fd.GetService(), f, sb); err != nil { return nil, err } diff --git a/reflect/protodesc/desc_init.go b/reflect/protodesc/desc_init.go index 84c76c47..b356cf75 100644 --- a/reflect/protodesc/desc_init.go +++ b/reflect/protodesc/desc_init.go @@ -7,6 +7,7 @@ package protodesc import ( "google.golang.org/protobuf/internal/errors" "google.golang.org/protobuf/internal/filedesc" + "google.golang.org/protobuf/internal/strs" "google.golang.org/protobuf/reflect/protoreflect" "google.golang.org/protobuf/types/descriptorpb" @@ -14,12 +15,12 @@ import ( type descsByName map[protoreflect.FullName]protoreflect.Descriptor -func (r descsByName) initEnumDeclarations(eds []*descriptorpb.EnumDescriptorProto, parent protoreflect.Descriptor) (es []filedesc.Enum, err error) { +func (r descsByName) initEnumDeclarations(eds []*descriptorpb.EnumDescriptorProto, parent protoreflect.Descriptor, sb *strs.Builder) (es []filedesc.Enum, err error) { es = make([]filedesc.Enum, len(eds)) // allocate up-front to ensure stable pointers for i, ed := range eds { e := &es[i] e.L2 = new(filedesc.EnumL2) - if e.L0, err = r.makeBase(e, parent, ed.GetName(), i); err != nil { + if e.L0, err = r.makeBase(e, parent, ed.GetName(), i, sb); err != nil { return nil, err } if opts := ed.GetOptions(); opts != nil { @@ -35,18 +36,18 @@ func (r descsByName) initEnumDeclarations(eds []*descriptorpb.EnumDescriptorProt protoreflect.EnumNumber(rr.GetEnd()), }) } - if e.L2.Values.List, err = r.initEnumValuesFromDescriptorProto(ed.GetValue(), e); err != nil { + if e.L2.Values.List, err = r.initEnumValuesFromDescriptorProto(ed.GetValue(), e, sb); err != nil { return nil, err } } return es, nil } -func (r descsByName) initEnumValuesFromDescriptorProto(vds []*descriptorpb.EnumValueDescriptorProto, parent protoreflect.Descriptor) (vs []filedesc.EnumValue, err error) { +func (r descsByName) initEnumValuesFromDescriptorProto(vds []*descriptorpb.EnumValueDescriptorProto, parent protoreflect.Descriptor, sb *strs.Builder) (vs []filedesc.EnumValue, err error) { vs = make([]filedesc.EnumValue, len(vds)) // allocate up-front to ensure stable pointers for i, vd := range vds { v := &vs[i] - if v.L0, err = r.makeBase(v, parent, vd.GetName(), i); err != nil { + if v.L0, err = r.makeBase(v, parent, vd.GetName(), i, sb); err != nil { return nil, err } if opts := vd.GetOptions(); opts != nil { @@ -58,12 +59,12 @@ func (r descsByName) initEnumValuesFromDescriptorProto(vds []*descriptorpb.EnumV return vs, nil } -func (r descsByName) initMessagesDeclarations(mds []*descriptorpb.DescriptorProto, parent protoreflect.Descriptor) (ms []filedesc.Message, err error) { +func (r descsByName) initMessagesDeclarations(mds []*descriptorpb.DescriptorProto, parent protoreflect.Descriptor, sb *strs.Builder) (ms []filedesc.Message, err error) { ms = make([]filedesc.Message, len(mds)) // allocate up-front to ensure stable pointers for i, md := range mds { m := &ms[i] m.L2 = new(filedesc.MessageL2) - if m.L0, err = r.makeBase(m, parent, md.GetName(), i); err != nil { + if m.L0, err = r.makeBase(m, parent, md.GetName(), i, sb); err != nil { return nil, err } if opts := md.GetOptions(); opts != nil { @@ -93,30 +94,30 @@ func (r descsByName) initMessagesDeclarations(mds []*descriptorpb.DescriptorProt } m.L2.ExtensionRangeOptions = append(m.L2.ExtensionRangeOptions, optsFunc) } - if m.L2.Fields.List, err = r.initFieldsFromDescriptorProto(md.GetField(), m); err != nil { + if m.L2.Fields.List, err = r.initFieldsFromDescriptorProto(md.GetField(), m, sb); err != nil { return nil, err } - if m.L2.Oneofs.List, err = r.initOneofsFromDescriptorProto(md.GetOneofDecl(), m); err != nil { + if m.L2.Oneofs.List, err = r.initOneofsFromDescriptorProto(md.GetOneofDecl(), m, sb); err != nil { return nil, err } - if m.L1.Enums.List, err = r.initEnumDeclarations(md.GetEnumType(), m); err != nil { + if m.L1.Enums.List, err = r.initEnumDeclarations(md.GetEnumType(), m, sb); err != nil { return nil, err } - if m.L1.Messages.List, err = r.initMessagesDeclarations(md.GetNestedType(), m); err != nil { + if m.L1.Messages.List, err = r.initMessagesDeclarations(md.GetNestedType(), m, sb); err != nil { return nil, err } - if m.L1.Extensions.List, err = r.initExtensionDeclarations(md.GetExtension(), m); err != nil { + if m.L1.Extensions.List, err = r.initExtensionDeclarations(md.GetExtension(), m, sb); err != nil { return nil, err } } return ms, nil } -func (r descsByName) initFieldsFromDescriptorProto(fds []*descriptorpb.FieldDescriptorProto, parent protoreflect.Descriptor) (fs []filedesc.Field, err error) { +func (r descsByName) initFieldsFromDescriptorProto(fds []*descriptorpb.FieldDescriptorProto, parent protoreflect.Descriptor, sb *strs.Builder) (fs []filedesc.Field, err error) { fs = make([]filedesc.Field, len(fds)) // allocate up-front to ensure stable pointers for i, fd := range fds { f := &fs[i] - if f.L0, err = r.makeBase(f, parent, fd.GetName(), i); err != nil { + if f.L0, err = r.makeBase(f, parent, fd.GetName(), i, sb); err != nil { return nil, err } if opts := fd.GetOptions(); opts != nil { @@ -138,11 +139,11 @@ func (r descsByName) initFieldsFromDescriptorProto(fds []*descriptorpb.FieldDesc return fs, nil } -func (r descsByName) initOneofsFromDescriptorProto(ods []*descriptorpb.OneofDescriptorProto, parent protoreflect.Descriptor) (os []filedesc.Oneof, err error) { +func (r descsByName) initOneofsFromDescriptorProto(ods []*descriptorpb.OneofDescriptorProto, parent protoreflect.Descriptor, sb *strs.Builder) (os []filedesc.Oneof, err error) { os = make([]filedesc.Oneof, len(ods)) // allocate up-front to ensure stable pointers for i, od := range ods { o := &os[i] - if o.L0, err = r.makeBase(o, parent, od.GetName(), i); err != nil { + if o.L0, err = r.makeBase(o, parent, od.GetName(), i, sb); err != nil { return nil, err } if opts := od.GetOptions(); opts != nil { @@ -153,12 +154,12 @@ func (r descsByName) initOneofsFromDescriptorProto(ods []*descriptorpb.OneofDesc return os, nil } -func (r descsByName) initExtensionDeclarations(xds []*descriptorpb.FieldDescriptorProto, parent protoreflect.Descriptor) (xs []filedesc.Extension, err error) { +func (r descsByName) initExtensionDeclarations(xds []*descriptorpb.FieldDescriptorProto, parent protoreflect.Descriptor, sb *strs.Builder) (xs []filedesc.Extension, err error) { xs = make([]filedesc.Extension, len(xds)) // allocate up-front to ensure stable pointers for i, xd := range xds { x := &xs[i] x.L2 = new(filedesc.ExtensionL2) - if x.L0, err = r.makeBase(x, parent, xd.GetName(), i); err != nil { + if x.L0, err = r.makeBase(x, parent, xd.GetName(), i, sb); err != nil { return nil, err } if opts := xd.GetOptions(); opts != nil { @@ -178,30 +179,30 @@ func (r descsByName) initExtensionDeclarations(xds []*descriptorpb.FieldDescript return xs, nil } -func (r descsByName) initServiceDeclarations(sds []*descriptorpb.ServiceDescriptorProto, parent protoreflect.Descriptor) (ss []filedesc.Service, err error) { +func (r descsByName) initServiceDeclarations(sds []*descriptorpb.ServiceDescriptorProto, parent protoreflect.Descriptor, sb *strs.Builder) (ss []filedesc.Service, err error) { ss = make([]filedesc.Service, len(sds)) // allocate up-front to ensure stable pointers for i, sd := range sds { s := &ss[i] s.L2 = new(filedesc.ServiceL2) - if s.L0, err = r.makeBase(s, parent, sd.GetName(), i); err != nil { + if s.L0, err = r.makeBase(s, parent, sd.GetName(), i, sb); err != nil { return nil, err } if opts := sd.GetOptions(); opts != nil { opts = clone(opts).(*descriptorpb.ServiceOptions) s.L2.Options = func() protoreflect.ProtoMessage { return opts } } - if s.L2.Methods.List, err = r.initMethodsFromDescriptorProto(sd.GetMethod(), s); err != nil { + if s.L2.Methods.List, err = r.initMethodsFromDescriptorProto(sd.GetMethod(), s, sb); err != nil { return nil, err } } return ss, nil } -func (r descsByName) initMethodsFromDescriptorProto(mds []*descriptorpb.MethodDescriptorProto, parent protoreflect.Descriptor) (ms []filedesc.Method, err error) { +func (r descsByName) initMethodsFromDescriptorProto(mds []*descriptorpb.MethodDescriptorProto, parent protoreflect.Descriptor, sb *strs.Builder) (ms []filedesc.Method, err error) { ms = make([]filedesc.Method, len(mds)) // allocate up-front to ensure stable pointers for i, md := range mds { m := &ms[i] - if m.L0, err = r.makeBase(m, parent, md.GetName(), i); err != nil { + if m.L0, err = r.makeBase(m, parent, md.GetName(), i, sb); err != nil { return nil, err } if opts := md.GetOptions(); opts != nil { @@ -214,7 +215,7 @@ func (r descsByName) initMethodsFromDescriptorProto(mds []*descriptorpb.MethodDe return ms, nil } -func (r descsByName) makeBase(child, parent protoreflect.Descriptor, name string, idx int) (filedesc.BaseL0, error) { +func (r descsByName) makeBase(child, parent protoreflect.Descriptor, name string, idx int, sb *strs.Builder) (filedesc.BaseL0, error) { if !protoreflect.Name(name).IsValid() { return filedesc.BaseL0{}, errors.New("descriptor %q has an invalid nested name: %q", parent.FullName(), name) } @@ -223,9 +224,9 @@ func (r descsByName) makeBase(child, parent protoreflect.Descriptor, name string // Note that enum values are a sibling to the enum parent in the namespace. var fullName protoreflect.FullName if _, ok := parent.(protoreflect.EnumDescriptor); ok { - fullName = parent.FullName().Parent().Append(protoreflect.Name(name)) + fullName = sb.AppendFullName(parent.FullName().Parent(), protoreflect.Name(name)) } else { - fullName = parent.FullName().Append(protoreflect.Name(name)) + fullName = sb.AppendFullName(parent.FullName(), protoreflect.Name(name)) } if _, ok := r[fullName]; ok { return filedesc.BaseL0{}, errors.New("descriptor %q already declared", fullName) diff --git a/reflect/protodesc/desc_validate.go b/reflect/protodesc/desc_validate.go index 7f5c8783..9a7fbac2 100644 --- a/reflect/protodesc/desc_validate.go +++ b/reflect/protodesc/desc_validate.go @@ -11,6 +11,7 @@ import ( "google.golang.org/protobuf/internal/encoding/wire" "google.golang.org/protobuf/internal/errors" "google.golang.org/protobuf/internal/filedesc" + "google.golang.org/protobuf/internal/strs" "google.golang.org/protobuf/reflect/protoreflect" "google.golang.org/protobuf/types/descriptorpb" @@ -53,7 +54,7 @@ func validateEnumDeclarations(es []filedesc.Enum, eds []*descriptorpb.EnumDescri prefix := strings.Replace(strings.ToLower(string(e.Name())), "_", "", -1) for i := 0; i < e.Values().Len(); i++ { v1 := e.Values().Get(i) - s := enumValueName(trimEnumPrefix(v1.Name(), prefix)) + s := strs.EnumValueName(strs.TrimEnumPrefix(string(v1.Name()), prefix)) if v2, ok := names[s]; ok && v1.Number() != v2.Number() { return errors.New("enum %q using proto3 semantics has conflict: %q with %q", e.FullName(), v1.Name(), v2.Name()) } @@ -209,7 +210,7 @@ func validateExtensionDeclarations(xs []filedesc.Extension, xds []*descriptorpb. return errors.New("extension field %q has an invalid cardinality: %d", x.FullName(), x.Cardinality()) } if xd.JsonName != nil { - if xd.GetJsonName() != jsonName(x.Name()) { + if xd.GetJsonName() != strs.JSONCamelCase(string(x.Name())) { return errors.New("extension field %q may not have an explicitly set JSON name: %q", x.FullName(), xd.GetJsonName()) } } @@ -305,7 +306,7 @@ func checkValidMap(fd protoreflect.FieldDescriptor) error { return nil case fd.FullName().Parent() != md.FullName().Parent(): return errors.New("message and field must be declared in the same scope") - case md.Name() != mapEntryName(fd.Name()): + case md.Name() != protoreflect.Name(strs.MapEntryName(string(fd.Name()))): return errors.New("incorrect implicit map entry name") case fd.Cardinality() != protoreflect.Repeated: return errors.New("field must be repeated") @@ -339,87 +340,3 @@ func checkValidMap(fd protoreflect.FieldDescriptor) error { } return nil } - -// mapEntryName derives the name of the map entry message given the field name. -// See protoc v3.8.0: src/google/protobuf/descriptor.cc:254-276,6057 -func mapEntryName(s protoreflect.Name) protoreflect.Name { - var b []byte - upperNext := true - for _, c := range s { - switch { - case c == '_': - upperNext = true - case upperNext: - b = append(b, byte(unicode.ToUpper(c))) - upperNext = false - default: - b = append(b, byte(c)) - } - } - b = append(b, "Entry"...) - return protoreflect.Name(b) -} - -// enumValueName derives the camel-cased enum value name. -// See protoc v3.8.0: src/google/protobuf/descriptor.cc:297-313 -func enumValueName(s protoreflect.Name) string { - var b []byte - upperNext := true - for _, c := range s { - switch { - case c == '_': - upperNext = true - case upperNext: - b = append(b, byte(unicode.ToUpper(c))) - upperNext = false - default: - b = append(b, byte(unicode.ToLower(c))) - upperNext = false - } - } - return string(b) -} - -// trimEnumPrefix trims the enum name prefix from an enum value name, -// where the prefix is all lowercase without underscores. -// See protoc v3.8.0: src/google/protobuf/descriptor.cc:330-375 -func trimEnumPrefix(s protoreflect.Name, prefix string) protoreflect.Name { - s0 := s // original input - for len(s) > 0 && len(prefix) > 0 { - if s[0] == '_' { - s = s[1:] - continue - } - if unicode.ToLower(rune(s[0])) != rune(prefix[0]) { - return s0 // no prefix match - } - s, prefix = s[1:], prefix[1:] - } - if len(prefix) > 0 { - return s0 // no prefix match - } - s = protoreflect.Name(strings.TrimLeft(string(s), "_")) - if len(s) == 0 { - return s0 // avoid returning empty string - } - return s -} - -// jsonName creates a JSON name from the protobuf short name. -// See protoc v3.8.0: src/google/protobuf/descriptor.cc:278-295 -func jsonName(s protoreflect.Name) string { - var b []byte - var wasUnderscore bool - for i := 0; i < len(s); i++ { // proto identifiers are always ASCII - c := s[i] - if c != '_' { - isLower := 'a' <= c && c <= 'z' - if wasUnderscore && isLower { - c -= 'a' - 'A' - } - b = append(b, c) - } - wasUnderscore = c == '_' - } - return string(b) -} diff --git a/reflect/protodesc/file_test.go b/reflect/protodesc/file_test.go index 26bc174c..dd6dde68 100644 --- a/reflect/protodesc/file_test.go +++ b/reflect/protodesc/file_test.go @@ -11,7 +11,6 @@ import ( "google.golang.org/protobuf/encoding/prototext" "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/reflect/protoreflect" "google.golang.org/protobuf/reflect/protoregistry" "google.golang.org/protobuf/types/descriptorpb" @@ -883,57 +882,3 @@ func TestNewFile(t *testing.T) { }) } } - -func TestName(t *testing.T) { - tests := []struct { - in protoreflect.Name - enumPrefix string - wantMapEntry protoreflect.Name - wantEnumValue string - wantTrimValue protoreflect.Name - wantJSON string - }{{ - in: "abc", - enumPrefix: "", - wantMapEntry: "AbcEntry", - wantEnumValue: "Abc", - wantTrimValue: "abc", - wantJSON: "abc", - }, { - in: "foo_baR_", - enumPrefix: "foo_bar", - wantMapEntry: "FooBaREntry", - wantEnumValue: "FooBar", - wantTrimValue: "foo_baR_", - wantJSON: "fooBaR", - }, { - in: "snake_caseCamelCase", - enumPrefix: "snakecasecamel", - wantMapEntry: "SnakeCaseCamelCaseEntry", - wantEnumValue: "SnakeCasecamelcase", - wantTrimValue: "Case", - wantJSON: "snakeCaseCamelCase", - }, { - in: "FiZz_BuZz", - enumPrefix: "fizz", - wantMapEntry: "FiZzBuZzEntry", - wantEnumValue: "FizzBuzz", - wantTrimValue: "BuZz", - wantJSON: "FiZzBuZz", - }} - - for _, tt := range tests { - if got := mapEntryName(tt.in); got != tt.wantMapEntry { - t.Errorf("mapEntryName(%q) = %q, want %q", tt.in, got, tt.wantMapEntry) - } - if got := enumValueName(tt.in); got != tt.wantEnumValue { - t.Errorf("enumValueName(%q) = %q, want %q", tt.in, got, tt.wantEnumValue) - } - if got := trimEnumPrefix(tt.in, tt.enumPrefix); got != tt.wantTrimValue { - t.Errorf("trimEnumPrefix(%q, %q) = %q, want %q", tt.in, tt.enumPrefix, got, tt.wantTrimValue) - } - if got := jsonName(tt.in); got != tt.wantJSON { - t.Errorf("jsonName(%q) = %q, want %q", tt.in, got, tt.wantJSON) - } - } -}