mirror of
https://github.com/protocolbuffers/protobuf-go.git
synced 2025-01-06 10:01:25 +00:00
8012b444ee
This CL takes a significantly different approach to generating support for protobuf reflection. The previous approach involved generating a large number of Go literals to represent the reflection information. While that approach was correct, it resulted in too much binary bloat. The approach taken here initializes the reflection information from the raw descriptor proto, which is a relatively dense representation of the protobuf reflection information. In order to keep initialization cost low, several measures were taken: * At program init, the bare minimum is parsed in order to initialize naming information for enums, messages, extensions, and services declared in the file. This is done because those top-level declarations are often relevant for registration. * Only upon first are most of the other data structures for protobuf reflection actually initialized. * Instead of using proto.Unmarshal, a hand-written unmarshaler is used. This allows us to avoid a dependendency on the descriptor proto and also because the API for the descriptor proto is fundamentally non-performant since it requires an allocation for every primitive field. At a high-level, the new implementation lives in internal/fileinit. Several changes were made to other parts of the repository: * cmd/protoc-gen-go: * Stop compressing the raw descriptors. While compression does reduce the size of the descriptors by approximately 2x, it is a pre-mature optimization since the descriptors themselves are around 1% of the total binary bloat that is due to generated protobufs. * Seeding protobuf reflection from the raw descriptor significantly simplifies the generator implementation since it is no longer responsible for constructing a tree of Go literals to represent the same information. * We remove the generation of the shadow types and instead call protoimpl.MessageType.MessageOf. Unfortunately, this incurs an allocation for every call to ProtoReflect since we need to allocate a tuple that wraps a pointer to the message value, and a pointer to message type. * internal/impl: * We add a MessageType.GoType field and make it required that it is set prior to first use. This is done so that we can avoid calling MessageType.init except for when it is actually needed. The allows code to call (*FooMessage)(nil).ProtoReflect().Type() without fearing that the init code will run, possibly triggering a recursive deadlock (where the init code depends on getting the Type of some dependency which may be declared within the same file). * internal/cmd/generate-types: * The code to generate reflect/prototype/protofile_list_gen.go was copied and altered to generated internal/fileinit.desc_list_gen.go. At a high-level this CL adds significant technical complexity. However, this is offset by several possible future changes: * The prototype package can be drastically simplified. We can probably reimplement internal/legacy to use internal/fileinit instead, allowing us to drop another dependency on the prototype package. As a result, we can probably delete most of the constructor types in that package. * With the prototype package significantly pruned, and the fact that generated code no longer depend on depends on that package, we can consider merging what's left of prototype into protodesc. Change-Id: I6090f023f2e1b6afaf62bd3ae883566242e30715 Reviewed-on: https://go-review.googlesource.com/c/158539 Reviewed-by: Herbie Ong <herbie@google.com> Reviewed-by: Joe Tsai <thebrokentoaster@gmail.com>
357 lines
9.2 KiB
Go
357 lines
9.2 KiB
Go
// Copyright 2018 The Go Authors. All rights reserved.
|
|
// Use of this source code is governed by a BSD-style
|
|
// license that can be found in the LICENSE file.
|
|
|
|
package fileinit
|
|
|
|
import (
|
|
wire "github.com/golang/protobuf/v2/internal/encoding/wire"
|
|
pref "github.com/golang/protobuf/v2/reflect/protoreflect"
|
|
)
|
|
|
|
func newFileDesc(fb FileBuilder) *fileDesc {
|
|
file := &fileDesc{fileInit: fileInit{
|
|
RawDescriptor: fb.RawDescriptor,
|
|
GoTypes: fb.GoTypes,
|
|
DependencyIndexes: fb.DependencyIndexes,
|
|
}}
|
|
file.initDecls(len(fb.EnumOutputTypes), len(fb.MessageOutputTypes), len(fb.ExtensionOutputTypes))
|
|
file.unmarshalSeed(fb.RawDescriptor)
|
|
|
|
// Extended message dependencies are eagerly handled since registration
|
|
// needs this information at program init time.
|
|
for i := range file.allExtensions {
|
|
xd := &file.allExtensions[i]
|
|
xd.extendedType = file.popMessageDependency()
|
|
}
|
|
|
|
file.checkDecls()
|
|
return file
|
|
}
|
|
|
|
// initDecls pre-allocates slices for the exact number of enums, messages
|
|
// (excluding map entries), and extensions declared in the proto file.
|
|
// This is done to avoid regrowing the slice, which would change the address
|
|
// for any previously seen declaration.
|
|
//
|
|
// The alloc methods "allocates" slices by pulling from the capacity.
|
|
func (fd *fileDecls) initDecls(numEnums, numMessages, numExtensions int) {
|
|
*fd = fileDecls{
|
|
allEnums: make([]enumDesc, 0, numEnums),
|
|
allMessages: make([]messageDesc, 0, numMessages),
|
|
allExtensions: make([]extensionDesc, 0, numExtensions),
|
|
}
|
|
}
|
|
|
|
func (fd *fileDecls) allocEnums(n int) []enumDesc {
|
|
total := len(fd.allEnums)
|
|
es := fd.allEnums[total : total+n]
|
|
fd.allEnums = fd.allEnums[:total+n]
|
|
return es
|
|
}
|
|
func (fd *fileDecls) allocMessages(n int) []messageDesc {
|
|
total := len(fd.allMessages)
|
|
ms := fd.allMessages[total : total+n]
|
|
fd.allMessages = fd.allMessages[:total+n]
|
|
return ms
|
|
}
|
|
func (fd *fileDecls) allocExtensions(n int) []extensionDesc {
|
|
total := len(fd.allExtensions)
|
|
xs := fd.allExtensions[total : total+n]
|
|
fd.allExtensions = fd.allExtensions[:total+n]
|
|
return xs
|
|
}
|
|
|
|
// checkDecls performs a sanity check that the expected number of expected
|
|
// declarations matches the number that were found in the descriptor proto.
|
|
func (fd *fileDecls) checkDecls() {
|
|
if len(fd.allEnums) != cap(fd.allEnums) ||
|
|
len(fd.allMessages) != cap(fd.allMessages) ||
|
|
len(fd.allExtensions) != cap(fd.allExtensions) {
|
|
panic("mismatching cardinality")
|
|
}
|
|
}
|
|
|
|
func (fd *fileDesc) unmarshalSeed(b []byte) {
|
|
nb := getNameBuilder()
|
|
defer putNameBuilder(nb)
|
|
|
|
var prevField pref.FieldNumber
|
|
var numEnums, numMessages, numExtensions, numServices int
|
|
var posEnums, posMessages, posExtensions, posServices int
|
|
b0 := b
|
|
for len(b) > 0 {
|
|
num, typ, n := wire.ConsumeTag(b)
|
|
b = b[n:]
|
|
switch typ {
|
|
case wire.BytesType:
|
|
v, m := wire.ConsumeBytes(b)
|
|
b = b[m:]
|
|
switch num {
|
|
case fileDesc_Name:
|
|
fd.path = nb.MakeString(v)
|
|
case fileDesc_Package:
|
|
fd.protoPackage = pref.FullName(nb.MakeString(v))
|
|
case fileDesc_Enums:
|
|
if prevField != fileDesc_Enums {
|
|
if numEnums > 0 {
|
|
panic("non-contiguous repeated field")
|
|
}
|
|
posEnums = len(b0) - len(b) - n - m
|
|
}
|
|
numEnums++
|
|
case fileDesc_Messages:
|
|
if prevField != fileDesc_Messages {
|
|
if numMessages > 0 {
|
|
panic("non-contiguous repeated field")
|
|
}
|
|
posMessages = len(b0) - len(b) - n - m
|
|
}
|
|
numMessages++
|
|
case fileDesc_Extensions:
|
|
if prevField != fileDesc_Extensions {
|
|
if numExtensions > 0 {
|
|
panic("non-contiguous repeated field")
|
|
}
|
|
posExtensions = len(b0) - len(b) - n - m
|
|
}
|
|
numExtensions++
|
|
case fileDesc_Services:
|
|
if prevField != fileDesc_Services {
|
|
if numServices > 0 {
|
|
panic("non-contiguous repeated field")
|
|
}
|
|
posServices = len(b0) - len(b) - n - m
|
|
}
|
|
numServices++
|
|
}
|
|
prevField = num
|
|
default:
|
|
m := wire.ConsumeFieldValue(num, typ, b)
|
|
b = b[m:]
|
|
prevField = -1 // ignore known field numbers of unknown wire type
|
|
}
|
|
}
|
|
|
|
// Must allocate all declarations before parsing each descriptor type
|
|
// to ensure we handled all descriptors in "flattened ordering".
|
|
if numEnums > 0 {
|
|
fd.enums.list = fd.allocEnums(numEnums)
|
|
}
|
|
if numMessages > 0 {
|
|
fd.messages.list = fd.allocMessages(numMessages)
|
|
}
|
|
if numExtensions > 0 {
|
|
fd.extensions.list = fd.allocExtensions(numExtensions)
|
|
}
|
|
if numServices > 0 {
|
|
fd.services.list = make([]serviceDesc, numServices)
|
|
}
|
|
|
|
if numEnums > 0 {
|
|
b := b0[posEnums:]
|
|
for i := range fd.enums.list {
|
|
_, n := wire.ConsumeVarint(b)
|
|
v, m := wire.ConsumeBytes(b[n:])
|
|
fd.enums.list[i].unmarshalSeed(v, nb, fd, fd, i)
|
|
b = b[n+m:]
|
|
}
|
|
}
|
|
if numMessages > 0 {
|
|
b := b0[posMessages:]
|
|
for i := range fd.messages.list {
|
|
_, n := wire.ConsumeVarint(b)
|
|
v, m := wire.ConsumeBytes(b[n:])
|
|
fd.messages.list[i].unmarshalSeed(v, nb, fd, fd, i)
|
|
b = b[n+m:]
|
|
}
|
|
}
|
|
if numExtensions > 0 {
|
|
b := b0[posExtensions:]
|
|
for i := range fd.extensions.list {
|
|
_, n := wire.ConsumeVarint(b)
|
|
v, m := wire.ConsumeBytes(b[n:])
|
|
fd.extensions.list[i].unmarshalSeed(v, nb, fd, fd, i)
|
|
b = b[n+m:]
|
|
}
|
|
}
|
|
if numServices > 0 {
|
|
b := b0[posServices:]
|
|
for i := range fd.services.list {
|
|
_, n := wire.ConsumeVarint(b)
|
|
v, m := wire.ConsumeBytes(b[n:])
|
|
fd.services.list[i].unmarshalSeed(v, nb, fd, fd, i)
|
|
b = b[n+m:]
|
|
}
|
|
}
|
|
}
|
|
|
|
func (ed *enumDesc) unmarshalSeed(b []byte, nb *nameBuilder, pf *fileDesc, pd pref.Descriptor, i int) {
|
|
ed.parentFile = pf
|
|
ed.parent = pd
|
|
ed.index = i
|
|
|
|
for len(b) > 0 {
|
|
num, typ, n := wire.ConsumeTag(b)
|
|
b = b[n:]
|
|
switch typ {
|
|
case wire.BytesType:
|
|
v, m := wire.ConsumeBytes(b)
|
|
b = b[m:]
|
|
switch num {
|
|
case enumDesc_Name:
|
|
ed.fullName = nb.AppendFullName(pd.FullName(), v)
|
|
}
|
|
default:
|
|
m := wire.ConsumeFieldValue(num, typ, b)
|
|
b = b[m:]
|
|
}
|
|
}
|
|
}
|
|
|
|
func (md *messageDesc) unmarshalSeed(b []byte, nb *nameBuilder, pf *fileDesc, pd pref.Descriptor, i int) {
|
|
md.parentFile = pf
|
|
md.parent = pd
|
|
md.index = i
|
|
|
|
var prevField pref.FieldNumber
|
|
var numEnums, numMessages, numExtensions int
|
|
var posEnums, posMessages, posExtensions int
|
|
b0 := b
|
|
for len(b) > 0 {
|
|
num, typ, n := wire.ConsumeTag(b)
|
|
b = b[n:]
|
|
switch typ {
|
|
case wire.BytesType:
|
|
v, m := wire.ConsumeBytes(b)
|
|
b = b[m:]
|
|
switch num {
|
|
case messageDesc_Name:
|
|
md.fullName = nb.AppendFullName(pd.FullName(), v)
|
|
case messageDesc_Enums:
|
|
if prevField != messageDesc_Enums {
|
|
if numEnums > 0 {
|
|
panic("non-contiguous repeated field")
|
|
}
|
|
posEnums = len(b0) - len(b) - n - m
|
|
}
|
|
numEnums++
|
|
case messageDesc_Messages:
|
|
if prevField != messageDesc_Messages {
|
|
if numMessages > 0 {
|
|
panic("non-contiguous repeated field")
|
|
}
|
|
posMessages = len(b0) - len(b) - n - m
|
|
}
|
|
numMessages++
|
|
case messageDesc_Extensions:
|
|
if prevField != messageDesc_Extensions {
|
|
if numExtensions > 0 {
|
|
panic("non-contiguous repeated field")
|
|
}
|
|
posExtensions = len(b0) - len(b) - n - m
|
|
}
|
|
numExtensions++
|
|
}
|
|
prevField = num
|
|
default:
|
|
m := wire.ConsumeFieldValue(num, typ, b)
|
|
b = b[m:]
|
|
prevField = -1 // ignore known field numbers of unknown wire type
|
|
}
|
|
}
|
|
|
|
// Must allocate all declarations before parsing each descriptor type
|
|
// to ensure we handled all descriptors in "flattened ordering".
|
|
if numEnums > 0 {
|
|
md.enums.list = md.parentFile.allocEnums(numEnums)
|
|
}
|
|
if numMessages > 0 {
|
|
md.messages.list = md.parentFile.allocMessages(numMessages)
|
|
}
|
|
if numExtensions > 0 {
|
|
md.extensions.list = md.parentFile.allocExtensions(numExtensions)
|
|
}
|
|
|
|
if numEnums > 0 {
|
|
b := b0[posEnums:]
|
|
for i := range md.enums.list {
|
|
_, n := wire.ConsumeVarint(b)
|
|
v, m := wire.ConsumeBytes(b[n:])
|
|
md.enums.list[i].unmarshalSeed(v, nb, pf, md, i)
|
|
b = b[n+m:]
|
|
}
|
|
}
|
|
if numMessages > 0 {
|
|
b := b0[posMessages:]
|
|
for i := range md.messages.list {
|
|
_, n := wire.ConsumeVarint(b)
|
|
v, m := wire.ConsumeBytes(b[n:])
|
|
md.messages.list[i].unmarshalSeed(v, nb, pf, md, i)
|
|
b = b[n+m:]
|
|
}
|
|
}
|
|
if numExtensions > 0 {
|
|
b := b0[posExtensions:]
|
|
for i := range md.extensions.list {
|
|
_, n := wire.ConsumeVarint(b)
|
|
v, m := wire.ConsumeBytes(b[n:])
|
|
md.extensions.list[i].unmarshalSeed(v, nb, pf, md, i)
|
|
b = b[n+m:]
|
|
}
|
|
}
|
|
}
|
|
|
|
func (xd *extensionDesc) unmarshalSeed(b []byte, nb *nameBuilder, pf *fileDesc, pd pref.Descriptor, i int) {
|
|
xd.parentFile = pf
|
|
xd.parent = pd
|
|
xd.index = i
|
|
|
|
for len(b) > 0 {
|
|
num, typ, n := wire.ConsumeTag(b)
|
|
b = b[n:]
|
|
switch typ {
|
|
case wire.VarintType:
|
|
v, m := wire.ConsumeVarint(b)
|
|
b = b[m:]
|
|
switch num {
|
|
case fieldDesc_Number:
|
|
xd.number = pref.FieldNumber(v)
|
|
}
|
|
case wire.BytesType:
|
|
v, m := wire.ConsumeBytes(b)
|
|
b = b[m:]
|
|
switch num {
|
|
case fieldDesc_Name:
|
|
xd.fullName = nb.AppendFullName(pd.FullName(), v)
|
|
}
|
|
default:
|
|
m := wire.ConsumeFieldValue(num, typ, b)
|
|
b = b[m:]
|
|
}
|
|
}
|
|
}
|
|
|
|
func (sd *serviceDesc) unmarshalSeed(b []byte, nb *nameBuilder, pf *fileDesc, pd pref.Descriptor, i int) {
|
|
sd.parentFile = pf
|
|
sd.parent = pd
|
|
sd.index = i
|
|
|
|
for len(b) > 0 {
|
|
num, typ, n := wire.ConsumeTag(b)
|
|
b = b[n:]
|
|
switch typ {
|
|
case wire.BytesType:
|
|
v, m := wire.ConsumeBytes(b)
|
|
b = b[m:]
|
|
switch num {
|
|
case serviceDesc_Name:
|
|
sd.fullName = nb.AppendFullName(pd.FullName(), v)
|
|
}
|
|
default:
|
|
m := wire.ConsumeFieldValue(num, typ, b)
|
|
b = b[m:]
|
|
}
|
|
}
|
|
}
|