// Copyright 2018 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // Package protoregistry provides data structures to register and lookup // protobuf descriptor types. // // The Files registry contains file descriptors and provides the ability // to iterate over the files or lookup a specific descriptor within the files. // Files only contains protobuf descriptors and has no understanding of Go // type information that may be associated with each descriptor. // // The Types registry contains descriptor types for which there is a known // Go type associated with that descriptor. It provides the ability to iterate // over the registered types or lookup a type by name. package protoregistry import ( "fmt" "reflect" "sort" "strings" "github.com/golang/protobuf/v2/internal/errors" "github.com/golang/protobuf/v2/reflect/protoreflect" ) // TODO: Perhaps Register should record the frame of where the function was // called and surface that in the error? That would help users debug duplicate // registration issues. This presumes that we provide a way to disable automatic // registration in generated code. // GlobalFiles is a global registry of file descriptors. var GlobalFiles *Files = new(Files) // GlobalTypes is the registry used by default for type lookups // unless a local registry is provided by the user. var GlobalTypes *Types = new(Types) // NotFound is a sentinel error value to indicate that the type was not found. var NotFound = errors.New("not found") // Files is a registry for looking up or iterating over files and the // descriptors contained within them. // The Find and Range methods are safe for concurrent use. type Files struct { filesByPackage filesByPackage filesByPath filesByPath } type ( filesByPackage struct { // files is a list of files all in the same package. files []protoreflect.FileDescriptor // subs is a tree of files all in a sub-package scope. // It also maps all top-level identifiers declared in files // as the notProtoPackage sentinel value. subs map[protoreflect.Name]*filesByPackage // invariant: len(Name) > 0 } filesByPath map[string][]protoreflect.FileDescriptor ) // notProtoPackage is a sentinel value to indicate that some identifier maps // to an actual protobuf declaration and is not a sub-package. var notProtoPackage = new(filesByPackage) // NewFiles returns a registry initialized with the provided set of files. // If there are duplicates, the first one takes precedence. func NewFiles(files ...protoreflect.FileDescriptor) *Files { // TODO: Should last take precedence? This allows a user to intentionally // overwrite an existing registration. // // The use case is for implementing the existing v1 proto.RegisterFile // function where the behavior is last wins. However, it could be argued // that the v1 behavior is broken, and we can switch to first wins // without violating compatibility. r := new(Files) r.Register(files...) // ignore errors; first takes precedence return r } // Register registers the provided list of file descriptors. // Placeholder files are ignored. // // If any descriptor within a file conflicts with the descriptor of any // previously registered file (e.g., two enums with the same full name), // then that file is not registered and an error is returned. // // It is permitted for multiple files to have the same file path. func (r *Files) Register(files ...protoreflect.FileDescriptor) error { var firstErr error fileLoop: for _, file := range files { if file.IsPlaceholder() { continue // TODO: Should this be an error instead? } // Register the file into the filesByPackage tree. // // The prototype package validates that a FileDescriptor is internally // consistent such it does not have conflicts within itself. // However, we need to ensure that the inserted file does not conflict // with other previously inserted files. pkg := file.Package() root := &r.filesByPackage for len(pkg) > 0 { var prefix protoreflect.Name prefix, pkg = splitPrefix(pkg) // Add a new sub-package segment. switch nextRoot := root.subs[prefix]; nextRoot { case nil: nextRoot = new(filesByPackage) if root.subs == nil { root.subs = make(map[protoreflect.Name]*filesByPackage) } root.subs[prefix] = nextRoot root = nextRoot case notProtoPackage: if firstErr == nil { name := strings.TrimSuffix(strings.TrimSuffix(string(file.Package()), string(pkg)), ".") firstErr = errors.New("file %q has a name conflict over %v", file.Path(), name) } continue fileLoop default: root = nextRoot } } // Check for top-level conflicts within the same package. // The current file cannot add any top-level declaration that conflict // with another top-level declaration or sub-package name. var conflicts []protoreflect.Name rangeTopLevelDeclarations(file, func(s protoreflect.Name) { if root.subs[s] == nil { if root.subs == nil { root.subs = make(map[protoreflect.Name]*filesByPackage) } root.subs[s] = notProtoPackage } else { conflicts = append(conflicts, s) } }) if len(conflicts) > 0 { // Remove inserted identifiers to make registration failure atomic. sort.Slice(conflicts, func(i, j int) bool { return conflicts[i] < conflicts[j] }) rangeTopLevelDeclarations(file, func(s protoreflect.Name) { i := sort.Search(len(conflicts), func(i int) bool { return conflicts[i] >= s }) if has := i < len(conflicts) && conflicts[i] == s; !has { delete(root.subs, s) // remove everything not in conflicts } }) if firstErr == nil { name := file.Package().Append(conflicts[0]) firstErr = errors.New("file %q has a name conflict over %v", file.Path(), name) } continue fileLoop } root.files = append(root.files, file) // Register the file into the filesByPath map. // // There is no check for conflicts in file path since the path is // heavily dependent on how protoc is invoked. When protoc is being // invoked by different parties in a distributed manner, it is // unreasonable to assume nor ensure that the path is unique. if r.filesByPath == nil { r.filesByPath = make(filesByPath) } r.filesByPath[file.Path()] = append(r.filesByPath[file.Path()], file) } return firstErr } // FindDescriptorByName looks up any descriptor (except files) by its full name. // Files are not handled since multiple file descriptors may belong in // the same package and have the same full name (see RangeFilesByPackage). // // This return (nil, NotFound) if not found. func (r *Files) FindDescriptorByName(name protoreflect.FullName) (protoreflect.Descriptor, error) { if r == nil { return nil, NotFound } pkg := name root := &r.filesByPackage for len(pkg) > 0 { var prefix protoreflect.Name prefix, pkg = splitPrefix(pkg) switch nextRoot := root.subs[prefix]; nextRoot { case nil: return nil, NotFound case notProtoPackage: // Search current root's package for the descriptor. for _, fd := range root.files { if d := fd.DescriptorByName(name); d != nil { return d, nil } } return nil, NotFound default: root = nextRoot } } return nil, NotFound } // RangeFiles iterates over all registered files. // The iteration order is undefined. func (r *Files) RangeFiles(f func(protoreflect.FileDescriptor) bool) { r.RangeFilesByPackage("", f) // empty package is a prefix for all packages } // RangeFilesByPackage iterates over all registered files filtered by // the given proto package prefix. It iterates over files with an exact package // match before iterating over files with general prefix match. // The iteration order is undefined within exact matches or prefix matches. func (r *Files) RangeFilesByPackage(pkg protoreflect.FullName, f func(protoreflect.FileDescriptor) bool) { if r == nil { return } if strings.HasSuffix(string(pkg), ".") { return // avoid edge case where splitPrefix allows trailing dot } root := &r.filesByPackage for len(pkg) > 0 && root != nil { var prefix protoreflect.Name prefix, pkg = splitPrefix(pkg) root = root.subs[prefix] } rangeFiles(root, f) } func rangeFiles(fs *filesByPackage, f func(protoreflect.FileDescriptor) bool) bool { if fs == nil { return true } // Iterate over exact matches. for _, fd := range fs.files { // TODO: iterate non-deterministically if !f(fd) { return false } } // Iterate over prefix matches. for _, fs := range fs.subs { if !rangeFiles(fs, f) { return false } } return true } // RangeFilesByPath iterates over all registered files filtered by // the given proto path. The iteration order is undefined. func (r *Files) RangeFilesByPath(path string, f func(protoreflect.FileDescriptor) bool) { if r == nil { return } for _, fd := range r.filesByPath[path] { // TODO: iterate non-deterministically if !f(fd) { return } } } func splitPrefix(name protoreflect.FullName) (protoreflect.Name, protoreflect.FullName) { if i := strings.IndexByte(string(name), '.'); i >= 0 { return protoreflect.Name(name[:i]), name[i+len("."):] } return protoreflect.Name(name), "" } // rangeTopLevelDeclarations iterates over the name of all top-level // declarations in the proto file. func rangeTopLevelDeclarations(fd protoreflect.FileDescriptor, f func(protoreflect.Name)) { for i := 0; i < fd.Enums().Len(); i++ { e := fd.Enums().Get(i) f(e.Name()) // TODO: Drop ranging over top-level enum values. The current // implementation of fileinit.FileBuilder does not initialize the names // for enum values in enums. Doing so reduces init time considerably. // If we drop this, it means that conflict checks in the registry // is not complete. However, this may be okay since the most common // reason for a conflict is due to vendored proto files, which are // most certainly going to have a name conflict on the parent enum. for i := 0; i < e.Values().Len(); i++ { f(e.Values().Get(i).Name()) } } for i := 0; i < fd.Messages().Len(); i++ { f(fd.Messages().Get(i).Name()) } for i := 0; i < fd.Extensions().Len(); i++ { f(fd.Extensions().Get(i).Name()) } for i := 0; i < fd.Services().Len(); i++ { f(fd.Services().Get(i).Name()) } } // Type is an interface satisfied by protoreflect.EnumType, // protoreflect.MessageType, or protoreflect.ExtensionType. type Type interface { protoreflect.Descriptor GoType() reflect.Type } var ( _ Type = protoreflect.EnumType(nil) _ Type = protoreflect.MessageType(nil) _ Type = protoreflect.ExtensionType(nil) ) // Types is a registry for looking up or iterating over descriptor types. // The Find and Range methods are safe for concurrent use. type Types struct { // Parent sets the parent registry to consult if a find operation // could not locate the appropriate entry. // // Setting a parent results in each Range operation also iterating over the // entries contained within the parent. In such a case, it is possible for // Range to emit duplicates (since they may exist in both child and parent). // Range iteration is guaranteed to iterate over local entries before // iterating over parent entries. Parent *Types // Resolver sets the local resolver to consult if the local registry does // not contain an entry. The resolver takes precedence over the parent. // // The url is a URL where the full name of the type is the last segment // of the path (i.e. string following the last '/' character). // When missing a '/' character, the URL is the full name of the type. // See documentation on the google.protobuf.Any.type_url field for details. // // If the resolver returns a result, it is not automatically registered // into the local registry. Thus, a resolver function should cache results // such that it deterministically returns the same result given the // same URL assuming the error returned is nil or NotFound. // // If the resolver returns the NotFound error, the registry will consult the // parent registry if it is set. // // Setting a resolver has no effect on the result of each Range operation. Resolver func(url string) (Type, error) // TODO: The syntax of the URL is ill-defined and the protobuf team recently // changed the documented semantics in a way that breaks prior usages. // I do not believe they can do this and need to sync up with the // protobuf team again to hash out what the proper syntax of the URL is. // TODO: Should we separate this out as a registry for each type? // // In Java, the extension and message registry are distinct classes. // Their extension registry has knowledge of distinct Java types, // while their message registry only contains descriptor information. // // In Go, we have always registered messages, enums, and extensions. // Messages and extensions are registered with Go information, while enums // are only registered with descriptor information. We cannot drop Go type // information for messages otherwise we would be unable to implement // portions of the v1 API such as ptypes.DynamicAny. // // There is no enum registry in Java. In v1, we used the enum registry // because enum types provided no reflective methods. The addition of // ProtoReflect removes that need. typesByName typesByName extensionsByMessage extensionsByMessage } type ( typesByName map[protoreflect.FullName]Type extensionsByMessage map[protoreflect.FullName]extensionsByNumber extensionsByNumber map[protoreflect.FieldNumber]protoreflect.ExtensionType ) // NewTypes returns a registry initialized with the provided set of types. // If there are conflicts, the first one takes precedence. func NewTypes(typs ...Type) *Types { // TODO: Allow setting resolver and parent via constructor? r := new(Types) r.Register(typs...) // ignore errors; first takes precedence return r } // Register registers the provided list of descriptor types. // // If a registration conflict occurs for enum, message, or extension types // (e.g., two different types have the same full name), // then the first type takes precedence and an error is returned. func (r *Types) Register(typs ...Type) error { var firstErr error typeLoop: for _, typ := range typs { switch typ.(type) { case protoreflect.EnumType, protoreflect.MessageType, protoreflect.ExtensionType: // Check for conflicts in typesByName. name := typ.FullName() if r.typesByName[name] != nil { if firstErr == nil { firstErr = errors.New("%v %v is already registered", typeName(typ), name) } continue typeLoop } // Check for conflicts in extensionsByMessage. if xt, _ := typ.(protoreflect.ExtensionType); xt != nil { field := xt.Number() message := xt.ExtendedType().FullName() if r.extensionsByMessage[message][field] != nil { if firstErr == nil { firstErr = errors.New("extension %v is already registered on message %v", name, message) } continue typeLoop } // Update extensionsByMessage. if r.extensionsByMessage == nil { r.extensionsByMessage = make(extensionsByMessage) } if r.extensionsByMessage[message] == nil { r.extensionsByMessage[message] = make(extensionsByNumber) } r.extensionsByMessage[message][field] = xt } // Update typesByName. if r.typesByName == nil { r.typesByName = make(typesByName) } r.typesByName[name] = typ default: if firstErr == nil { firstErr = errors.New("invalid type: %v", typeName(typ)) } } } return firstErr } // FindEnumByName looks up an enum by its full name. // E.g., "google.protobuf.Field.Kind". // // This returns (nil, NotFound) if not found. func (r *Types) FindEnumByName(enum protoreflect.FullName) (protoreflect.EnumType, error) { r.globalCheck() if r == nil { return nil, NotFound } v, _ := r.typesByName[enum] if v == nil && r.Resolver != nil { var err error v, err = r.Resolver(string(enum)) if err != nil && err != NotFound { return nil, err } } if v != nil { if et, _ := v.(protoreflect.EnumType); et != nil { return et, nil } return nil, errors.New("found wrong type: got %v, want enum", typeName(v)) } return r.Parent.FindEnumByName(enum) } // FindMessageByName looks up a message by its full name. // E.g., "google.protobuf.Any" // // This return (nil, NotFound) if not found. func (r *Types) FindMessageByName(message protoreflect.FullName) (protoreflect.MessageType, error) { // The full name by itself is a valid URL. return r.FindMessageByURL(string(message)) } // FindMessageByURL looks up a message by a URL identifier. // See Resolver for the format of the URL. // // This returns (nil, NotFound) if not found. func (r *Types) FindMessageByURL(url string) (protoreflect.MessageType, error) { r.globalCheck() if r == nil { return nil, NotFound } message := protoreflect.FullName(url) if i := strings.LastIndexByte(url, '/'); i >= 0 { message = message[i+len("/"):] } v, _ := r.typesByName[message] if v == nil && r.Resolver != nil { var err error v, err = r.Resolver(url) if err != nil && err != NotFound { return nil, err } } if v != nil { if mt, _ := v.(protoreflect.MessageType); mt != nil { return mt, nil } return nil, errors.New("found wrong type: got %v, want message", typeName(v)) } return r.Parent.FindMessageByURL(url) } // FindExtensionByName looks up a extension field by the field's full name. // Note that this is the full name of the field as determined by // where the extension is declared and is unrelated to the full name of the // message being extended. // // This returns (nil, NotFound) if not found. func (r *Types) FindExtensionByName(field protoreflect.FullName) (protoreflect.ExtensionType, error) { r.globalCheck() if r == nil { return nil, NotFound } v, _ := r.typesByName[field] if v == nil && r.Resolver != nil { var err error v, err = r.Resolver(string(field)) if err != nil && err != NotFound { return nil, err } } if v != nil { if xt, _ := v.(protoreflect.ExtensionType); xt != nil { return xt, nil } return nil, errors.New("found wrong type: got %v, want extension", typeName(v)) } return r.Parent.FindExtensionByName(field) } // FindExtensionByNumber looks up a extension field by the field number // within some parent message, identified by full name. // // This returns (nil, NotFound) if not found. func (r *Types) FindExtensionByNumber(message protoreflect.FullName, field protoreflect.FieldNumber) (protoreflect.ExtensionType, error) { r.globalCheck() if r == nil { return nil, NotFound } if xt, ok := r.extensionsByMessage[message][field]; ok { return xt, nil } return r.Parent.FindExtensionByNumber(message, field) } // RangeEnums iterates over all registered enums. // Iteration order is undefined. func (r *Types) RangeEnums(f func(protoreflect.EnumType) bool) { r.globalCheck() if r == nil { return } for _, typ := range r.typesByName { if et, ok := typ.(protoreflect.EnumType); ok { if !f(et) { return } } } r.Parent.RangeEnums(f) } // RangeMessages iterates over all registered messages. // Iteration order is undefined. func (r *Types) RangeMessages(f func(protoreflect.MessageType) bool) { r.globalCheck() if r == nil { return } for _, typ := range r.typesByName { if mt, ok := typ.(protoreflect.MessageType); ok { if !f(mt) { return } } } r.Parent.RangeMessages(f) } // RangeExtensions iterates over all registered extensions. // Iteration order is undefined. func (r *Types) RangeExtensions(f func(protoreflect.ExtensionType) bool) { r.globalCheck() if r == nil { return } for _, typ := range r.typesByName { if xt, ok := typ.(protoreflect.ExtensionType); ok { if !f(xt) { return } } } r.Parent.RangeExtensions(f) } // RangeExtensionsByMessage iterates over all registered extensions filtered // by a given message type. Iteration order is undefined. func (r *Types) RangeExtensionsByMessage(message protoreflect.FullName, f func(protoreflect.ExtensionType) bool) { r.globalCheck() if r == nil { return } for _, xt := range r.extensionsByMessage[message] { if !f(xt) { return } } r.Parent.RangeExtensionsByMessage(message, f) } func (r *Types) globalCheck() { if r == GlobalTypes && (r.Parent != nil || r.Resolver != nil) { panic("GlobalTypes.Parent and GlobalTypes.Resolver cannot be set") } } func typeName(t Type) string { switch t.(type) { case protoreflect.EnumType: return "enum" case protoreflect.MessageType: return "message" case protoreflect.ExtensionType: return "extension" default: return fmt.Sprintf("%T", t) } }