diff --git a/.gitignore b/.gitignore index 22d0d82f..20e54cfa 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ vendor +cmd/protoc-gen-go/protoc-gen-go diff --git a/cmd/protoc-gen-go/golden_test.go b/cmd/protoc-gen-go/golden_test.go new file mode 100644 index 00000000..d0ed8eaa --- /dev/null +++ b/cmd/protoc-gen-go/golden_test.go @@ -0,0 +1,145 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "bytes" + "flag" + "go/build" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "testing" +) + +// Set --regenerate to regenerate the golden files. +var regenerate = flag.Bool("regenerate", false, "regenerate golden files") + +// When the environment variable RUN_AS_PROTOC_GEN_GO is set, we skip running +// tests and instead act as protoc-gen-go. This allows the test binary to +// pass itself to protoc. +func init() { + if os.Getenv("RUN_AS_PROTOC_GEN_GO") != "" { + main() + os.Exit(0) + } +} + +func TestGolden(t *testing.T) { + workdir, err := ioutil.TempDir("", "proto-test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(workdir) + + // Find all the proto files we need to compile. We assume that each directory + // contains the files for a single package. + supportTypeAliases := hasReleaseTag("go1.9") + packages := map[string][]string{} + err = filepath.Walk("testdata", func(path string, info os.FileInfo, err error) error { + if filepath.Base(path) == "import_public" && !supportTypeAliases { + // Public imports require type alias support. + return filepath.SkipDir + } + if !strings.HasSuffix(path, ".proto") { + return nil + } + dir := filepath.Dir(path) + packages[dir] = append(packages[dir], path) + return nil + }) + if err != nil { + t.Fatal(err) + } + + // Compile each package, using this binary as protoc-gen-go. + for _, sources := range packages { + args := []string{"-Itestdata", "--go_out=plugins=grpc,paths=source_relative:" + workdir} + args = append(args, sources...) + protoc(t, args) + } + + // Compare each generated file to the golden version. + filepath.Walk(workdir, func(genPath string, info os.FileInfo, _ error) error { + if info.IsDir() { + return nil + } + + // For each generated file, figure out the path to the corresponding + // golden file in the testdata directory. + relPath, err := filepath.Rel(workdir, genPath) + if err != nil { + t.Errorf("filepath.Rel(%q, %q): %v", workdir, genPath, err) + return nil + } + if filepath.SplitList(relPath)[0] == ".." { + t.Errorf("generated file %q is not relative to %q", genPath, workdir) + } + goldenPath := filepath.Join("testdata", relPath) + + got, err := ioutil.ReadFile(genPath) + if err != nil { + t.Error(err) + return nil + } + if *regenerate { + // If --regenerate set, just rewrite the golden files. + err := ioutil.WriteFile(goldenPath, got, 0666) + if err != nil { + t.Error(err) + } + return nil + } + + want, err := ioutil.ReadFile(goldenPath) + if err != nil { + t.Error(err) + return nil + } + + want = fdescRE.ReplaceAll(want, nil) + got = fdescRE.ReplaceAll(got, nil) + if bytes.Equal(got, want) { + return nil + } + + cmd := exec.Command("diff", "-u", goldenPath, genPath) + out, _ := cmd.CombinedOutput() + t.Errorf("golden file differs: %v\n%v", relPath, string(out)) + return nil + }) +} + +var fdescRE = regexp.MustCompile(`(?ms)^var fileDescriptor.*}`) + +func protoc(t *testing.T, args []string) { + cmd := exec.Command("protoc", "--plugin=protoc-gen-go="+os.Args[0]) + cmd.Args = append(cmd.Args, args...) + // We set the RUN_AS_PROTOC_GEN_GO environment variable to indicate that + // the subprocess should act as a proto compiler rather than a test. + cmd.Env = append(os.Environ(), "RUN_AS_PROTOC_GEN_GO=1") + out, err := cmd.CombinedOutput() + if len(out) > 0 || err != nil { + t.Log("RUNNING: ", strings.Join(cmd.Args, " ")) + } + if len(out) > 0 { + t.Log(string(out)) + } + if err != nil { + t.Fatalf("protoc: %v", err) + } +} + +func hasReleaseTag(want string) bool { + for _, tag := range build.Default.ReleaseTags { + if tag == want { + return true + } + } + return false +} diff --git a/cmd/protoc-gen-go/main.go b/cmd/protoc-gen-go/main.go new file mode 100644 index 00000000..9ea14206 --- /dev/null +++ b/cmd/protoc-gen-go/main.go @@ -0,0 +1,35 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// The protoc-gen-go binary is a protoc plugin to generate a Go protocol +// buffer package. +package main + +import ( + "strings" + + "google.golang.org/proto/protogen" +) + +func main() { + protogen.Run(func(gen *protogen.Plugin) error { + for _, f := range gen.Files { + if !f.Generate { + continue + } + genFile(gen, f) + } + return nil + }) +} + +func genFile(gen *protogen.Plugin, f *protogen.File) { + g := gen.NewGeneratedFile(strings.TrimSuffix(f.Desc.GetName(), ".proto") + ".pb.go") + g.P("// Code generated by protoc-gen-go. DO NOT EDIT.") + g.P("// source: ", f.Desc.GetName()) + g.P() + g.P("package TODO") + + // TODO: Everything. +} diff --git a/cmd/protoc-gen-go/testdata/proto2/proto2.pb.go b/cmd/protoc-gen-go/testdata/proto2/proto2.pb.go new file mode 100644 index 00000000..558363d3 --- /dev/null +++ b/cmd/protoc-gen-go/testdata/proto2/proto2.pb.go @@ -0,0 +1,4 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: proto2/proto2.proto + +package TODO diff --git a/cmd/protoc-gen-go/testdata/proto2/proto2.proto b/cmd/protoc-gen-go/testdata/proto2/proto2.proto new file mode 100644 index 00000000..055bc717 --- /dev/null +++ b/cmd/protoc-gen-go/testdata/proto2/proto2.proto @@ -0,0 +1,12 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +syntax = "proto2"; + +package goproto.protoc.proto2; + +option go_package = "google.golang.org/proto/cmd/protoc-gen-go/testdata/proto2"; + +message Message { +} diff --git a/go.mod b/go.mod index d94d9033..6aa64ad9 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,8 @@ module google.golang.org/proto -require github.com/google/go-cmp v0.2.0 +require ( + github.com/golang/protobuf v1.2.0 + github.com/google/go-cmp v0.2.0 + golang.org/x/net v0.0.0-20180821023952-922f4815f713 // indirect + golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f // indirect +) diff --git a/go.sum b/go.sum index 5f4f6368..1bfafa6e 100644 --- a/go.sum +++ b/go.sum @@ -1,2 +1,8 @@ +github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +golang.org/x/net v0.0.0-20180821023952-922f4815f713 h1:rMJUcaDGbG+X967I4zGKCq5laYqcGKJmpB+3jhpOhPw= +golang.org/x/net v0.0.0-20180821023952-922f4815f713/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/protogen/names.go b/protogen/names.go new file mode 100644 index 00000000..1976cfd8 --- /dev/null +++ b/protogen/names.go @@ -0,0 +1,54 @@ +package protogen + +import ( + "go/token" + "strconv" + "strings" + "unicode" + "unicode/utf8" +) + +// A GoImportPath is the import path of a Go package. e.g., "google.golang.org/genproto/protobuf". +type GoImportPath string + +func (p GoImportPath) String() string { return strconv.Quote(string(p)) } + +// A GoPackageName is the name of a Go package. e.g., "protobuf". +type GoPackageName string + +// cleanPacakgeName converts a string to a valid Go package name. +func cleanPackageName(name string) GoPackageName { + name = strings.Map(badToUnderscore, name) + // Identifier must not be keyword: insert _. + if token.Lookup(name).IsKeyword() { + name = "_" + name + } + // Identifier must not begin with digit: insert _. + if r, _ := utf8.DecodeRuneInString(name); unicode.IsDigit(r) { + name = "_" + name + } + return GoPackageName(name) +} + +// badToUnderscore is the mapping function used to generate Go names from package names, +// which can be dotted in the input .proto file. It replaces non-identifier characters such as +// dot or dash with underscore. +func badToUnderscore(r rune) rune { + if unicode.IsLetter(r) || unicode.IsDigit(r) || r == '_' { + return r + } + return '_' +} + +// baseName returns the last path element of the name, with the last dotted suffix removed. +func baseName(name string) string { + // First, find the last element + if i := strings.LastIndex(name, "/"); i >= 0 { + name = name[i+1:] + } + // Now drop the suffix + if i := strings.LastIndex(name, "."); i >= 0 { + name = name[:i] + } + return name +} diff --git a/protogen/protogen.go b/protogen/protogen.go new file mode 100644 index 00000000..b10edadb --- /dev/null +++ b/protogen/protogen.go @@ -0,0 +1,224 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package protogen provides support for writing protoc plugins. +// +// Plugins for protoc, the Protocol Buffers Compiler, are programs which read +// a CodeGeneratorRequest protocol buffer from standard input and write a +// CodeGeneratorResponse protocol buffer to standard output. This package +// provides support for writing plugins which generate Go code. +package protogen + +import ( + "bytes" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + + "github.com/golang/protobuf/proto" + descpb "github.com/golang/protobuf/protoc-gen-go/descriptor" + pluginpb "github.com/golang/protobuf/protoc-gen-go/plugin" +) + +// Run executes a function as a protoc plugin. +// +// It reads a CodeGeneratorRequest message from os.Stdin, invokes the plugin +// function, and writes a CodeGeneratorResponse message to os.Stdout. +// +// If a failure occurs while reading or writing, Run prints an error to +// os.Stderr and calls os.Exit(1). +func Run(f func(*Plugin) error) { + if err := run(f); err != nil { + fmt.Fprintf(os.Stderr, "%s: %v\n", filepath.Base(os.Args[0]), err) + os.Exit(1) + } +} + +func run(f func(*Plugin) error) error { + in, err := ioutil.ReadAll(os.Stdin) + if err != nil { + return err + } + req := &pluginpb.CodeGeneratorRequest{} + if err := proto.Unmarshal(in, req); err != nil { + return err + } + gen, err := New(req) + if err != nil { + return err + } + if err := f(gen); err != nil { + // Errors from the plugin function are reported by setting the + // error field in the CodeGeneratorResponse. + // + // In contrast, errors that indicate a problem in protoc + // itself (unparsable input, I/O errors, etc.) are reported + // to stderr. + gen.Error(err) + } + resp := gen.Response() + out, err := proto.Marshal(resp) + if err != nil { + return err + } + if _, err := os.Stdout.Write(out); err != nil { + return err + } + return nil +} + +// A Plugin is a protoc plugin invocation. +type Plugin struct { + // Request is the CodeGeneratorRequest provided by protoc. + Request *pluginpb.CodeGeneratorRequest + + // Files is the set of files to generate and everything they import. + // Files appear in topological order, so each file appears before any + // file that imports it. + Files []*File + filesByName map[string]*File + + packageImportPath string // Go import path of the package we're generating code for. + + genFiles []*GeneratedFile + err error +} + +// New returns a new Plugin. +func New(req *pluginpb.CodeGeneratorRequest) (*Plugin, error) { + gen := &Plugin{ + Request: req, + filesByName: make(map[string]*File), + } + + // TODO: Figure out how to pass parameters to the generator. + for _, param := range strings.Split(req.GetParameter(), ",") { + var value string + if i := strings.Index(param, "="); i >= 0 { + value = param[i+1:] + param = param[0:i] + } + switch param { + case "": + // Ignore. + case "import_prefix": + // TODO + case "import_path": + gen.packageImportPath = value + case "paths": + // TODO + case "plugins": + // TODO + case "annotate_code": + // TODO + default: + if param[0] != 'M' { + return nil, fmt.Errorf("unknown parameter %q", param) + } + // TODO + } + } + + for _, fdesc := range gen.Request.ProtoFile { + f := newFile(gen, fdesc) + name := f.Desc.GetName() + if gen.filesByName[name] != nil { + return nil, fmt.Errorf("duplicate file name: %q", name) + } + gen.Files = append(gen.Files, f) + gen.filesByName[name] = f + } + for _, name := range gen.Request.FileToGenerate { + f, ok := gen.FileByName(name) + if !ok { + return nil, fmt.Errorf("no descriptor for generated file: %v", name) + } + f.Generate = true + } + return gen, nil +} + +// Error records an error in code generation. The generator will report the +// error back to protoc and will not produce output. +func (gen *Plugin) Error(err error) { + if gen.err == nil { + gen.err = err + } +} + +// Response returns the generator output. +func (gen *Plugin) Response() *pluginpb.CodeGeneratorResponse { + resp := &pluginpb.CodeGeneratorResponse{} + if gen.err != nil { + resp.Error = proto.String(gen.err.Error()) + return resp + } + for _, gf := range gen.genFiles { + resp.File = append(resp.File, &pluginpb.CodeGeneratorResponse_File{ + Name: proto.String(gf.path), + Content: proto.String(string(gf.Content())), + }) + } + return resp +} + +// FileByName returns the file with the given name. +func (gen *Plugin) FileByName(name string) (f *File, ok bool) { + f, ok = gen.filesByName[name] + return f, ok +} + +// A File is a .proto source file. +type File struct { + // TODO: Replace with protoreflect.FileDescriptor. + Desc *descpb.FileDescriptorProto + + // Generate is true if the generator should generate code for this file. + Generate bool +} + +func newFile(gen *Plugin, p *descpb.FileDescriptorProto) *File { + return &File{ + Desc: p, + } +} + +// A GeneratedFile is a generated file. +type GeneratedFile struct { + path string + buf bytes.Buffer +} + +// NewGeneratedFile creates a new generated file with the given path. +func (gen *Plugin) NewGeneratedFile(path string) *GeneratedFile { + g := &GeneratedFile{ + path: path, + } + gen.genFiles = append(gen.genFiles, g) + return g +} + +// P prints a line to the generated output. It converts each parameter to a +// string following the same rules as fmt.Print. It never inserts spaces +// between parameters. +// +// TODO: .meta file annotations. +func (g *GeneratedFile) P(v ...interface{}) { + for _, x := range v { + fmt.Fprint(&g.buf, x) + } + fmt.Fprintln(&g.buf) +} + +// Write implements io.Writer. +func (g *GeneratedFile) Write(p []byte) (n int, err error) { + return g.buf.Write(p) +} + +// Content returns the contents of the generated file. +func (g *GeneratedFile) Content() []byte { + return g.buf.Bytes() +} diff --git a/protogen/protogen_test.go b/protogen/protogen_test.go new file mode 100644 index 00000000..1d23cc07 --- /dev/null +++ b/protogen/protogen_test.go @@ -0,0 +1,94 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package protogen + +import ( + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/golang/protobuf/proto" + pluginpb "github.com/golang/protobuf/protoc-gen-go/plugin" +) + +func TestFiles(t *testing.T) { + gen, err := New(makeRequest(t, "testdata/go_package/no_go_package_import.proto")) + if err != nil { + t.Fatal(err) + } + for _, test := range []struct { + path string + wantGenerate bool + }{ + { + path: "go_package/no_go_package_import.proto", + wantGenerate: true, + }, + { + path: "go_package/no_go_package.proto", + wantGenerate: false, + }, + } { + f, ok := gen.FileByName(test.path) + if !ok { + t.Errorf("%q: not found by gen.FileByName", test.path) + continue + } + if f.Generate != test.wantGenerate { + t.Errorf("%q: Generate=%v, want %v", test.path, f.Generate, test.wantGenerate) + } + } +} + +// makeRequest returns a CodeGeneratorRequest for the given protoc inputs. +// +// It does this by running protoc with the current binary as the protoc-gen-go +// plugin. This "plugin" produces a single file, named 'request', which contains +// the code generator request. +func makeRequest(t *testing.T, args ...string) *pluginpb.CodeGeneratorRequest { + workdir, err := ioutil.TempDir("", "test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(workdir) + + cmd := exec.Command("protoc", "--plugin=protoc-gen-go="+os.Args[0]) + cmd.Args = append(cmd.Args, "--go_out="+workdir, "-Itestdata") + cmd.Args = append(cmd.Args, args...) + cmd.Env = append(os.Environ(), "RUN_AS_PROTOC_PLUGIN=1") + out, err := cmd.CombinedOutput() + if len(out) > 0 || err != nil { + t.Log("RUNNING: ", strings.Join(cmd.Args, " ")) + } + if len(out) > 0 { + t.Log(string(out)) + } + if err != nil { + t.Fatalf("protoc: %v", err) + } + + b, err := ioutil.ReadFile(filepath.Join(workdir, "request")) + if err != nil { + t.Fatal(err) + } + req := &pluginpb.CodeGeneratorRequest{} + if err := proto.UnmarshalText(string(b), req); err != nil { + t.Fatal(err) + } + return req +} + +func init() { + if os.Getenv("RUN_AS_PROTOC_PLUGIN") != "" { + Run(func(p *Plugin) error { + g := p.NewGeneratedFile("request") + return proto.MarshalText(g, p.Request) + }) + os.Exit(0) + } +} diff --git a/protogen/testdata/go_package/no_go_package.proto b/protogen/testdata/go_package/no_go_package.proto new file mode 100644 index 00000000..d71884c6 --- /dev/null +++ b/protogen/testdata/go_package/no_go_package.proto @@ -0,0 +1,9 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Proto source file with no go_package option. + +syntax = "proto3"; +package goproto.testdata; +message M {} diff --git a/protogen/testdata/go_package/no_go_package_import.proto b/protogen/testdata/go_package/no_go_package_import.proto new file mode 100644 index 00000000..bb1a73f7 --- /dev/null +++ b/protogen/testdata/go_package/no_go_package_import.proto @@ -0,0 +1,13 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Import of a proto source file with no go_package option. + +syntax = "proto3"; +package goproto.testdata; +import "go_package/no_go_package.proto"; +message M1 { + M Field = 1; +} +