testing/protocmp: make Message implement proto.Message

By having the Message type implement proto.Message, it can be passed
to other general-purpose protobuf functions such as proto.Merge.
This provides a convenient way to convert the Message back into a
concrete form that may be easier to work with.

A minor consequence of this change is that invalid messages are
converted to an invalid Message that preserves type information.
Previously, they were simply transformed to a nil Message.

Change-Id: I6fca8a0879408c7f44a99d52734613302fa23f70
Reviewed-on: https://go-review.googlesource.com/c/protobuf/+/221422
Reviewed-by: Damien Neil <dneil@google.com>

testing/protocmp/reflect.go

@@ -0,0 +1,260 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package protocmp
+
+import (
+	"reflect"
+	"sort"
+	"strconv"
+	"strings"
+
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/reflect/protoreflect"
+	"google.golang.org/protobuf/runtime/protoiface"
+)
+
+func reflectValueOf(v interface{}) protoreflect.Value {
+	switch v := v.(type) {
+	case Enum:
+		return protoreflect.ValueOfEnum(v.Number())
+	case Message:
+		return protoreflect.ValueOfMessage(v.ProtoReflect())
+	case []byte:
+		return protoreflect.ValueOfBytes(v) // avoid overlap with reflect.Slice check below
+	default:
+		switch rv := reflect.ValueOf(v); {
+		case rv.Kind() == reflect.Slice:
+			return protoreflect.ValueOfList(reflectList{rv})
+		case rv.Kind() == reflect.Map:
+			return protoreflect.ValueOfMap(reflectMap{rv})
+		default:
+			return protoreflect.ValueOf(v)
+		}
+	}
+}
+
+type reflectMessage Message
+
+func (m reflectMessage) stringKey(fd protoreflect.FieldDescriptor) string {
+	if m.Descriptor() != fd.Parent() {
+		panic("mismatching containing message")
+	}
+	if fd.IsExtension() {
+		return string("[" + fd.FullName() + "]")
+	}
+	return string(fd.Name())
+}
+
+func (m reflectMessage) Descriptor() protoreflect.MessageDescriptor {
+	return (Message)(m).Descriptor()
+}
+func (m reflectMessage) Type() protoreflect.MessageType {
+	return reflectMessageType{m.Descriptor()}
+}
+func (m reflectMessage) New() protoreflect.Message {
+	return m.Type().New()
+}
+func (m reflectMessage) Interface() protoreflect.ProtoMessage {
+	return Message(m)
+}
+func (m reflectMessage) Range(f func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool) {
+	// Range over populated known fields.
+	fds := m.Descriptor().Fields()
+	for i := 0; i < fds.Len(); i++ {
+		fd := fds.Get(i)
+		if m.Has(fd) && !f(fd, m.Get(fd)) {
+			return
+		}
+	}
+
+	// Range over populated extension fields.
+	for _, xd := range m[messageTypeKey].(messageType).xds {
+		if m.Has(xd) && !f(xd, m.Get(xd)) {
+			return
+		}
+	}
+}
+func (m reflectMessage) Has(fd protoreflect.FieldDescriptor) bool {
+	_, ok := m[m.stringKey(fd)]
+	return ok
+}
+func (m reflectMessage) Clear(protoreflect.FieldDescriptor) {
+	panic("invalid mutation of read-only message")
+}
+func (m reflectMessage) Get(fd protoreflect.FieldDescriptor) protoreflect.Value {
+	v, ok := m[m.stringKey(fd)]
+	if !ok {
+		switch {
+		case fd.IsList():
+			return protoreflect.ValueOfList(reflectList{})
+		case fd.IsMap():
+			return protoreflect.ValueOfMap(reflectMap{})
+		case fd.Message() != nil:
+			return protoreflect.ValueOfMessage(reflectMessage{
+				messageTypeKey: messageType{md: m.Descriptor()},
+			})
+		default:
+			return fd.Default()
+		}
+	}
+
+	// The transformation may leave Any messages in structured form.
+	// If so, convert them back to a raw-encoded form.
+	if fd.FullName() == "google.protobuf.Any.value" {
+		if m, ok := v.(Message); ok {
+			b, err := proto.MarshalOptions{Deterministic: true}.Marshal(m)
+			if err != nil {
+				panic("BUG: " + err.Error())
+			}
+			return protoreflect.ValueOfBytes(b)
+		}
+	}
+
+	return reflectValueOf(v)
+}
+func (m reflectMessage) Set(protoreflect.FieldDescriptor, protoreflect.Value) {
+	panic("invalid mutation of read-only message")
+}
+func (m reflectMessage) Mutable(fd protoreflect.FieldDescriptor) protoreflect.Value {
+	panic("invalid mutation of read-only message")
+}
+func (m reflectMessage) NewField(protoreflect.FieldDescriptor) protoreflect.Value {
+	panic("not implemented")
+}
+func (m reflectMessage) WhichOneof(od protoreflect.OneofDescriptor) protoreflect.FieldDescriptor {
+	if m.Descriptor().Oneofs().ByName(od.Name()) != od {
+		panic("oneof descriptor does not belong to this message")
+	}
+	fds := od.Fields()
+	for i := 0; i < fds.Len(); i++ {
+		fd := fds.Get(i)
+		if _, ok := m[m.stringKey(fd)]; ok {
+			return fd
+		}
+	}
+	return nil
+}
+func (m reflectMessage) GetUnknown() protoreflect.RawFields {
+	var nums []protoreflect.FieldNumber
+	for k := range m {
+		if len(strings.Trim(k, "0123456789")) == 0 {
+			n, _ := strconv.ParseUint(k, 10, 32)
+			nums = append(nums, protoreflect.FieldNumber(n))
+		}
+	}
+	sort.Slice(nums, func(i, j int) bool { return nums[i] < nums[j] })
+
+	var raw protoreflect.RawFields
+	for _, num := range nums {
+		b, _ := m[strconv.FormatUint(uint64(num), 10)].(protoreflect.RawFields)
+		raw = append(raw, b...)
+	}
+	return raw
+}
+func (m reflectMessage) SetUnknown(protoreflect.RawFields) {
+	panic("invalid mutation of read-only message")
+}
+func (m reflectMessage) IsValid() bool {
+	invalid, _ := m[messageInvalidKey].(bool)
+	return !invalid
+}
+func (m reflectMessage) ProtoMethods() *protoiface.Methods {
+	return nil
+}
+
+type reflectMessageType struct{ protoreflect.MessageDescriptor }
+
+func (t reflectMessageType) New() protoreflect.Message {
+	panic("not implemented")
+}
+func (t reflectMessageType) Zero() protoreflect.Message {
+	panic("not implemented")
+}
+func (t reflectMessageType) Descriptor() protoreflect.MessageDescriptor {
+	return t.MessageDescriptor
+}
+
+type reflectList struct{ v reflect.Value }
+
+func (ls reflectList) Len() int {
+	if !ls.IsValid() {
+		return 0
+	}
+	return ls.v.Len()
+}
+func (ls reflectList) Get(i int) protoreflect.Value {
+	return reflectValueOf(ls.v.Index(i).Interface())
+}
+func (ls reflectList) Set(int, protoreflect.Value) {
+	panic("invalid mutation of read-only list")
+}
+func (ls reflectList) Append(protoreflect.Value) {
+	panic("invalid mutation of read-only list")
+}
+func (ls reflectList) AppendMutable() protoreflect.Value {
+	panic("invalid mutation of read-only list")
+}
+func (ls reflectList) Truncate(int) {
+	panic("invalid mutation of read-only list")
+}
+func (ls reflectList) NewElement() protoreflect.Value {
+	panic("not implemented")
+}
+func (ls reflectList) IsValid() bool {
+	return ls.v.IsValid()
+}
+
+type reflectMap struct{ v reflect.Value }
+
+func (ms reflectMap) Len() int {
+	if !ms.IsValid() {
+		return 0
+	}
+	return ms.v.Len()
+}
+func (ms reflectMap) Range(f func(protoreflect.MapKey, protoreflect.Value) bool) {
+	if !ms.IsValid() {
+		return
+	}
+	ks := ms.v.MapKeys()
+	for _, k := range ks {
+		pk := reflectValueOf(k.Interface()).MapKey()
+		pv := reflectValueOf(ms.v.MapIndex(k).Interface())
+		if !f(pk, pv) {
+			return
+		}
+	}
+}
+func (ms reflectMap) Has(k protoreflect.MapKey) bool {
+	if !ms.IsValid() {
+		return false
+	}
+	return ms.v.MapIndex(reflect.ValueOf(k.Interface())).IsValid()
+}
+func (ms reflectMap) Clear(protoreflect.MapKey) {
+	panic("invalid mutation of read-only list")
+}
+func (ms reflectMap) Get(k protoreflect.MapKey) protoreflect.Value {
+	if !ms.IsValid() {
+		return protoreflect.Value{}
+	}
+	v := ms.v.MapIndex(reflect.ValueOf(k.Interface()))
+	if !v.IsValid() {
+		return protoreflect.Value{}
+	}
+	return reflectValueOf(v.Interface())
+}
+func (ms reflectMap) Set(protoreflect.MapKey, protoreflect.Value) {
+	panic("invalid mutation of read-only list")
+}
+func (ms reflectMap) Mutable(k protoreflect.MapKey) protoreflect.Value {
+	panic("invalid mutation of read-only list")
+}
+func (ms reflectMap) NewValue() protoreflect.Value {
+	panic("not implemented")
+}
+func (ms reflectMap) IsValid() bool {
+	return ms.v.IsValid()
+}

testing/protocmp/reflect_test.go

@@ -0,0 +1,129 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package protocmp
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+
+	"google.golang.org/protobuf/proto"
+
+	testpb "google.golang.org/protobuf/internal/testprotos/test"
+	textpb "google.golang.org/protobuf/internal/testprotos/textpb2"
+	anypb "google.golang.org/protobuf/types/known/anypb"
+	wrapperspb "google.golang.org/protobuf/types/known/wrapperspb"
+)
+
+func TestReflect(t *testing.T) {
+	optMsg := &testpb.TestAllTypes{
+		OptionalInt32:         proto.Int32(-32),
+		OptionalInt64:         proto.Int64(-64),
+		OptionalUint32:        proto.Uint32(32),
+		OptionalUint64:        proto.Uint64(64),
+		OptionalFloat:         proto.Float32(32.32),
+		OptionalDouble:        proto.Float64(64.64),
+		OptionalBool:          proto.Bool(true),
+		OptionalString:        proto.String("string"),
+		OptionalBytes:         []byte("bytes"),
+		OptionalNestedMessage: &testpb.TestAllTypes_NestedMessage{A: proto.Int32(-32)},
+		OptionalNestedEnum:    testpb.TestAllTypes_NEG.Enum(),
+	}
+	repMsg := &testpb.TestAllTypes{
+		RepeatedInt32:         []int32{-32, +32},
+		RepeatedInt64:         []int64{-64, +64},
+		RepeatedUint32:        []uint32{0, 32},
+		RepeatedUint64:        []uint64{0, 64},
+		RepeatedFloat:         []float32{-32.32, +32.32},
+		RepeatedDouble:        []float64{-64.64, +64.64},
+		RepeatedBool:          []bool{false, true},
+		RepeatedString:        []string{"hello", "goodbye"},
+		RepeatedBytes:         [][]byte{[]byte("hello"), []byte("goodbye")},
+		RepeatedNestedMessage: []*testpb.TestAllTypes_NestedMessage{{A: proto.Int32(-32)}, {A: proto.Int32(+32)}},
+		RepeatedNestedEnum:    []testpb.TestAllTypes_NestedEnum{testpb.TestAllTypes_FOO, testpb.TestAllTypes_NEG},
+	}
+	mapMsg := &testpb.TestAllTypes{
+		MapInt32Int32:          map[int32]int32{-1: -32, +1: +32},
+		MapInt64Int64:          map[int64]int64{-1: -32, +1: +64},
+		MapUint32Uint32:        map[uint32]uint32{0: 0, 1: 32},
+		MapUint64Uint64:        map[uint64]uint64{0: 0, 1: 64},
+		MapInt32Float:          map[int32]float32{-1: -32.32, +1: +32.32},
+		MapInt32Double:         map[int32]float64{-1: -64.64, +1: +64.64},
+		MapBoolBool:            map[bool]bool{false: true, true: false},
+		MapStringString:        map[string]string{"k1": "v1", "k2": "v2"},
+		MapStringBytes:         map[string][]byte{"k1": []byte("v1"), "k2": []byte("v2")},
+		MapStringNestedMessage: map[string]*testpb.TestAllTypes_NestedMessage{"k1": {A: proto.Int32(-32)}, "k2": {A: proto.Int32(+32)}},
+		MapStringNestedEnum:    map[string]testpb.TestAllTypes_NestedEnum{"k1": testpb.TestAllTypes_FOO, "k2": testpb.TestAllTypes_NEG},
+	}
+
+	tests := []proto.Message{
+		optMsg,
+		repMsg,
+		mapMsg,
+		&testpb.TestAllTypes{
+			OneofField: &testpb.TestAllTypes_OneofUint32{32},
+		},
+		&testpb.TestAllTypes{
+			OneofField: &testpb.TestAllTypes_OneofUint64{64},
+		},
+		&testpb.TestAllTypes{
+			OneofField: &testpb.TestAllTypes_OneofFloat{32.32},
+		},
+		&testpb.TestAllTypes{
+			OneofField: &testpb.TestAllTypes_OneofDouble{64.64},
+		},
+		&testpb.TestAllTypes{
+			OneofField: &testpb.TestAllTypes_OneofBool{true},
+		},
+		&testpb.TestAllTypes{
+			OneofField: &testpb.TestAllTypes_OneofString{"string"},
+		},
+		&testpb.TestAllTypes{
+			OneofField: &testpb.TestAllTypes_OneofBytes{[]byte("bytes")},
+		},
+		&testpb.TestAllTypes{
+			OneofField: &testpb.TestAllTypes_OneofNestedMessage{&testpb.TestAllTypes_NestedMessage{A: proto.Int32(-32)}},
+		},
+		&testpb.TestAllTypes{
+			OneofField: &testpb.TestAllTypes_OneofEnum{testpb.TestAllTypes_NEG},
+		},
+		&textpb.KnownTypes{
+			OptBool:   &wrapperspb.BoolValue{Value: true},
+			OptInt32:  &wrapperspb.Int32Value{Value: -32},
+			OptInt64:  &wrapperspb.Int64Value{Value: -64},
+			OptUint32: &wrapperspb.UInt32Value{Value: +32},
+			OptUint64: &wrapperspb.UInt64Value{Value: +64},
+			OptFloat:  &wrapperspb.FloatValue{Value: 32.32},
+			OptDouble: &wrapperspb.DoubleValue{Value: 64.64},
+			OptString: &wrapperspb.StringValue{Value: "string"},
+			OptBytes:  &wrapperspb.BytesValue{Value: []byte("bytes")},
+		},
+		&textpb.KnownTypes{
+			OptAny: &anypb.Any{
+				TypeUrl: "google.golang.org/goproto.proto.test.TestAllTypes",
+				Value: func() []byte {
+					b1, _ := proto.MarshalOptions{Deterministic: true}.Marshal(optMsg)
+					b2, _ := proto.MarshalOptions{Deterministic: true}.Marshal(repMsg)
+					b3, _ := proto.MarshalOptions{Deterministic: true}.Marshal(mapMsg)
+					return append(append(append([]byte(nil), b1...), b2...), b3...)
+				}(),
+			},
+		},
+		&textpb.KnownTypes{
+			OptAny: &anypb.Any{
+				TypeUrl: "unknown_type",
+				Value:   []byte("invalid_value"),
+			},
+		},
+	}
+
+	for _, src := range tests {
+		dst := src.ProtoReflect().Type().New().Interface()
+		proto.Merge(dst, transformMessage(src.ProtoReflect()))
+		if diff := cmp.Diff(src, dst, Transform()); diff != "" {
+			t.Errorf("Merge mismatch (-want +got):\n%s", diff)
+		}
+	}
+}

testing/protocmp/util.go

@@ -472,7 +472,12 @@ func isEmptyMessage(v reflect.Value) bool {
 		return false // implies unexported struct field
 	}
 	if m, ok := v.Interface().(Message); ok {
-		return len(m) == 0 || (len(m) == 1 && m[messageTypeKey] != nil)
+		for k := range m {
+			if k != messageTypeKey && k != messageInvalidKey {
+				return false
+			}
+		}
+		return true
 	}
 	return false
 }

testing/protocmp/util_test.go

@@ -33,6 +33,11 @@ func TestEqual(t *testing.T) {
 		y:    (*testpb.TestAllTypes)(nil),
 		opts: cmp.Options{Transform()},
 		want: true,
+	}, {
+		x:    (*testpb.TestAllTypes)(nil),
+		y:    (*testpb.TestAllExtensions)(nil),
+		opts: cmp.Options{Transform()},
+		want: false,
 	}, {
 		x:    (*testpb.TestAllTypes)(nil),
 		y:    new(testpb.TestAllTypes),

testing/protocmp/xform.go

@@ -63,7 +63,10 @@ func (e Enum) String() string {
 	return strconv.Itoa(int(e.num))
 }
 
-const messageTypeKey = "@type"
+const (
+	messageTypeKey    = "@type"
+	messageInvalidKey = "@invalid"
+)
 
 type messageType struct {
 	md  protoreflect.MessageDescriptor
@@ -109,6 +112,21 @@ func (m Message) Descriptor() protoreflect.MessageDescriptor {
 	return mt.md
 }
 
+// ProtoReflect returns a reflective view of m.
+// It only implements the read-only operations of protoreflect.Message.
+// Calling any mutating operations on m panics.
+func (m Message) ProtoReflect() protoreflect.Message {
+	return (reflectMessage)(m)
+}
+
+// ProtoMessage is a marker method from the legacy message interface.
+func (m Message) ProtoMessage() {}
+
+// Reset is the required Reset method from the legacy message interface.
+func (m Message) Reset() {
+	panic("invalid mutation of a read-only message")
+}
+
 // TODO: There is currently no public API for retrieving the FieldDescriptors
 // for extension fields. Rather than adding a specialized API to support that,
 // perhaps Message should just implement protoreflect.ProtoMessage instead.
@@ -117,10 +135,14 @@ func (m Message) Descriptor() protoreflect.MessageDescriptor {
 // It is intended for human debugging and has no guarantees about its
 // exact format or the stability of its output.
 func (m Message) String() string {
-	if m == nil {
+	switch {
+	case m == nil:
 		return "<nil>"
+	case !m.ProtoReflect().IsValid():
+		return "<invalid>"
+	default:
+		return string(appendMessage(nil, m))
 	}
-	return string(appendMessage(nil, m))
 }
 
 type option struct{}
@@ -131,6 +153,10 @@ type option struct{}
 // The google.protobuf.Any message is automatically unmarshaled such that the
 // "value" field is a Message representing the underlying message value
 // assuming it could be resolved and properly unmarshaled.
+//
+// This does not directly transform higher-order composite Go types.
+// For example, []*foopb.Message is not transformed into []Message,
+// but rather the individual message elements of the slice are transformed.
 func Transform(...option) cmp.Option {
 	// NOTE: There are currently no custom options for Transform,
 	// but the use of an unexported type keeps the future open.
@@ -155,14 +181,22 @@ func Transform(...option) cmp.Option {
 		return false
 	}, cmp.Transformer("protocmp.Transform", func(v interface{}) Message {
 		m := protoimpl.X.MessageOf(v)
-		if m == nil || !m.IsValid() {
+		switch {
+		case m == nil:
 			return nil
+		case !m.IsValid():
+			return Message{messageTypeKey: messageType{md: m.Descriptor()}, messageInvalidKey: true}
+		default:
+			return transformMessage(m)
 		}
-		return transformMessage(m)
 	}))
 }
 
 func isMessageType(t reflect.Type) bool {
+	// Avoid tranforming the Message itself.
+	if t == reflect.TypeOf(Message(nil)) || t == reflect.TypeOf((*Message)(nil)) {
+		return false
+	}
 	return t.Implements(messageV1Type) || t.Implements(messageV2Type)
 }