proto: rearrange test messages

Move the test inputs for the wire marshaler and unmarshaler out of
decode_test.go and into a new file. Consolidate some tests for invalid
messages (UTF-8 validation failures, field numbers out of range) into
a single list of invalid messages. Break out the no-enforce-utf8 test
into a separate file, since it is both complicated and conditional on
legacy support.

Change-Id: Ide80fa9d3aec2b6d42a57e6f9265358aa5e661a7
Reviewed-on: https://go-review.googlesource.com/c/protobuf/+/211557
Reviewed-by: Joe Tsai <joetsai@google.com>

proto/bench_test.go

@@ -25,7 +25,7 @@ var (
 
 // BenchmarkEncode benchmarks encoding all the test messages.
 func BenchmarkEncode(b *testing.B) {
-	for _, test := range testProtos {
+	for _, test := range testValidMessages {
 		for _, want := range test.decodeTo {
 			v1 := want.(protoV1.Message)
 			opts := proto.MarshalOptions{AllowPartial: *allowPartial}
@@ -50,7 +50,7 @@ func BenchmarkEncode(b *testing.B) {
 
 // BenchmarkDecode benchmarks decoding all the test messages.
 func BenchmarkDecode(b *testing.B) {
-	for _, test := range testProtos {
+	for _, test := range testValidMessages {
 		for _, want := range test.decodeTo {
 			opts := proto.UnmarshalOptions{AllowPartial: *allowPartial}
 			b.Run(fmt.Sprintf("%s (%T)", test.desc, want), func(b *testing.B) {

proto/encode_test.go

@@ -12,7 +12,6 @@ import (
 
 	"github.com/google/go-cmp/cmp"
 	"google.golang.org/protobuf/internal/encoding/wire"
-	"google.golang.org/protobuf/internal/flags"
 	"google.golang.org/protobuf/proto"
 	pref "google.golang.org/protobuf/reflect/protoreflect"
 
@@ -22,7 +21,7 @@ import (
 )
 
 func TestEncode(t *testing.T) {
-	for _, test := range testProtos {
+	for _, test := range testValidMessages {
 		for _, want := range test.decodeTo {
 			t.Run(fmt.Sprintf("%s (%T)", test.desc, want), func(t *testing.T) {
 				opts := proto.MarshalOptions{
@@ -55,7 +54,7 @@ func TestEncode(t *testing.T) {
 }
 
 func TestEncodeDeterministic(t *testing.T) {
-	for _, test := range testProtos {
+	for _, test := range testValidMessages {
 		for _, want := range test.decodeTo {
 			t.Run(fmt.Sprintf("%s (%T)", test.desc, want), func(t *testing.T) {
 				opts := proto.MarshalOptions{
@@ -90,37 +89,8 @@ func TestEncodeDeterministic(t *testing.T) {
 	}
 }
 
-func TestEncodeInvalidUTF8(t *testing.T) {
-	for _, test := range invalidUTF8TestProtos {
-		for _, want := range test.decodeTo {
-			t.Run(fmt.Sprintf("%s (%T)", test.desc, want), func(t *testing.T) {
-				_, err := proto.Marshal(want)
-				if err == nil {
-					t.Errorf("Marshal did not return expected error for invalid UTF8: %v\nMessage:\n%v", err, marshalText(want))
-				}
-			})
-		}
-	}
-}
-
-func TestEncodeNoEnforceUTF8(t *testing.T) {
-	for _, test := range noEnforceUTF8TestProtos {
-		for _, want := range test.decodeTo {
-			t.Run(fmt.Sprintf("%s (%T)", test.desc, want), func(t *testing.T) {
-				_, err := proto.Marshal(want)
-				switch {
-				case flags.ProtoLegacy && err != nil:
-					t.Errorf("Marshal returned unexpected error: %v\nMessage:\n%v", err, marshalText(want))
-				case !flags.ProtoLegacy && err == nil:
-					t.Errorf("Marshal did not return expected error for invalid UTF8: %v\nMessage:\n%v", err, marshalText(want))
-				}
-			})
-		}
-	}
-}
-
 func TestEncodeRequiredFieldChecks(t *testing.T) {
-	for _, test := range testProtos {
+	for _, test := range testValidMessages {
 		if !test.partial {
 			continue
 		}
@@ -149,6 +119,26 @@ func TestEncodeAppend(t *testing.T) {
 	}
 }
 
+func TestEncodeInvalidMessages(t *testing.T) {
+	for _, test := range testInvalidMessages {
+		for _, m := range test.decodeTo {
+			if !m.ProtoReflect().IsValid() {
+				continue
+			}
+			t.Run(fmt.Sprintf("%s (%T)", test.desc, m), func(t *testing.T) {
+				t.Logf("%v %v", m, m.ProtoReflect().IsValid())
+				opts := proto.MarshalOptions{
+					AllowPartial: test.partial,
+				}
+				got, err := opts.Marshal(m)
+				if err == nil {
+					t.Fatalf("Marshal unexpectedly succeeded\noutput bytes: [%x]\nMessage:\n%v", got, marshalText(m))
+				}
+			})
+		}
+	}
+}
+
 func TestEncodeOneofNilWrapper(t *testing.T) {
 	m := &testpb.TestAllTypes{OneofField: (*testpb.TestAllTypes_OneofUint32)(nil)}
 	b, err := proto.Marshal(m)

proto/messageset_test.go

@@ -16,7 +16,7 @@ import (
 
 func init() {
 	if flags.ProtoLegacy {
-		testProtos = append(testProtos, messageSetTestProtos...)
+		testValidMessages = append(testValidMessages, messageSetTestProtos...)
 	}
 }
 

proto/noenforceutf8_test.go

@@ -0,0 +1,146 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style.
+// license that can be found in the LICENSE file.
+
+package proto_test
+
+import (
+	"reflect"
+
+	"google.golang.org/protobuf/encoding/prototext"
+	"google.golang.org/protobuf/internal/encoding/pack"
+	"google.golang.org/protobuf/internal/filedesc"
+	"google.golang.org/protobuf/internal/flags"
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/reflect/protodesc"
+	"google.golang.org/protobuf/reflect/protoreflect"
+	"google.golang.org/protobuf/runtime/protoimpl"
+
+	"google.golang.org/protobuf/types/descriptorpb"
+)
+
+func init() {
+	if flags.ProtoLegacy {
+		testValidMessages = append(testValidMessages, noEnforceUTF8TestProtos...)
+	} else {
+		testInvalidMessages = append(testInvalidMessages, noEnforceUTF8TestProtos...)
+	}
+}
+
+var noEnforceUTF8TestProtos = []testProto{
+	{
+		desc: "invalid UTF-8 in optional string field",
+		decodeTo: []proto.Message{&TestNoEnforceUTF8{
+			OptionalString: string("abc\xff"),
+		}},
+		wire: pack.Message{
+			pack.Tag{1, pack.BytesType}, pack.String("abc\xff"),
+		}.Marshal(),
+	},
+	{
+		desc: "invalid UTF-8 in optional string field of Go bytes",
+		decodeTo: []proto.Message{&TestNoEnforceUTF8{
+			OptionalBytes: []byte("abc\xff"),
+		}},
+		wire: pack.Message{
+			pack.Tag{2, pack.BytesType}, pack.String("abc\xff"),
+		}.Marshal(),
+	},
+	{
+		desc: "invalid UTF-8 in repeated string field",
+		decodeTo: []proto.Message{&TestNoEnforceUTF8{
+			RepeatedString: []string{string("foo"), string("abc\xff")},
+		}},
+		wire: pack.Message{
+			pack.Tag{3, pack.BytesType}, pack.String("foo"),
+			pack.Tag{3, pack.BytesType}, pack.String("abc\xff"),
+		}.Marshal(),
+	},
+	{
+		desc: "invalid UTF-8 in repeated string field of Go bytes",
+		decodeTo: []proto.Message{&TestNoEnforceUTF8{
+			RepeatedBytes: [][]byte{[]byte("foo"), []byte("abc\xff")},
+		}},
+		wire: pack.Message{
+			pack.Tag{4, pack.BytesType}, pack.String("foo"),
+			pack.Tag{4, pack.BytesType}, pack.String("abc\xff"),
+		}.Marshal(),
+	},
+	{
+		desc: "invalid UTF-8 in oneof string field",
+		decodeTo: []proto.Message{
+			&TestNoEnforceUTF8{OneofField: &TestNoEnforceUTF8_OneofString{string("abc\xff")}},
+		},
+		wire: pack.Message{pack.Tag{5, pack.BytesType}, pack.String("abc\xff")}.Marshal(),
+	},
+	{
+		desc: "invalid UTF-8 in oneof string field of Go bytes",
+		decodeTo: []proto.Message{
+			&TestNoEnforceUTF8{OneofField: &TestNoEnforceUTF8_OneofBytes{[]byte("abc\xff")}},
+		},
+		wire: pack.Message{pack.Tag{6, pack.BytesType}, pack.String("abc\xff")}.Marshal(),
+	},
+}
+
+type TestNoEnforceUTF8 struct {
+	OptionalString string       `protobuf:"bytes,1,opt,name=optional_string"`
+	OptionalBytes  []byte       `protobuf:"bytes,2,opt,name=optional_bytes"`
+	RepeatedString []string     `protobuf:"bytes,3,rep,name=repeated_string"`
+	RepeatedBytes  [][]byte     `protobuf:"bytes,4,rep,name=repeated_bytes"`
+	OneofField     isOneofField `protobuf_oneof:"oneof_field"`
+}
+
+type isOneofField interface{ isOneofField() }
+
+type TestNoEnforceUTF8_OneofString struct {
+	OneofString string `protobuf:"bytes,5,opt,name=oneof_string,oneof"`
+}
+type TestNoEnforceUTF8_OneofBytes struct {
+	OneofBytes []byte `protobuf:"bytes,6,opt,name=oneof_bytes,oneof"`
+}
+
+func (*TestNoEnforceUTF8_OneofString) isOneofField() {}
+func (*TestNoEnforceUTF8_OneofBytes) isOneofField()  {}
+
+func (m *TestNoEnforceUTF8) ProtoReflect() protoreflect.Message {
+	return messageInfo_TestNoEnforceUTF8.MessageOf(m)
+}
+
+var messageInfo_TestNoEnforceUTF8 = protoimpl.MessageInfo{
+	GoReflectType: reflect.TypeOf((*TestNoEnforceUTF8)(nil)),
+	Desc: func() protoreflect.MessageDescriptor {
+		pb := new(descriptorpb.FileDescriptorProto)
+		if err := prototext.Unmarshal([]byte(`
+				syntax:  "proto3"
+				name:    "test.proto"
+				message_type: [{
+					name: "TestNoEnforceUTF8"
+					field: [
+						{name:"optional_string" number:1 label:LABEL_OPTIONAL type:TYPE_STRING},
+						{name:"optional_bytes"  number:2 label:LABEL_OPTIONAL type:TYPE_STRING},
+						{name:"repeated_string" number:3 label:LABEL_REPEATED type:TYPE_STRING},
+						{name:"repeated_bytes"  number:4 label:LABEL_REPEATED type:TYPE_STRING},
+						{name:"oneof_string"    number:5 label:LABEL_OPTIONAL type:TYPE_STRING, oneof_index:0},
+						{name:"oneof_bytes"     number:6 label:LABEL_OPTIONAL type:TYPE_STRING, oneof_index:0}
+					]
+					oneof_decl: [{name:"oneof_field"}]
+				}]
+			`), pb); err != nil {
+			panic(err)
+		}
+		fd, err := protodesc.NewFile(pb, nil)
+		if err != nil {
+			panic(err)
+		}
+		md := fd.Messages().Get(0)
+		for i := 0; i < md.Fields().Len(); i++ {
+			md.Fields().Get(i).(*filedesc.Field).L1.HasEnforceUTF8 = true
+			md.Fields().Get(i).(*filedesc.Field).L1.EnforceUTF8 = false
+		}
+		return md
+	}(),
+	OneofWrappers: []interface{}{
+		(*TestNoEnforceUTF8_OneofString)(nil),
+		(*TestNoEnforceUTF8_OneofBytes)(nil),
+	},
+}