mirror of
https://github.com/protocolbuffers/protobuf-go.git
synced 2025-01-01 11:58:21 +00:00
c51e2e0293
In 2014, when proto3 was being developed, there were a number of early adopters of the new syntax. Before the finalization of proto3 when it was released in open-source in July 2016, a decision was made to strictly validate strings in proto3. However, some of the early adopters were already using invalid UTF-8 with string fields. The google.protobuf.FieldOptions.enforce_utf8 option only exists to support those grandfathered users where they can opt-out of the validation logic. Practical use of that option in open source is impossible even if a user specifies the proto1_legacy build tag since it requires a hacked variant of descriptor.proto that is not externally available. This CL supports enforce_utf8 by modifiyng internal/filedesc to expose the flag if it detects it in the raw descriptor. We add an strs.EnforceUTF8 function as a centralized place to determine whether to perform validation. Validation opt-out is supported only in builds with legacy support. We implement support for validating UTF-8 in all proto3 string fields, even if they are backed by a Go []byte. Change-Id: I9c0628b84909bc7181125f09db730c80d490e485 Reviewed-on: https://go-review.googlesource.com/c/protobuf/+/186002 Reviewed-by: Damien Neil <dneil@google.com>
146 lines
4.1 KiB
Go
146 lines
4.1 KiB
Go
// Copyright 2019 The Go Authors. All rights reserved.
|
|
// Use of this source code is governed by a BSD-style
|
|
// license that can be found in the LICENSE file.
|
|
|
|
package proto_test
|
|
|
|
import (
|
|
"bytes"
|
|
"fmt"
|
|
"testing"
|
|
|
|
"github.com/google/go-cmp/cmp"
|
|
"google.golang.org/protobuf/internal/flags"
|
|
"google.golang.org/protobuf/proto"
|
|
|
|
test3pb "google.golang.org/protobuf/internal/testprotos/test3"
|
|
)
|
|
|
|
func TestEncode(t *testing.T) {
|
|
for _, test := range testProtos {
|
|
for _, want := range test.decodeTo {
|
|
t.Run(fmt.Sprintf("%s (%T)", test.desc, want), func(t *testing.T) {
|
|
opts := proto.MarshalOptions{
|
|
AllowPartial: test.partial,
|
|
}
|
|
wire, err := opts.Marshal(want)
|
|
if err != nil {
|
|
t.Fatalf("Marshal error: %v\nMessage:\n%v", err, marshalText(want))
|
|
}
|
|
|
|
size := proto.Size(want)
|
|
if size != len(wire) {
|
|
t.Errorf("Size and marshal disagree: Size(m)=%v; len(Marshal(m))=%v\nMessage:\n%v", size, len(wire), marshalText(want))
|
|
}
|
|
|
|
got := want.ProtoReflect().New().Interface()
|
|
uopts := proto.UnmarshalOptions{
|
|
AllowPartial: test.partial,
|
|
}
|
|
if err := uopts.Unmarshal(wire, got); err != nil {
|
|
t.Errorf("Unmarshal error: %v\nMessage:\n%v", err, marshalText(want))
|
|
return
|
|
}
|
|
if !proto.Equal(got, want) {
|
|
t.Errorf("Unmarshal returned unexpected result; got:\n%v\nwant:\n%v", marshalText(got), marshalText(want))
|
|
}
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestEncodeDeterministic(t *testing.T) {
|
|
for _, test := range testProtos {
|
|
for _, want := range test.decodeTo {
|
|
t.Run(fmt.Sprintf("%s (%T)", test.desc, want), func(t *testing.T) {
|
|
opts := proto.MarshalOptions{
|
|
Deterministic: true,
|
|
AllowPartial: test.partial,
|
|
}
|
|
wire, err := opts.Marshal(want)
|
|
if err != nil {
|
|
t.Fatalf("Marshal error: %v\nMessage:\n%v", err, marshalText(want))
|
|
}
|
|
wire2, err := opts.Marshal(want)
|
|
if err != nil {
|
|
t.Fatalf("Marshal error: %v\nMessage:\n%v", err, marshalText(want))
|
|
}
|
|
if !bytes.Equal(wire, wire2) {
|
|
t.Fatalf("deterministic marshal returned varying results:\n%v", cmp.Diff(wire, wire2))
|
|
}
|
|
|
|
got := want.ProtoReflect().New().Interface()
|
|
uopts := proto.UnmarshalOptions{
|
|
AllowPartial: test.partial,
|
|
}
|
|
if err := uopts.Unmarshal(wire, got); err != nil {
|
|
t.Errorf("Unmarshal error: %v\nMessage:\n%v", err, marshalText(want))
|
|
return
|
|
}
|
|
if !proto.Equal(got, want) {
|
|
t.Errorf("Unmarshal returned unexpected result; got:\n%v\nwant:\n%v", marshalText(got), marshalText(want))
|
|
}
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestEncodeInvalidUTF8(t *testing.T) {
|
|
for _, test := range invalidUTF8TestProtos {
|
|
for _, want := range test.decodeTo {
|
|
t.Run(fmt.Sprintf("%s (%T)", test.desc, want), func(t *testing.T) {
|
|
_, err := proto.Marshal(want)
|
|
if err == nil {
|
|
t.Errorf("Marshal did not return expected error for invalid UTF8: %v\nMessage:\n%v", err, marshalText(want))
|
|
}
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestEncodeNoEnforceUTF8(t *testing.T) {
|
|
for _, test := range noEnforceUTF8TestProtos {
|
|
for _, want := range test.decodeTo {
|
|
t.Run(fmt.Sprintf("%s (%T)", test.desc, want), func(t *testing.T) {
|
|
_, err := proto.Marshal(want)
|
|
switch {
|
|
case flags.Proto1Legacy && err != nil:
|
|
t.Errorf("Marshal returned unexpected error: %v\nMessage:\n%v", err, marshalText(want))
|
|
case !flags.Proto1Legacy && err == nil:
|
|
t.Errorf("Marshal did not return expected error for invalid UTF8: %v\nMessage:\n%v", err, marshalText(want))
|
|
}
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestEncodeRequiredFieldChecks(t *testing.T) {
|
|
for _, test := range testProtos {
|
|
if !test.partial {
|
|
continue
|
|
}
|
|
for _, m := range test.decodeTo {
|
|
t.Run(fmt.Sprintf("%s (%T)", test.desc, m), func(t *testing.T) {
|
|
_, err := proto.Marshal(m)
|
|
if err == nil {
|
|
t.Fatalf("Marshal succeeded (want error)\nMessage:\n%v", marshalText(m))
|
|
}
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestMarshalAppend(t *testing.T) {
|
|
want := []byte("prefix")
|
|
got := append([]byte(nil), want...)
|
|
got, err := proto.MarshalOptions{}.MarshalAppend(got, &test3pb.TestAllTypes{
|
|
OptionalString: "value",
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !bytes.HasPrefix(got, want) {
|
|
t.Fatalf("MarshalAppend modified prefix: got %v, want prefix %v", got, want)
|
|
}
|
|
}
|