From f92988f90090c8fe8d58dc381c7919e215b57e65 Mon Sep 17 00:00:00 2001 From: Joe Tsai Date: Sun, 1 Mar 2020 03:01:46 -0800 Subject: [PATCH] all: upgrade protobuf to v3.11.4 This introduces some conformance test failures, which occur not because our implementation changed behavior, but because new cases were added. Future work will be to investigate these failuress. Change-Id: Ifb17465883c417acd46865744572f8cd0c858383 Reviewed-on: https://go-review.googlesource.com/c/protobuf/+/222857 Reviewed-by: Damien Neil --- integration_test.go | 43 +------------------ internal/benchmarks/download_benchdata.bash | 10 ++++- internal/conformance/failing_tests.txt | 39 +++++++++++++++++ .../testprotos/fieldmaskpb/field_mask.pb.go | 2 +- 4 files changed, 49 insertions(+), 45 deletions(-) diff --git a/integration_test.go b/integration_test.go index 422ad33c..e4444a4a 100644 --- a/integration_test.go +++ b/integration_test.go @@ -32,7 +32,7 @@ var ( regenerate = flag.Bool("regenerate", false, "regenerate files") buildRelease = flag.Bool("buildRelease", false, "build release binaries") - protobufVersion = "3.9.1" + protobufVersion = "3.11.4" golangVersions = []string{"1.9.7", "1.10.8", "1.11.13", "1.12.17", "1.13.8", "1.14"} golangLatest = golangVersions[len(golangVersions)-1] @@ -218,27 +218,6 @@ func mustInitDeps(t *testing.T) { command{Dir: protobufPath}.mustRun(t, "make") command{Dir: filepath.Join(protobufPath, "conformance")}.mustRun(t, "make") } - // The benchmark directory isn't present in the release download, - // so fetch needed files directly. - for _, path := range benchmarkProtos { - src := fmt.Sprintf("https://raw.githubusercontent.com/protocolbuffers/protobuf/v%v/%v", protobufVersion, path) - dst := filepath.Join(protobufPath, path) - if _, err := os.Stat(dst); err != nil { - downloadFile(check, dst, src) - } - } - benchdataPath := filepath.Join(testDir, "benchdata") - for _, path := range []string{ - "benchmarks/datasets/google_message1/proto2/dataset.google_message1_proto2.pb", - "benchmarks/datasets/google_message1/proto3/dataset.google_message1_proto3.pb", - "benchmarks/datasets/google_message2/dataset.google_message2.pb", - } { - src := fmt.Sprintf("https://raw.githubusercontent.com/protocolbuffers/protobuf/v%v/%v", protobufVersion, path) - dst := filepath.Join(benchdataPath, filepath.Base(path)) - if _, err := os.Stat(dst); err != nil { - downloadFile(check, dst, src) - } - } check(os.Setenv("PROTOBUF_ROOT", protobufPath)) // for generate-protos registerBinary("conform-test-runner", filepath.Join(protobufPath, "conformance", "conformance-test-runner")) registerBinary("protoc", filepath.Join(protobufPath, "src", "protoc")) @@ -452,23 +431,3 @@ func mustRunCommand(t *testing.T, args ...string) string { t.Helper() return command{}.mustRun(t, args...) } - -var benchmarkProtos = []string{ - "benchmarks/benchmarks.proto", - "benchmarks/datasets/google_message1/proto2/benchmark_message1_proto2.proto", - "benchmarks/datasets/google_message1/proto3/benchmark_message1_proto3.proto", - "benchmarks/datasets/google_message2/benchmark_message2.proto", - "benchmarks/datasets/google_message3/benchmark_message3.proto", - "benchmarks/datasets/google_message3/benchmark_message3_1.proto", - "benchmarks/datasets/google_message3/benchmark_message3_2.proto", - "benchmarks/datasets/google_message3/benchmark_message3_3.proto", - "benchmarks/datasets/google_message3/benchmark_message3_4.proto", - "benchmarks/datasets/google_message3/benchmark_message3_5.proto", - "benchmarks/datasets/google_message3/benchmark_message3_6.proto", - "benchmarks/datasets/google_message3/benchmark_message3_7.proto", - "benchmarks/datasets/google_message3/benchmark_message3_8.proto", - "benchmarks/datasets/google_message4/benchmark_message4.proto", - "benchmarks/datasets/google_message4/benchmark_message4_1.proto", - "benchmarks/datasets/google_message4/benchmark_message4_2.proto", - "benchmarks/datasets/google_message4/benchmark_message4_3.proto", -} diff --git a/internal/benchmarks/download_benchdata.bash b/internal/benchmarks/download_benchdata.bash index f33faab4..30399e7f 100755 --- a/internal/benchmarks/download_benchdata.bash +++ b/internal/benchmarks/download_benchdata.bash @@ -3,9 +3,15 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. -# Download large benchmark datasets. - cd "$(git rev-parse --show-toplevel)" mkdir -p .cache/benchdata cd .cache/benchdata + +# Download small benchmark datasets. +PROTOBUF_VERSION=v3.11.4 +curl -s -O https://raw.githubusercontent.com/protocolbuffers/protobuf/$PROTOBUF_VERSION/benchmarks/datasets/google_message1/proto2/dataset.google_message1_proto2.pb +curl -s -O https://raw.githubusercontent.com/protocolbuffers/protobuf/$PROTOBUF_VERSION/benchmarks/datasets/google_message1/proto3/dataset.google_message1_proto3.pb +curl -s -O https://raw.githubusercontent.com/protocolbuffers/protobuf/$PROTOBUF_VERSION/benchmarks/datasets/google_message2/dataset.google_message2.pb + +# Download large benchmark datasets. curl -s https://storage.googleapis.com/protobuf_opensource_benchmark_data/datasets.tar.gz | tar zx diff --git a/internal/conformance/failing_tests.txt b/internal/conformance/failing_tests.txt index 92dc3743..e18a9e2d 100644 --- a/internal/conformance/failing_tests.txt +++ b/internal/conformance/failing_tests.txt @@ -1 +1,40 @@ +Recommended.Proto2.ProtobufInput.ValidDataRepeated.BOOL.PackedInput.PackedOutput.ProtobufOutput +Recommended.Proto2.ProtobufInput.ValidDataRepeated.BOOL.UnpackedInput.PackedOutput.ProtobufOutput +Recommended.Proto2.ProtobufInput.ValidDataRepeated.DOUBLE.UnpackedInput.PackedOutput.ProtobufOutput +Recommended.Proto2.ProtobufInput.ValidDataRepeated.ENUM.PackedInput.PackedOutput.ProtobufOutput +Recommended.Proto2.ProtobufInput.ValidDataRepeated.ENUM.UnpackedInput.PackedOutput.ProtobufOutput +Recommended.Proto2.ProtobufInput.ValidDataRepeated.FIXED32.UnpackedInput.PackedOutput.ProtobufOutput +Recommended.Proto2.ProtobufInput.ValidDataRepeated.FIXED64.UnpackedInput.PackedOutput.ProtobufOutput +Recommended.Proto2.ProtobufInput.ValidDataRepeated.FLOAT.UnpackedInput.PackedOutput.ProtobufOutput +Recommended.Proto2.ProtobufInput.ValidDataRepeated.INT32.PackedInput.PackedOutput.ProtobufOutput +Recommended.Proto2.ProtobufInput.ValidDataRepeated.INT32.UnpackedInput.PackedOutput.ProtobufOutput +Recommended.Proto2.ProtobufInput.ValidDataRepeated.INT64.UnpackedInput.PackedOutput.ProtobufOutput +Recommended.Proto2.ProtobufInput.ValidDataRepeated.SFIXED32.UnpackedInput.PackedOutput.ProtobufOutput +Recommended.Proto2.ProtobufInput.ValidDataRepeated.SFIXED64.UnpackedInput.PackedOutput.ProtobufOutput +Recommended.Proto2.ProtobufInput.ValidDataRepeated.SINT32.PackedInput.PackedOutput.ProtobufOutput +Recommended.Proto2.ProtobufInput.ValidDataRepeated.SINT32.UnpackedInput.PackedOutput.ProtobufOutput +Recommended.Proto2.ProtobufInput.ValidDataRepeated.SINT64.UnpackedInput.PackedOutput.ProtobufOutput +Recommended.Proto2.ProtobufInput.ValidDataRepeated.UINT32.PackedInput.PackedOutput.ProtobufOutput +Recommended.Proto2.ProtobufInput.ValidDataRepeated.UINT32.UnpackedInput.PackedOutput.ProtobufOutput +Recommended.Proto2.ProtobufInput.ValidDataRepeated.UINT64.UnpackedInput.PackedOutput.ProtobufOutput +Recommended.Proto3.ProtobufInput.ValidDataRepeated.BOOL.PackedInput.UnpackedOutput.ProtobufOutput +Recommended.Proto3.ProtobufInput.ValidDataRepeated.BOOL.UnpackedInput.UnpackedOutput.ProtobufOutput +Recommended.Proto3.ProtobufInput.ValidDataRepeated.DOUBLE.PackedInput.UnpackedOutput.ProtobufOutput +Recommended.Proto3.ProtobufInput.ValidDataRepeated.ENUM.PackedInput.UnpackedOutput.ProtobufOutput +Recommended.Proto3.ProtobufInput.ValidDataRepeated.ENUM.UnpackedInput.UnpackedOutput.ProtobufOutput +Recommended.Proto3.ProtobufInput.ValidDataRepeated.FIXED32.PackedInput.UnpackedOutput.ProtobufOutput +Recommended.Proto3.ProtobufInput.ValidDataRepeated.FIXED64.PackedInput.UnpackedOutput.ProtobufOutput +Recommended.Proto3.ProtobufInput.ValidDataRepeated.FLOAT.PackedInput.UnpackedOutput.ProtobufOutput +Recommended.Proto3.ProtobufInput.ValidDataRepeated.INT32.PackedInput.UnpackedOutput.ProtobufOutput +Recommended.Proto3.ProtobufInput.ValidDataRepeated.INT32.UnpackedInput.UnpackedOutput.ProtobufOutput +Recommended.Proto3.ProtobufInput.ValidDataRepeated.INT64.PackedInput.UnpackedOutput.ProtobufOutput +Recommended.Proto3.ProtobufInput.ValidDataRepeated.SFIXED32.PackedInput.UnpackedOutput.ProtobufOutput +Recommended.Proto3.ProtobufInput.ValidDataRepeated.SFIXED64.PackedInput.UnpackedOutput.ProtobufOutput +Recommended.Proto3.ProtobufInput.ValidDataRepeated.SINT32.PackedInput.UnpackedOutput.ProtobufOutput +Recommended.Proto3.ProtobufInput.ValidDataRepeated.SINT32.UnpackedInput.UnpackedOutput.ProtobufOutput +Recommended.Proto3.ProtobufInput.ValidDataRepeated.SINT64.PackedInput.UnpackedOutput.ProtobufOutput +Recommended.Proto3.ProtobufInput.ValidDataRepeated.UINT32.PackedInput.UnpackedOutput.ProtobufOutput +Recommended.Proto3.ProtobufInput.ValidDataRepeated.UINT32.UnpackedInput.UnpackedOutput.ProtobufOutput +Recommended.Proto3.ProtobufInput.ValidDataRepeated.UINT64.PackedInput.UnpackedOutput.ProtobufOutput +Required.Proto3.ProtobufInput.ValidDataOneof.MESSAGE.Merge.JsonOutput Recommended.Proto3.JsonInput.FieldMaskInvalidCharacter diff --git a/internal/testprotos/fieldmaskpb/field_mask.pb.go b/internal/testprotos/fieldmaskpb/field_mask.pb.go index 5d18615d..b9bc1306 100644 --- a/internal/testprotos/fieldmaskpb/field_mask.pb.go +++ b/internal/testprotos/fieldmaskpb/field_mask.pb.go @@ -238,7 +238,7 @@ import ( // // The implementation of any API method which has a FieldMask type field in the // request should verify the included field paths, and return an -// `INVALID_ARGUMENT` error if any path is duplicated or unmappable. +// `INVALID_ARGUMENT` error if any path is unmappable. type FieldMask struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache