integration_test: add conformance tests

Change-Id: I3c21e5069c34a6417c9177ae5f1bdcb801418c8a
Reviewed-on: https://go-review.googlesource.com/c/protobuf/+/173665
Reviewed-by: Joe Tsai <thebrokentoaster@gmail.com>

integration_test.go

@@ -36,8 +36,9 @@ var (
 	purgeTimeout = 30 * 24 * time.Hour // 1 month
 
 	// Variables initialized by mustInitDeps.
-	goPath     string
-	modulePath string
+	goPath       string
+	modulePath   string
+	protobufPath string
 )
 
 func Test(t *testing.T) {
@@ -74,6 +75,13 @@ func Test(t *testing.T) {
 		})
 	}
 
+	t.Run("ConformanceTests", func(t *testing.T) {
+		driverPath := filepath.Join("internal", "cmd", "conformance")
+		driver := filepath.Join(driverPath, "conformance.sh")
+		failureList := filepath.Join(driverPath, "failure_list_go.txt")
+		runner := filepath.Join(protobufPath, "conformance", "conformance-test-runner")
+		mustRunCommand(t, ".", runner, "--failure_list", failureList, "--enforce_recommended", driver)
+	})
 	t.Run("GeneratedGoFiles", func(t *testing.T) {
 		diff := mustRunCommand(t, ".", "go", "run", "./internal/cmd/generate-types")
 		if strings.TrimSpace(diff) != "" {
@@ -184,21 +192,22 @@ func mustInitDeps(t *testing.T) {
 	// We avoid downloading the pre-compiled binaries since they do not contain
 	// the conformance test runner.
 	workingDir = filepath.Join(testDir, "protobuf-"+protobufVersion)
-	if _, err := os.Stat(workingDir); err != nil {
-		fmt.Printf("download %v\n", filepath.Base(workingDir))
+	protobufPath = workingDir
+	if _, err := os.Stat(protobufPath); err != nil {
+		fmt.Printf("download %v\n", filepath.Base(protobufPath))
 		url := fmt.Sprintf("https://github.com/google/protobuf/releases/download/v%v/protobuf-all-%v.tar.gz", protobufVersion, protobufVersion)
-		downloadArchive(check, workingDir, url, "protobuf-"+protobufVersion)
+		downloadArchive(check, protobufPath, url, "protobuf-"+protobufVersion)
 
-		fmt.Printf("build %v\n", filepath.Base(workingDir))
-		mustRunCommand(t, workingDir, "./autogen.sh")
-		mustRunCommand(t, workingDir, "./configure")
-		mustRunCommand(t, workingDir, "make")
-		mustRunCommand(t, filepath.Join(workingDir, "conformance"), "make")
+		fmt.Printf("build %v\n", filepath.Base(protobufPath))
+		mustRunCommand(t, protobufPath, "./autogen.sh")
+		mustRunCommand(t, protobufPath, "./configure")
+		mustRunCommand(t, protobufPath, "make")
+		mustRunCommand(t, filepath.Join(protobufPath, "conformance"), "make")
 	}
-	patchProtos(check, workingDir)
-	check(os.Setenv("PROTOBUF_ROOT", workingDir)) // for generate-protos
-	registerBinary("conform-test-runner", filepath.Join(workingDir, "conformance", "conformance-test-runner"))
-	registerBinary("protoc", filepath.Join(workingDir, "src", "protoc"))
+	patchProtos(check, protobufPath)
+	check(os.Setenv("PROTOBUF_ROOT", protobufPath)) // for generate-protos
+	registerBinary("conform-test-runner", filepath.Join(protobufPath, "conformance", "conformance-test-runner"))
+	registerBinary("protoc", filepath.Join(protobufPath, "src", "protoc"))
 	workingDir = ""
 
 	// Download each Go toolchain version.
@@ -281,19 +290,21 @@ func patchProtos(check func(error), repoRoot string) {
 	javaPackageRx := regexp.MustCompile(`^option\s+java_package\s*=\s*".*"\s*;\s*$`)
 	goPackageRx := regexp.MustCompile(`^option\s+go_package\s*=\s*".*"\s*;\s*$`)
 	files := map[string]string{
-		"src/google/protobuf/any.proto":             "github.com/golang/protobuf/v2/types/known;known_proto",
-		"src/google/protobuf/api.proto":             "github.com/golang/protobuf/v2/types/known;known_proto",
-		"src/google/protobuf/duration.proto":        "github.com/golang/protobuf/v2/types/known;known_proto",
-		"src/google/protobuf/empty.proto":           "github.com/golang/protobuf/v2/types/known;known_proto",
-		"src/google/protobuf/field_mask.proto":      "github.com/golang/protobuf/v2/types/known;known_proto",
-		"src/google/protobuf/source_context.proto":  "github.com/golang/protobuf/v2/types/known;known_proto",
-		"src/google/protobuf/struct.proto":          "github.com/golang/protobuf/v2/types/known;known_proto",
-		"src/google/protobuf/timestamp.proto":       "github.com/golang/protobuf/v2/types/known;known_proto",
-		"src/google/protobuf/type.proto":            "github.com/golang/protobuf/v2/types/known;known_proto",
-		"src/google/protobuf/wrappers.proto":        "github.com/golang/protobuf/v2/types/known;known_proto",
-		"src/google/protobuf/descriptor.proto":      "github.com/golang/protobuf/v2/types/descriptor;descriptor_proto",
-		"src/google/protobuf/compiler/plugin.proto": "github.com/golang/protobuf/v2/types/plugin;plugin_proto",
-		"conformance/conformance.proto":             "github.com/golang/protobuf/v2/internal/testprotos/conformance;conformance_proto",
+		"src/google/protobuf/any.proto":                  "github.com/golang/protobuf/v2/types/known;known_proto",
+		"src/google/protobuf/api.proto":                  "github.com/golang/protobuf/v2/types/known;known_proto",
+		"src/google/protobuf/duration.proto":             "github.com/golang/protobuf/v2/types/known;known_proto",
+		"src/google/protobuf/empty.proto":                "github.com/golang/protobuf/v2/types/known;known_proto",
+		"src/google/protobuf/field_mask.proto":           "github.com/golang/protobuf/v2/types/known;known_proto",
+		"src/google/protobuf/source_context.proto":       "github.com/golang/protobuf/v2/types/known;known_proto",
+		"src/google/protobuf/struct.proto":               "github.com/golang/protobuf/v2/types/known;known_proto",
+		"src/google/protobuf/timestamp.proto":            "github.com/golang/protobuf/v2/types/known;known_proto",
+		"src/google/protobuf/type.proto":                 "github.com/golang/protobuf/v2/types/known;known_proto",
+		"src/google/protobuf/wrappers.proto":             "github.com/golang/protobuf/v2/types/known;known_proto",
+		"src/google/protobuf/descriptor.proto":           "github.com/golang/protobuf/v2/types/descriptor;descriptor_proto",
+		"src/google/protobuf/compiler/plugin.proto":      "github.com/golang/protobuf/v2/types/plugin;plugin_proto",
+		"conformance/conformance.proto":                  "github.com/golang/protobuf/v2/internal/testprotos/conformance;conformance_proto",
+		"src/google/protobuf/test_messages_proto2.proto": "github.com/golang/protobuf/v2/internal/testprotos/conformance;conformance_proto",
+		"src/google/protobuf/test_messages_proto3.proto": "github.com/golang/protobuf/v2/internal/testprotos/conformance;conformance_proto",
 	}
 	for pbpath, gopath := range files {
 		b, err := ioutil.ReadFile(filepath.Join(repoRoot, pbpath))

internal/cmd/conformance/conformance.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+# Copyright 2019 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+cd $(dirname $0)
+exec go run main.go $@

internal/cmd/conformance/failure_list_go.txt

@@ -0,0 +1,7 @@
+Required.Proto3.JsonInput.IgnoreUnknownJsonFalse.ProtobufOutput
+Required.Proto3.JsonInput.IgnoreUnknownJsonNull.ProtobufOutput
+Required.Proto3.JsonInput.IgnoreUnknownJsonNumber.ProtobufOutput
+Required.Proto3.JsonInput.IgnoreUnknownJsonObject.ProtobufOutput
+Required.Proto3.JsonInput.IgnoreUnknownJsonString.ProtobufOutput
+Required.Proto3.JsonInput.IgnoreUnknownJsonTrue.ProtobufOutput
+Recommended.Proto3.JsonInput.FieldMaskInvalidCharacter

internal/cmd/conformance/main.go

@@ -0,0 +1,124 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This binary implements the conformance test subprocess protocol as documented
+// in conformance.proto.
+package main
+
+import (
+	"encoding/binary"
+	"io"
+	"log"
+	"os"
+
+	"github.com/golang/protobuf/v2/encoding/jsonpb"
+	"github.com/golang/protobuf/v2/proto"
+
+	pb "github.com/golang/protobuf/v2/internal/testprotos/conformance"
+)
+
+func main() {
+	var sizeBuf [4]byte
+	inbuf := make([]byte, 0, 4096)
+	for {
+		_, err := io.ReadFull(os.Stdin, sizeBuf[:])
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			log.Fatalf("conformance: read request: %v", err)
+		}
+		size := binary.LittleEndian.Uint32(sizeBuf[:])
+		if int(size) > cap(inbuf) {
+			inbuf = make([]byte, size)
+		}
+		inbuf = inbuf[:size]
+		if _, err := io.ReadFull(os.Stdin, inbuf); err != nil {
+			log.Fatalf("conformance: read request: %v", err)
+		}
+
+		req := &pb.ConformanceRequest{}
+		if err := proto.Unmarshal(inbuf, req); err != nil {
+			log.Fatalf("conformance: parse request: %v", err)
+		}
+		res := handle(req)
+
+		out, err := proto.Marshal(res)
+		if err != nil {
+			log.Fatalf("conformance: marshal response: %v", err)
+		}
+		binary.LittleEndian.PutUint32(sizeBuf[:], uint32(len(out)))
+		if _, err := os.Stdout.Write(sizeBuf[:]); err != nil {
+			log.Fatalf("conformance: write response: %v", err)
+		}
+		if _, err := os.Stdout.Write(out); err != nil {
+			log.Fatalf("conformance: write response: %v", err)
+		}
+	}
+}
+
+func handle(req *pb.ConformanceRequest) *pb.ConformanceResponse {
+	var err error
+	var msg proto.Message = &pb.TestAllTypesProto2{}
+	if req.GetMessageType() == "protobuf_test_messages.proto3.TestAllTypesProto3" {
+		msg = &pb.TestAllTypesProto3{}
+	}
+
+	switch p := req.Payload.(type) {
+	case *pb.ConformanceRequest_ProtobufPayload:
+		err = proto.Unmarshal(p.ProtobufPayload, msg)
+	case *pb.ConformanceRequest_JsonPayload:
+		err = jsonpb.Unmarshal(msg, []byte(p.JsonPayload))
+	default:
+		return &pb.ConformanceResponse{
+			Result: &pb.ConformanceResponse_RuntimeError{
+				RuntimeError: "unknown request payload type",
+			},
+		}
+	}
+	if err != nil {
+		return &pb.ConformanceResponse{
+			Result: &pb.ConformanceResponse_ParseError{
+				ParseError: err.Error(),
+			},
+		}
+	}
+
+	switch req.RequestedOutputFormat {
+	case pb.WireFormat_PROTOBUF:
+		p, err := proto.Marshal(msg)
+		if err != nil {
+			return &pb.ConformanceResponse{
+				Result: &pb.ConformanceResponse_SerializeError{
+					SerializeError: err.Error(),
+				},
+			}
+		}
+		return &pb.ConformanceResponse{
+			Result: &pb.ConformanceResponse_ProtobufPayload{
+				ProtobufPayload: p,
+			},
+		}
+	case pb.WireFormat_JSON:
+		p, err := jsonpb.Marshal(msg)
+		if err != nil {
+			return &pb.ConformanceResponse{
+				Result: &pb.ConformanceResponse_SerializeError{
+					SerializeError: err.Error(),
+				},
+			}
+		}
+		return &pb.ConformanceResponse{
+			Result: &pb.ConformanceResponse_JsonPayload{
+				JsonPayload: string(p),
+			},
+		}
+	default:
+		return &pb.ConformanceResponse{
+			Result: &pb.ConformanceResponse_RuntimeError{
+				RuntimeError: "unknown output format",
+			},
+		}
+	}
+}

internal/cmd/generate-protos/main.go

@@ -185,6 +185,8 @@ func generateRemoteProtos() {
 		{"src", "google/protobuf/field_mask.proto"},
 		{"src", "google/protobuf/source_context.proto"},
 		{"src", "google/protobuf/struct.proto"},
+		{"src", "google/protobuf/test_messages_proto2.proto"},
+		{"src", "google/protobuf/test_messages_proto3.proto"},
 		{"src", "google/protobuf/timestamp.proto"},
 		{"src", "google/protobuf/type.proto"},
 		{"src", "google/protobuf/wrappers.proto"},