go proto: include (opt-in) vtprotobuf generation (#31172)

See envoyproxy/go-control-plane#824 for more information

This PR adds the vtprotobuf protoc plugin for Go. This works on top of the existing protoc-gen-go, to add optimized marshal functions that callers can opt in to using. This is not like gogo, which was a very invasive change -- everything is layered and opt-in. See issue for benchmarks, etc.

Additionally, to avoid possible binary size increase, the entire new code is protected under a go build tag. Users will need to opt-in at build time (-tags=vtprotobuf). By default, there is no impact for users at all.

Risk Level: Low - only additional opt-in code
Testing: Manually tested in Istio codebase

Signed-off-by: John Howard <[email protected]>

Author: howardjohn

api/bazel/BUILD

@@ -26,6 +26,15 @@ go_proto_compiler(
     visibility = ["//visibility:public"],
 )
 
+go_proto_compiler(
+    name = "vtprotobuf_plugin_go",
+    options = ["features=marshal_strict+size"],
+    plugin = "@com_github_planetscale_vtprotobuf//cmd/protoc-gen-go-vtproto",
+    suffix = "_vtproto.pb.go",
+    valid_archive = False,
+    visibility = ["//visibility:public"],
+)
+
 json_data(
     name = "repository_locations",
     data = load_repository_locations_spec(REPOSITORY_LOCATIONS_SPEC),

api/bazel/api_build_system.bzl

@@ -146,9 +146,9 @@ def api_proto_package(
         has_services = has_services,
     )
 
-    compilers = ["@io_bazel_rules_go//proto:go_proto", "@envoy_api//bazel:pgv_plugin_go"]
+    compilers = ["@io_bazel_rules_go//proto:go_proto", "@envoy_api//bazel:pgv_plugin_go", "@envoy_api//bazel:vtprotobuf_plugin_go"]
     if has_services:
-        compilers = ["@io_bazel_rules_go//proto:go_grpc", "@envoy_api//bazel:pgv_plugin_go"]
+        compilers = ["@io_bazel_rules_go//proto:go_grpc", "@envoy_api//bazel:pgv_plugin_go", "@envoy_api//bazel:vtprotobuf_plugin_go"]
 
     deps = (
         [_go_proto_mapping(dep) for dep in deps] +
@@ -162,6 +162,13 @@ def api_proto_package(
             "@io_bazel_rules_go//proto/wkt:struct_go_proto",
             "@io_bazel_rules_go//proto/wkt:timestamp_go_proto",
             "@io_bazel_rules_go//proto/wkt:wrappers_go_proto",
+            "@com_github_planetscale_vtprotobuf//types/known/anypb",
+            "@com_github_planetscale_vtprotobuf//types/known/durationpb",
+            "@com_github_planetscale_vtprotobuf//types/known/emptypb",
+            "@com_github_planetscale_vtprotobuf//types/known/fieldmaskpb",
+            "@com_github_planetscale_vtprotobuf//types/known/structpb",
+            "@com_github_planetscale_vtprotobuf//types/known/timestamppb",
+            "@com_github_planetscale_vtprotobuf//types/known/wrapperspb",
         ]
     )
     go_proto_library(

bazel/dependency_imports.bzl

@@ -146,6 +146,13 @@ def envoy_dependency_imports(go_version = GO_VERSION, jq_version = JQ_VERSION, y
         # use_category = ["api"],
         # source = "https://github.com/bufbuild/protoc-gen-validate/blob/v0.6.1/dependencies.bzl#L23-L28"
     )
+    go_repository(
+        name = "com_github_planetscale_vtprotobuf",
+        importpath = "github.com/planetscale/vtprotobuf",
+        sum = "h1:nve54OLsoKDQhb8ZnnHHUyvAK3vjBiwTEJeC3UsqzJ8=",
+        version = "v0.5.1-0.20231205081218-d930d8ac92f8",
+        build_external = "external",
+    )
 
     protoc_gen_jsonschema_go_dependencies()
 

ci/do_ci.sh

@@ -308,6 +308,11 @@ case $CI_TARGET in
                 if [[ "$GO_FILE" = *.validate.go ]]; then
                     sed -i '1s;^;//go:build !disable_pgv\n;' "$OUTPUT_DIR/$(basename "$GO_FILE")"
                 fi
+                # TODO(https://github.com/planetscale/vtprotobuf/pull/122) do this directly in the generator.
+                # Make vtprotobuf opt-in as it has some impact on binary sizes
+                if [[ "$GO_FILE" = *.vtproto.pb.go ]]; then
+                    sed -i '1s;^;//go:build vtprotobuf\n;' "$OUTPUT_DIR/$(basename "$GO_FILE")"
+                fi
             done <<< "$(find "$INPUT_DIR" -name "*.go")"
         done
         ;;