feat: add parsing of autopopulated fields from serviceyaml (#2312)

* feat: add parsing of autopopulated fields from serviceyaml

* fix lint

* fix lint and update goldens

* refactor

* add todo

* fix lint

* update comment

* add unit tests

* update hashcode()

* update comments

* remove empty check

* remove unit test

* fix lint

* add todo

Author: alicejli

gapic-generator-java/src/main/java/com/google/api/generator/gapic/model/Field.java

@@ -14,6 +14,7 @@
 
 package com.google.api.generator.gapic.model;
 
+import com.google.api.FieldInfo.Format;
 import com.google.api.generator.engine.ast.TypeNode;
 import com.google.auto.value.AutoValue;
 import java.util.Objects;
@@ -32,6 +33,15 @@ public abstract class Field {
 
   public abstract TypeNode type();
 
+  // If the field is annotated with google.api.field_behavior = REQUIRED, then this is true. This is
+  // currently only used to check if a field should be auto-populated. If it is true, then the field
+  // should
+  // *not* be autopopulated.
+  public abstract boolean isRequired();
+
+  @Nullable
+  public abstract Format fieldInfoFormat();
+
   public abstract boolean isMessage();
 
   public abstract boolean isEnum();
@@ -72,6 +82,8 @@ public boolean equals(Object o) {
     return name().equals(other.name())
         && originalName().equals(other.originalName())
         && type().equals(other.type())
+        && isRequired() == other.isRequired()
+        && fieldInfoFormat() == other.fieldInfoFormat()
         && isMessage() == other.isMessage()
         && isEnum() == other.isEnum()
         && isRepeated() == other.isRepeated()
@@ -89,6 +101,8 @@ public int hashCode() {
         + 19 * type().hashCode()
         + (isMessage() ? 1 : 0) * 23
         + (isEnum() ? 1 : 0) * 29
+        + (isRequired() ? 1 : 0) * 31
+        + (fieldInfoFormat() == null ? 0 : fieldInfoFormat().hashCode())
         + (isRepeated() ? 1 : 0) * 31
         + (isMap() ? 1 : 0) * 37
         + (isContainedInOneof() ? 1 : 0) * 41
@@ -101,6 +115,7 @@ public int hashCode() {
 
   public static Builder builder() {
     return new AutoValue_Field.Builder()
+        .setIsRequired(false)
         .setIsMessage(false)
         .setIsEnum(false)
         .setIsRepeated(false)
@@ -117,6 +132,10 @@ public abstract static class Builder {
 
     public abstract Builder setType(TypeNode type);
 
+    public abstract Builder setIsRequired(boolean isRequired);
+
+    public abstract Builder setFieldInfoFormat(Format fieldInfoFormat);
+
     public abstract Builder setIsMessage(boolean isMessage);
 
     public abstract Builder setIsEnum(boolean isEnum);

gapic-generator-java/src/main/java/com/google/api/generator/gapic/model/Method.java

@@ -17,6 +17,7 @@
 import com.google.api.generator.engine.ast.TypeNode;
 import com.google.auto.value.AutoValue;
 import com.google.common.collect.ImmutableList;
+import java.util.ArrayList;
 import java.util.List;
 import javax.annotation.Nullable;
 
@@ -69,6 +70,16 @@ public boolean isPaged() {
   // [["content", "error"], ["content", "error", "info"]].
   public abstract ImmutableList<List<MethodArgument>> methodSignatures();
 
+  public abstract List<String> autoPopulatedFields();
+
+  /**
+   * If a service's service_config.yaml file contains method_settings.auto_populated_fields for this
+   * method, and the method is a Unary-type, then this is true
+   */
+  public boolean hasAutoPopulatedFields() {
+    return !autoPopulatedFields().isEmpty() && stream() == Stream.NONE;
+  }
+
   public abstract boolean operationPollingMethod();
 
   public boolean hasLro() {
@@ -123,6 +134,7 @@ public boolean isSupportedByTransport(Transport transport) {
   public static Builder builder() {
     return new AutoValue_Method.Builder()
         .setStream(Stream.NONE)
+        .setAutoPopulatedFields(new ArrayList<>())
         .setMethodSignatures(ImmutableList.of())
         .setIsBatching(false)
         .setIsDeprecated(false)
@@ -170,6 +182,8 @@ public abstract static class Builder {
 
     public abstract Builder setOperationPollingMethod(boolean operationPollingMethod);
 
+    public abstract Builder setAutoPopulatedFields(List<String> autoPopulatedFields);
+
     public abstract Builder setRoutingHeaderRule(RoutingHeaderRule routingHeaderRule);
 
     public abstract Method build();

gapic-generator-java/src/main/java/com/google/api/generator/gapic/protoparser/Parser.java

@@ -16,7 +16,12 @@
 
 import com.google.api.ClientProto;
 import com.google.api.DocumentationRule;
+import com.google.api.FieldBehavior;
+import com.google.api.FieldBehaviorProto;
+import com.google.api.FieldInfo.Format;
+import com.google.api.FieldInfoProto;
 import com.google.api.HttpRule;
+import com.google.api.MethodSettings;
 import com.google.api.ResourceDescriptor;
 import com.google.api.ResourceProto;
 import com.google.api.generator.engine.ast.TypeNode;
@@ -47,6 +52,7 @@
 import com.google.common.collect.BiMap;
 import com.google.common.collect.HashBiMap;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Maps;
 import com.google.longrunning.OperationInfo;
@@ -493,6 +499,7 @@ public static List<Service> parseService(
                           messageTypes,
                           resourceNames,
                           serviceConfigOpt,
+                          serviceYamlProtoOpt,
                           outputArgResourceNames,
                           transport))
                   .build();
@@ -683,9 +690,15 @@ static List<Method> parseMethods(
       Map<String, Message> messageTypes,
       Map<String, ResourceName> resourceNames,
       Optional<GapicServiceConfig> serviceConfigOpt,
+      Optional<com.google.api.Service> serviceYamlProtoOpt,
       Set<ResourceName> outputArgResourceNames,
       Transport transport) {
     List<Method> methods = new ArrayList<>();
+
+    // Parse the serviceYaml for autopopulated methods and fields once and put into a map
+    Map<String, List<String>> autoPopulatedMethodsWithFields =
+        parseAutoPopulatedMethodsAndFields(serviceYamlProtoOpt);
+
     for (MethodDescriptor protoMethod : serviceDescriptor.getMethods()) {
       // Parse the method.
       TypeNode inputType = TypeParser.parseType(protoMethod.getInputType());
@@ -699,6 +712,12 @@ static List<Method> parseMethods(
         }
       }
 
+      // Associate the autopopulated fields with the correct method
+      List<String> autoPopulatedFields = new ArrayList<>();
+      if (autoPopulatedMethodsWithFields.containsKey(protoMethod.getFullName())) {
+        autoPopulatedFields = autoPopulatedMethodsWithFields.get(protoMethod.getFullName());
+      }
+
       boolean isDeprecated = false;
       if (protoMethod.getOptions().hasDeprecated()) {
         isDeprecated = protoMethod.getOptions().getDeprecated();
@@ -743,6 +762,7 @@ static List<Method> parseMethods(
                       resourceNames,
                       outputArgResourceNames))
               .setHttpBindings(httpBindings)
+              .setAutoPopulatedFields(autoPopulatedFields)
               .setRoutingHeaderRule(routingHeaderRule)
               .setIsBatching(isBatching)
               .setPageSizeFieldName(parsePageSizeFieldName(protoMethod, messageTypes, transport))
@@ -970,6 +990,8 @@ private static Field parseField(
     FieldOptions fieldOptions = fieldDescriptor.getOptions();
     MessageOptions messageOptions = messageDescriptor.getOptions();
     ResourceReference resourceReference = null;
+    boolean isRequired = false;
+    Format fieldInfoFormat = null;
     if (fieldOptions.hasExtension(ResourceProto.resourceReference)) {
       com.google.api.ResourceReference protoResourceReference =
           fieldOptions.getExtension(ResourceProto.resourceReference);
@@ -1000,6 +1022,16 @@ private static Field parseField(
       }
     }
 
+    if (fieldOptions.hasExtension(FieldInfoProto.fieldInfo)) {
+      fieldInfoFormat = fieldOptions.getExtension(FieldInfoProto.fieldInfo).getFormat();
+    }
+    if (fieldOptions.getExtensionCount(FieldBehaviorProto.fieldBehavior) > 0) {
+      if (fieldOptions
+          .getExtension(FieldBehaviorProto.fieldBehavior)
+          .contains(FieldBehavior.REQUIRED)) ;
+      isRequired = true;
+    }
+
     Field.Builder fieldBuilder = Field.builder();
     if (fieldDescriptor.getFile().toProto().hasSourceCodeInfo()) {
       SourceCodeInfoLocation protoFieldLocation =
@@ -1030,6 +1062,8 @@ private static Field parseField(
             fieldDescriptor.getContainingOneof() != null
                 && fieldDescriptor.getContainingOneof().isSynthetic())
         .setIsRepeated(fieldDescriptor.isRepeated())
+        .setIsRequired(isRequired)
+        .setFieldInfoFormat(fieldInfoFormat)
         .setIsMap(fieldDescriptor.isMapField())
         .setResourceReference(resourceReference)
         .build();
@@ -1124,4 +1158,25 @@ static String parseNestedProtoTypeName(String fullyQualifiedName) {
             .collect(Collectors.toList());
     return String.join(".", nestedTypeComponents);
   }
+
+  /**
+   * Converts a serviceYaml file to a map of methods and autopopulated fields. Note: this does NOT
+   * currently support wildcards in MethodSettings.selectors.
+   */
+  @VisibleForTesting
+  static Map<String, List<String>> parseAutoPopulatedMethodsAndFields(
+      Optional<com.google.api.Service> serviceYamlProtoOpt) {
+    if (!hasMethodSettings(serviceYamlProtoOpt)) {
+      return ImmutableMap.<String, List<String>>builder().build();
+    }
+    return serviceYamlProtoOpt.get().getPublishing().getMethodSettingsList().stream()
+        .collect(
+            Collectors.toMap(
+                MethodSettings::getSelector, MethodSettings::getAutoPopulatedFieldsList));
+  }
+
+  @VisibleForTesting
+  static boolean hasMethodSettings(Optional<com.google.api.Service> serviceYamlProtoOpt) {
+    return serviceYamlProtoOpt.isPresent() && serviceYamlProtoOpt.get().hasPublishing();
+  }
 }

gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/defaultvalue/DefaultValueComposerTest.java

@@ -568,6 +568,7 @@ public void createSimpleMessage_containsMessagesEnumsAndResourceName() {
         "EchoRequest.newBuilder().setName("
             + "FoobarName.ofProjectFoobarName(\"[PROJECT]\", \"[FOOBAR]\").toString())"
             + ".setParent(FoobarName.ofProjectFoobarName(\"[PROJECT]\", \"[FOOBAR]\").toString())"
+            + ".setRequestId(\"requestId693933066\")"
             + ".setSeverity(Severity.forNumber(0))"
             + ".setFoobar(Foobar.newBuilder().build()).build()",
         writerVisitor.write());

gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/grpc/goldens/EchoClient.golden

@@ -516,6 +516,7 @@ public class EchoClient implements BackgroundResource {
    *       EchoRequest.newBuilder()
    *           .setName(FoobarName.ofProjectFoobarName("[PROJECT]", "[FOOBAR]").toString())
    *           .setParent(FoobarName.ofProjectFoobarName("[PROJECT]", "[FOOBAR]").toString())
+   *           .setRequestId("requestId693933066")
    *           .setSeverity(Severity.forNumber(0))
    *           .setFoobar(Foobar.newBuilder().build())
    *           .build();
@@ -545,6 +546,7 @@ public class EchoClient implements BackgroundResource {
    *       EchoRequest.newBuilder()
    *           .setName(FoobarName.ofProjectFoobarName("[PROJECT]", "[FOOBAR]").toString())
    *           .setParent(FoobarName.ofProjectFoobarName("[PROJECT]", "[FOOBAR]").toString())
+   *           .setRequestId("requestId693933066")
    *           .setSeverity(Severity.forNumber(0))
    *           .setFoobar(Foobar.newBuilder().build())
    *           .build();
@@ -570,7 +572,11 @@ public class EchoClient implements BackgroundResource {
    * // https://cloud.google.com/java/docs/setup#configure_endpoints_for_the_client_library
    * try (EchoClient echoClient = EchoClient.create()) {
    *   ExpandRequest request =
-   *       ExpandRequest.newBuilder().setContent("content951530617").setInfo("info3237038").build();
+   *       ExpandRequest.newBuilder()
+   *           .setContent("content951530617")
+   *           .setInfo("info3237038")
+   *           .setRequestId("requestId693933066")
+   *           .build();
    *   ServerStream<EchoResponse> stream = echoClient.expandCallable().call(request);
    *   for (EchoResponse response : stream) {
    *     // Do something when a response is received.
@@ -616,6 +622,7 @@ public class EchoClient implements BackgroundResource {
    *       EchoRequest.newBuilder()
    *           .setName(FoobarName.ofProjectFoobarName("[PROJECT]", "[FOOBAR]").toString())
    *           .setParent(FoobarName.ofProjectFoobarName("[PROJECT]", "[FOOBAR]").toString())
+   *           .setRequestId("requestId693933066")
    *           .setSeverity(Severity.forNumber(0))
    *           .setFoobar(Foobar.newBuilder().build())
    *           .build();
@@ -643,6 +650,7 @@ public class EchoClient implements BackgroundResource {
    *       EchoRequest.newBuilder()
    *           .setName(FoobarName.ofProjectFoobarName("[PROJECT]", "[FOOBAR]").toString())
    *           .setParent(FoobarName.ofProjectFoobarName("[PROJECT]", "[FOOBAR]").toString())
+   *           .setRequestId("requestId693933066")
    *           .setSeverity(Severity.forNumber(0))
    *           .setFoobar(Foobar.newBuilder().build())
    *           .build();
@@ -673,6 +681,7 @@ public class EchoClient implements BackgroundResource {
    *       EchoRequest.newBuilder()
    *           .setName(FoobarName.ofProjectFoobarName("[PROJECT]", "[FOOBAR]").toString())
    *           .setParent(FoobarName.ofProjectFoobarName("[PROJECT]", "[FOOBAR]").toString())
+   *           .setRequestId("requestId693933066")
    *           .setSeverity(Severity.forNumber(0))
    *           .setFoobar(Foobar.newBuilder().build())
    *           .build();
@@ -1081,6 +1090,7 @@ public class EchoClient implements BackgroundResource {
    *       EchoRequest.newBuilder()
    *           .setName(FoobarName.ofProjectFoobarName("[PROJECT]", "[FOOBAR]").toString())
    *           .setParent(FoobarName.ofProjectFoobarName("[PROJECT]", "[FOOBAR]").toString())
+   *           .setRequestId("requestId693933066")
    *           .setSeverity(Severity.forNumber(0))
    *           .setFoobar(Foobar.newBuilder().build())
    *           .build();
@@ -1110,6 +1120,7 @@ public class EchoClient implements BackgroundResource {
    *       EchoRequest.newBuilder()
    *           .setName(FoobarName.ofProjectFoobarName("[PROJECT]", "[FOOBAR]").toString())
    *           .setParent(FoobarName.ofProjectFoobarName("[PROJECT]", "[FOOBAR]").toString())
+   *           .setRequestId("requestId693933066")
    *           .setSeverity(Severity.forNumber(0))
    *           .setFoobar(Foobar.newBuilder().build())
    *           .build();