feat: add lifecycle config and reservation affinity support to v1 (via synth) (#10)

Author: busunkim96

packages/google-cloud-dataproc/google/cloud/dataproc_v1/gapic/cluster_controller_client.py

@@ -416,10 +416,12 @@ def update_cluster(
                 message :class:`~google.cloud.dataproc_v1.types.FieldMask`
             graceful_decommission_timeout (Union[dict, ~google.cloud.dataproc_v1.types.Duration]): Optional. Timeout for graceful YARN decomissioning. Graceful
                 decommissioning allows removing nodes from the cluster without
-                interrupting jobs in progress. Timeout specifies how long to wait for jobs
-                in progress to finish before forcefully removing nodes (and potentially
-                interrupting jobs). Default timeout is 0 (for forceful decommission), and
-                the maximum allowed timeout is 1 day.
+                interrupting jobs in progress. Timeout specifies how long to wait for
+                jobs in progress to finish before forcefully removing nodes (and
+                potentially interrupting jobs). Default timeout is 0 (for forceful
+                decommission), and the maximum allowed timeout is 1 day. (see JSON
+                representation of
+                `Duration <https://developers.google.com/protocol-buffers/docs/proto3#json>`__).
 
                 Only supported on Dataproc image versions 1.2 and higher.
 

packages/google-cloud-dataproc/google/cloud/dataproc_v1/gapic/enums.py

@@ -207,6 +207,25 @@ class Level(enum.IntEnum):
         OFF = 8
 
 
+class ReservationAffinity(object):
+    class Type(enum.IntEnum):
+        """
+        Indicates whether to consume capacity from an reservation or not.
+
+        Attributes:
+          TYPE_UNSPECIFIED (int)
+          NO_RESERVATION (int): Do not consume from any allocated capacity.
+          ANY_RESERVATION (int): Consume any reservation available.
+          SPECIFIC_RESERVATION (int): Must consume from a specific reservation. Must specify key value fields
+          for specifying the reservations.
+        """
+
+        TYPE_UNSPECIFIED = 0
+        NO_RESERVATION = 1
+        ANY_RESERVATION = 2
+        SPECIFIC_RESERVATION = 3
+
+
 class WorkflowMetadata(object):
     class State(enum.IntEnum):
         """

packages/google-cloud-dataproc/google/cloud/dataproc_v1/proto/autoscaling_policies.proto

@@ -1,4 +1,4 @@
-// Copyright 2019 Google LLC.
+// Copyright 2020 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,7 +11,6 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-//
 
 syntax = "proto3";
 
@@ -28,7 +27,6 @@ option go_package = "google.golang.org/genproto/googleapis/cloud/dataproc/v1;dat
 option java_multiple_files = true;
 option java_outer_classname = "AutoscalingPoliciesProto";
 option java_package = "com.google.cloud.dataproc.v1";
-
 option (google.api.resource_definition) = {
   type: "dataproc.googleapis.com/Region"
   pattern: "projects/{project}/regions/{region}"

packages/google-cloud-dataproc/google/cloud/dataproc_v1/proto/clusters.proto

@@ -1,4 +1,4 @@
-// Copyright 2019 Google LLC.
+// Copyright 2020 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,7 +11,6 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-//
 
 syntax = "proto3";
 
@@ -20,7 +19,6 @@ package google.cloud.dataproc.v1;
 import "google/api/annotations.proto";
 import "google/api/client.proto";
 import "google/api/field_behavior.proto";
-import "google/cloud/dataproc/v1/operations.proto";
 import "google/cloud/dataproc/v1/shared.proto";
 import "google/longrunning/operations.proto";
 import "google/protobuf/duration.proto";
@@ -40,7 +38,7 @@ service ClusterController {
 
   // Creates a cluster in a project. The returned
   // [Operation.metadata][google.longrunning.Operation.metadata] will be
-  // [ClusterOperationMetadata](/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata).
+  // [ClusterOperationMetadata](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata).
   rpc CreateCluster(CreateClusterRequest) returns (google.longrunning.Operation) {
     option (google.api.http) = {
       post: "/v1/projects/{project_id}/regions/{region}/clusters"
@@ -55,22 +53,22 @@ service ClusterController {
 
   // Updates a cluster in a project. The returned
   // [Operation.metadata][google.longrunning.Operation.metadata] will be
-  // [ClusterOperationMetadata](/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata).
+  // [ClusterOperationMetadata](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata).
   rpc UpdateCluster(UpdateClusterRequest) returns (google.longrunning.Operation) {
     option (google.api.http) = {
       patch: "/v1/projects/{project_id}/regions/{region}/clusters/{cluster_name}"
       body: "cluster"
     };
+    option (google.api.method_signature) = "project_id,region,cluster_name,cluster,update_mask";
     option (google.longrunning.operation_info) = {
       response_type: "Cluster"
       metadata_type: "google.cloud.dataproc.v1.ClusterOperationMetadata"
     };
-    option (google.api.method_signature) = "project_id,region,cluster_name,cluster,update_mask";
   }
 
   // Deletes a cluster in a project. The returned
   // [Operation.metadata][google.longrunning.Operation.metadata] will be
-  // [ClusterOperationMetadata](/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata).
+  // [ClusterOperationMetadata](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata).
   rpc DeleteCluster(DeleteClusterRequest) returns (google.longrunning.Operation) {
     option (google.api.http) = {
       delete: "/v1/projects/{project_id}/regions/{region}/clusters/{cluster_name}"
@@ -101,11 +99,11 @@ service ClusterController {
 
   // Gets cluster diagnostic information. The returned
   // [Operation.metadata][google.longrunning.Operation.metadata] will be
-  // [ClusterOperationMetadata](/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata).
+  // [ClusterOperationMetadata](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata).
   // After the operation completes,
   // [Operation.response][google.longrunning.Operation.response]
   // contains
-  // [DiagnoseClusterResults](/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#diagnoseclusterresults).
+  // [DiagnoseClusterResults](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#diagnoseclusterresults).
   rpc DiagnoseCluster(DiagnoseClusterRequest) returns (google.longrunning.Operation) {
     option (google.api.http) = {
       post: "/v1/projects/{project_id}/regions/{region}/clusters/{cluster_name}:diagnose"
@@ -215,6 +213,9 @@ message ClusterConfig {
 
   // Optional. Security settings for the cluster.
   SecurityConfig security_config = 16 [(google.api.field_behavior) = OPTIONAL];
+
+  // Optional. Lifecycle setting for the cluster.
+  LifecycleConfig lifecycle_config = 17 [(google.api.field_behavior) = OPTIONAL];
 }
 
 // Autoscaling Policy config associated with the cluster.
@@ -322,9 +323,12 @@ message GceClusterConfig {
   // [Project and instance
   // metadata](https://cloud.google.com/compute/docs/storing-retrieving-metadata#project_and_instance_metadata)).
   map<string, string> metadata = 5;
+
+  // Optional. Reservation Affinity for consuming Zonal reservation.
+  ReservationAffinity reservation_affinity = 11 [(google.api.field_behavior) = OPTIONAL];
 }
 
-// Optional. The config settings for Compute Engine resources in
+// The config settings for Compute Engine resources in
 // an instance group, such as a master or worker group.
 message InstanceGroupConfig {
   // Optional. The number of VM instances in the instance group.
@@ -438,7 +442,10 @@ message NodeInitializationAction {
   string executable_file = 1 [(google.api.field_behavior) = REQUIRED];
 
   // Optional. Amount of time executable has to complete. Default is
-  // 10 minutes. Cluster creation fails with an explanatory error message (the
+  // 10 minutes (see JSON representation of
+  // [Duration](https://developers.google.com/protocol-buffers/docs/proto3#json)).
+  //
+  // Cluster creation fails with an explanatory error message (the
   // name of the executable that caused the error and the exceeded timeout
   // period) if the executable is not completed at end of the timeout period.
   google.protobuf.Duration execution_timeout = 2 [(google.api.field_behavior) = OPTIONAL];
@@ -495,7 +502,8 @@ message ClusterStatus {
     (google.api.field_behavior) = OPTIONAL
   ];
 
-  // Output only. Time when this state was entered.
+  // Output only. Time when this state was entered (see JSON representation of
+  // [Timestamp](https://developers.google.com/protocol-buffers/docs/proto3#json)).
   google.protobuf.Timestamp state_start_time = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
 
   // Output only. Additional state information that includes
@@ -613,6 +621,36 @@ message SoftwareConfig {
   repeated Component optional_components = 3 [(google.api.field_behavior) = OPTIONAL];
 }
 
+// Specifies the cluster auto-delete schedule configuration.
+message LifecycleConfig {
+  // Optional. The duration to keep the cluster alive while idling (when no jobs
+  // are running). Passing this threshold will cause the cluster to be
+  // deleted. Minimum value is 10 minutes; maximum value is 14 days (see JSON
+  // representation of
+  // [Duration](https://developers.google.com/protocol-buffers/docs/proto3#json).
+  google.protobuf.Duration idle_delete_ttl = 1 [(google.api.field_behavior) = OPTIONAL];
+
+  // Either the exact time the cluster should be deleted at or
+  // the cluster maximum age.
+  oneof ttl {
+    // Optional. The time when cluster will be auto-deleted (see JSON representation of
+    // [Timestamp](https://developers.google.com/protocol-buffers/docs/proto3#json)).
+    google.protobuf.Timestamp auto_delete_time = 2 [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. The lifetime duration of cluster. The cluster will be
+    // auto-deleted at the end of this period. Minimum value is 10 minutes;
+    // maximum value is 14 days (see JSON representation of
+    // [Duration](https://developers.google.com/protocol-buffers/docs/proto3#json)).
+    google.protobuf.Duration auto_delete_ttl = 3 [(google.api.field_behavior) = OPTIONAL];
+  }
+
+  // Output only. The time when cluster became idle (most recent job finished)
+  // and became eligible for deletion due to idleness (see JSON representation
+  // of
+  // [Timestamp](https://developers.google.com/protocol-buffers/docs/proto3#json)).
+  google.protobuf.Timestamp idle_start_time = 4 [(google.api.field_behavior) = OUTPUT_ONLY];
+}
+
 // Contains cluster daemon metrics, such as HDFS and YARN stats.
 //
 // **Beta Feature**: This report is available for testing purposes only. It may
@@ -671,7 +709,8 @@ message UpdateClusterRequest {
   // interrupting jobs in progress. Timeout specifies how long to wait for jobs
   // in progress to finish before forcefully removing nodes (and potentially
   // interrupting jobs). Default timeout is 0 (for forceful decommission), and
-  // the maximum allowed timeout is 1 day.
+  // the maximum allowed timeout is 1 day. (see JSON representation of
+  // [Duration](https://developers.google.com/protocol-buffers/docs/proto3#json)).
   //
   // Only supported on Dataproc image versions 1.2 and higher.
   google.protobuf.Duration graceful_decommission_timeout = 6 [(google.api.field_behavior) = OPTIONAL];
@@ -854,3 +893,30 @@ message DiagnoseClusterResults {
   // diagnostics.
   string output_uri = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
 }
+
+// Reservation Affinity for consuming Zonal reservation.
+message ReservationAffinity {
+  // Indicates whether to consume capacity from an reservation or not.
+  enum Type {
+    TYPE_UNSPECIFIED = 0;
+
+    // Do not consume from any allocated capacity.
+    NO_RESERVATION = 1;
+
+    // Consume any reservation available.
+    ANY_RESERVATION = 2;
+
+    // Must consume from a specific reservation. Must specify key value fields
+    // for specifying the reservations.
+    SPECIFIC_RESERVATION = 3;
+  }
+
+  // Optional. Type of reservation to consume
+  Type consume_reservation_type = 1 [(google.api.field_behavior) = OPTIONAL];
+
+  // Optional. Corresponds to the label key of reservation resource.
+  string key = 2 [(google.api.field_behavior) = OPTIONAL];
+
+  // Optional. Corresponds to the label values of reservation resource.
+  repeated string values = 3 [(google.api.field_behavior) = OPTIONAL];
+}