Skip to content

Commit 974d2b4

Browse files
yoshi-automationyoshi-automation
authored
feat: add long running operation client (#355)
1 parent 176a68b commit 974d2b4

24 files changed

+3598
-207
lines changed

‎packages/google-cloud-dataproc/protos/google/cloud/dataproc/v1/autoscaling_policies.proto

+17-34
Original file line numberDiff line numberDiff line change
@@ -36,12 +36,10 @@ option (google.api.resource_definition) = {
3636
// Dataproc API.
3737
service AutoscalingPolicyService {
3838
option (google.api.default_host) = "dataproc.googleapis.com";
39-
option (google.api.oauth_scopes) =
40-
"https://www.googleapis.com/auth/cloud-platform";
39+
option (google.api.oauth_scopes) = "https://www.googleapis.com/auth/cloud-platform";
4140

4241
// Creates new autoscaling policy.
43-
rpc CreateAutoscalingPolicy(CreateAutoscalingPolicyRequest)
44-
returns (AutoscalingPolicy) {
42+
rpc CreateAutoscalingPolicy(CreateAutoscalingPolicyRequest) returns (AutoscalingPolicy) {
4543
option (google.api.http) = {
4644
post: "/v1/{parent=projects/*/locations/*}/autoscalingPolicies"
4745
body: "policy"
@@ -57,8 +55,7 @@ service AutoscalingPolicyService {
5755
//
5856
// Disabled check for update_mask, because all updates will be full
5957
// replacements.
60-
rpc UpdateAutoscalingPolicy(UpdateAutoscalingPolicyRequest)
61-
returns (AutoscalingPolicy) {
58+
rpc UpdateAutoscalingPolicy(UpdateAutoscalingPolicyRequest) returns (AutoscalingPolicy) {
6259
option (google.api.http) = {
6360
put: "/v1/{policy.name=projects/*/locations/*/autoscalingPolicies/*}"
6461
body: "policy"
@@ -71,8 +68,7 @@ service AutoscalingPolicyService {
7168
}
7269

7370
// Retrieves autoscaling policy.
74-
rpc GetAutoscalingPolicy(GetAutoscalingPolicyRequest)
75-
returns (AutoscalingPolicy) {
71+
rpc GetAutoscalingPolicy(GetAutoscalingPolicyRequest) returns (AutoscalingPolicy) {
7672
option (google.api.http) = {
7773
get: "/v1/{name=projects/*/locations/*/autoscalingPolicies/*}"
7874
additional_bindings {
@@ -83,8 +79,7 @@ service AutoscalingPolicyService {
8379
}
8480

8581
// Lists autoscaling policies in the project.
86-
rpc ListAutoscalingPolicies(ListAutoscalingPoliciesRequest)
87-
returns (ListAutoscalingPoliciesResponse) {
82+
rpc ListAutoscalingPolicies(ListAutoscalingPoliciesRequest) returns (ListAutoscalingPoliciesResponse) {
8883
option (google.api.http) = {
8984
get: "/v1/{parent=projects/*/locations/*}/autoscalingPolicies"
9085
additional_bindings {
@@ -96,8 +91,7 @@ service AutoscalingPolicyService {
9691

9792
// Deletes an autoscaling policy. It is an error to delete an autoscaling
9893
// policy that is in use by one or more clusters.
99-
rpc DeleteAutoscalingPolicy(DeleteAutoscalingPolicyRequest)
100-
returns (google.protobuf.Empty) {
94+
rpc DeleteAutoscalingPolicy(DeleteAutoscalingPolicyRequest) returns (google.protobuf.Empty) {
10195
option (google.api.http) = {
10296
delete: "/v1/{name=projects/*/locations/*/autoscalingPolicies/*}"
10397
additional_bindings {
@@ -138,31 +132,26 @@ message AutoscalingPolicy {
138132

139133
// Autoscaling algorithm for policy.
140134
oneof algorithm {
141-
BasicAutoscalingAlgorithm basic_algorithm = 3
142-
[(google.api.field_behavior) = REQUIRED];
135+
BasicAutoscalingAlgorithm basic_algorithm = 3 [(google.api.field_behavior) = REQUIRED];
143136
}
144137

145138
// Required. Describes how the autoscaler will operate for primary workers.
146-
InstanceGroupAutoscalingPolicyConfig worker_config = 4
147-
[(google.api.field_behavior) = REQUIRED];
139+
InstanceGroupAutoscalingPolicyConfig worker_config = 4 [(google.api.field_behavior) = REQUIRED];
148140

149141
// Optional. Describes how the autoscaler will operate for secondary workers.
150-
InstanceGroupAutoscalingPolicyConfig secondary_worker_config = 5
151-
[(google.api.field_behavior) = OPTIONAL];
142+
InstanceGroupAutoscalingPolicyConfig secondary_worker_config = 5 [(google.api.field_behavior) = OPTIONAL];
152143
}
153144

154145
// Basic algorithm for autoscaling.
155146
message BasicAutoscalingAlgorithm {
156147
// Required. YARN autoscaling configuration.
157-
BasicYarnAutoscalingConfig yarn_config = 1
158-
[(google.api.field_behavior) = REQUIRED];
148+
BasicYarnAutoscalingConfig yarn_config = 1 [(google.api.field_behavior) = REQUIRED];
159149

160150
// Optional. Duration between scaling events. A scaling period starts after
161151
// the update operation from the previous event has completed.
162152
//
163153
// Bounds: [2m, 1d]. Default: 2m.
164-
google.protobuf.Duration cooldown_period = 2
165-
[(google.api.field_behavior) = OPTIONAL];
154+
google.protobuf.Duration cooldown_period = 2 [(google.api.field_behavior) = OPTIONAL];
166155
}
167156

168157
// Basic autoscaling configurations for YARN.
@@ -173,8 +162,7 @@ message BasicYarnAutoscalingConfig {
173162
// downscaling operations.
174163
//
175164
// Bounds: [0s, 1d].
176-
google.protobuf.Duration graceful_decommission_timeout = 5
177-
[(google.api.field_behavior) = REQUIRED];
165+
google.protobuf.Duration graceful_decommission_timeout = 5 [(google.api.field_behavior) = REQUIRED];
178166

179167
// Required. Fraction of average pending memory in the last cooldown period
180168
// for which to add workers. A scale-up factor of 1.0 will result in scaling
@@ -201,8 +189,7 @@ message BasicYarnAutoscalingConfig {
201189
// on any recommended change.
202190
//
203191
// Bounds: [0.0, 1.0]. Default: 0.0.
204-
double scale_up_min_worker_fraction = 3
205-
[(google.api.field_behavior) = OPTIONAL];
192+
double scale_up_min_worker_fraction = 3 [(google.api.field_behavior) = OPTIONAL];
206193

207194
// Optional. Minimum scale-down threshold as a fraction of total cluster size
208195
// before scaling occurs. For example, in a 20-worker cluster, a threshold of
@@ -211,8 +198,7 @@ message BasicYarnAutoscalingConfig {
211198
// on any recommended change.
212199
//
213200
// Bounds: [0.0, 1.0]. Default: 0.0.
214-
double scale_down_min_worker_fraction = 4
215-
[(google.api.field_behavior) = OPTIONAL];
201+
double scale_down_min_worker_fraction = 4 [(google.api.field_behavior) = OPTIONAL];
216202
}
217203

218204
// Configuration for the size bounds of an instance group, including its
@@ -272,10 +258,8 @@ message CreateAutoscalingPolicyRequest {
272258
}
273259
];
274260

275-
// The autoscaling policy to create.
276-
AutoscalingPolicy policy = 2 [
277-
(google.api.field_behavior) = REQUIRED
278-
];
261+
// Required. The autoscaling policy to create.
262+
AutoscalingPolicy policy = 2 [(google.api.field_behavior) = REQUIRED];
279263
}
280264

281265
// A request to fetch an autoscaling policy.
@@ -357,8 +341,7 @@ message ListAutoscalingPoliciesRequest {
357341
// A response to a request to list autoscaling policies in a project.
358342
message ListAutoscalingPoliciesResponse {
359343
// Output only. Autoscaling policies list.
360-
repeated AutoscalingPolicy policies = 1
361-
[(google.api.field_behavior) = OUTPUT_ONLY];
344+
repeated AutoscalingPolicy policies = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
362345

363346
// Output only. This token is included in the response if there are more
364347
// results to fetch.

‎packages/google-cloud-dataproc/protos/google/cloud/dataproc/v1/clusters.proto

+44-26
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ service ClusterController {
8888
option (google.api.method_signature) = "project_id,region,cluster_name";
8989
}
9090

91-
// Lists all regions/{region}/clusters in a project.
91+
// Lists all regions/{region}/clusters in a project alphabetically.
9292
rpc ListClusters(ListClustersRequest) returns (ListClustersResponse) {
9393
option (google.api.http) = {
9494
get: "/v1/projects/{project_id}/regions/{region}/clusters"
@@ -150,11 +150,11 @@ message Cluster {
150150
// generates this value when it creates the cluster.
151151
string cluster_uuid = 6 [(google.api.field_behavior) = OUTPUT_ONLY];
152152

153-
// Contains cluster daemon metrics such as HDFS and YARN stats.
153+
// Output only. Contains cluster daemon metrics such as HDFS and YARN stats.
154154
//
155155
// **Beta Feature**: This report is available for testing purposes only. It
156156
// may be changed before final release.
157-
ClusterMetrics metrics = 9;
157+
ClusterMetrics metrics = 9 [(google.api.field_behavior) = OUTPUT_ONLY];
158158
}
159159

160160
// The cluster config.
@@ -167,7 +167,7 @@ message ClusterConfig {
167167
// Compute Engine zone where your cluster is deployed, and then create
168168
// and manage this project-level, per-location bucket (see
169169
// [Dataproc staging
170-
// bucket](/dataproc/docs/concepts/configuring-clusters/staging-bucket)).
170+
// bucket](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/staging-bucket)).
171171
string config_bucket = 1 [(google.api.field_behavior) = OPTIONAL];
172172

173173
// Optional. The shared Compute Engine config settings for
@@ -259,7 +259,8 @@ message GceClusterConfig {
259259
// communications. Cannot be specified with subnetwork_uri. If neither
260260
// `network_uri` nor `subnetwork_uri` is specified, the "default" network of
261261
// the project is used, if it exists. Cannot be a "Custom Subnet Network" (see
262-
// [Using Subnetworks](/compute/docs/subnetworks) for more information).
262+
// [Using Subnetworks](https://cloud.google.com/compute/docs/subnetworks) for
263+
// more information).
263264
//
264265
// A full URL, partial URI, or short name are valid. Examples:
265266
//
@@ -287,15 +288,15 @@ message GceClusterConfig {
287288
bool internal_ip_only = 7 [(google.api.field_behavior) = OPTIONAL];
288289

289290
// Optional. The [Dataproc service
290-
// account](/dataproc/docs/concepts/configuring-clusters/service-accounts#service_accounts_in_cloud_dataproc)
291+
// account](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/service-accounts#service_accounts_in_cloud_dataproc)
291292
// (also see [VM Data Plane
292-
// identity](/dataproc/docs/concepts/iam/dataproc-principals#vm_service_account_data_plane_identity))
293+
// identity](https://cloud.google.com/dataproc/docs/concepts/iam/dataproc-principals#vm_service_account_data_plane_identity))
293294
// used by Dataproc cluster VM instances to access Google Cloud Platform
294295
// services.
295296
//
296297
// If not specified, the
297298
// [Compute Engine default service
298-
// account](/compute/docs/access/service-accounts#default_service_account)
299+
// account](https://cloud.google.com/compute/docs/access/service-accounts#default_service_account)
299300
// is used.
300301
string service_account = 8 [(google.api.field_behavior) = OPTIONAL];
301302

@@ -315,8 +316,8 @@ message GceClusterConfig {
315316
// * https://www.googleapis.com/auth/devstorage.full_control
316317
repeated string service_account_scopes = 3 [(google.api.field_behavior) = OPTIONAL];
317318

318-
// The Compute Engine tags to add to all instances (see
319-
// [Tagging instances](/compute/docs/label-or-tag-resources#tags)).
319+
// The Compute Engine tags to add to all instances (see [Tagging
320+
// instances](https://cloud.google.com/compute/docs/label-or-tag-resources#tags)).
320321
repeated string tags = 4;
321322

322323
// The Compute Engine metadata entries to add to all instances (see
@@ -339,9 +340,24 @@ message InstanceGroupConfig {
339340
// from `cluster_name`, `num_instances`, and the instance group.
340341
repeated string instance_names = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
341342

342-
// Optional. The Compute Engine image resource used for cluster
343-
// instances. It can be specified or may be inferred from
344-
// `SoftwareConfig.image_version`.
343+
// Optional. The Compute Engine image resource used for cluster instances.
344+
//
345+
// The URI can represent an image or image family.
346+
//
347+
// Image examples:
348+
//
349+
// * `https://www.googleapis.com/compute/beta/projects/[project_id]/global/images/[image-id]`
350+
// * `projects/[project_id]/global/images/[image-id]`
351+
// * `image-id`
352+
//
353+
// Image family examples. Dataproc will use the most recent
354+
// image from the family:
355+
//
356+
// * `https://www.googleapis.com/compute/beta/projects/[project_id]/global/images/family/[custom-image-family-name]`
357+
// * `projects/[project_id]/global/images/family/[custom-image-family-name]`
358+
//
359+
// If the URI is unspecified, it will be inferred from
360+
// `SoftwareConfig.image_version` or the system default.
345361
string image_uri = 3 [(google.api.field_behavior) = OPTIONAL];
346362

347363
// Optional. The Compute Engine machine type used for cluster instances.
@@ -354,17 +370,17 @@ message InstanceGroupConfig {
354370
//
355371
// **Auto Zone Exception**: If you are using the Dataproc
356372
// [Auto Zone
357-
// Placement](/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement)
373+
// Placement](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement)
358374
// feature, you must use the short name of the machine type
359375
// resource, for example, `n1-standard-2`.
360376
string machine_type_uri = 4 [(google.api.field_behavior) = OPTIONAL];
361377

362378
// Optional. Disk option config settings.
363379
DiskConfig disk_config = 5 [(google.api.field_behavior) = OPTIONAL];
364380

365-
// Optional. Specifies that this instance group contains preemptible
381+
// Output only. Specifies that this instance group contains preemptible
366382
// instances.
367-
bool is_preemptible = 6 [(google.api.field_behavior) = OPTIONAL];
383+
bool is_preemptible = 6 [(google.api.field_behavior) = OUTPUT_ONLY];
368384

369385
// Output only. The config for Compute Engine Instance Group
370386
// Manager that manages this group.
@@ -376,8 +392,8 @@ message InstanceGroupConfig {
376392
repeated AcceleratorConfig accelerators = 8 [(google.api.field_behavior) = OPTIONAL];
377393

378394
// Optional. Specifies the minimum cpu platform for the Instance Group.
379-
// See [Dataproc→Minimum CPU Platform]
380-
// (/dataproc/docs/concepts/compute/dataproc-min-cpu).
395+
// See [Dataproc -> Minimum CPU
396+
// Platform](https://cloud.google.com/dataproc/docs/concepts/compute/dataproc-min-cpu).
381397
string min_cpu_platform = 9 [(google.api.field_behavior) = OPTIONAL];
382398
}
383399

@@ -392,12 +408,13 @@ message ManagedGroupConfig {
392408
}
393409

394410
// Specifies the type and number of accelerator cards attached to the instances
395-
// of an instance. See [GPUs on Compute Engine](/compute/docs/gpus/).
411+
// of an instance. See [GPUs on Compute
412+
// Engine](https://cloud.google.com/compute/docs/gpus/).
396413
message AcceleratorConfig {
397414
// Full URL, partial URI, or short name of the accelerator type resource to
398415
// expose to this instance. See
399416
// [Compute Engine
400-
// AcceleratorTypes](/compute/docs/reference/beta/acceleratorTypes).
417+
// AcceleratorTypes](https://cloud.google.com/compute/docs/reference/beta/acceleratorTypes).
401418
//
402419
// Examples:
403420
//
@@ -407,7 +424,7 @@ message AcceleratorConfig {
407424
//
408425
// **Auto Zone Exception**: If you are using the Dataproc
409426
// [Auto Zone
410-
// Placement](/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement)
427+
// Placement](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement)
411428
// feature, you must use the short name of the accelerator type
412429
// resource, for example, `nvidia-tesla-k80`.
413430
string accelerator_type_uri = 1;
@@ -519,7 +536,8 @@ message SecurityConfig {
519536

520537
// Specifies Kerberos related configuration.
521538
message KerberosConfig {
522-
// Optional. Flag to indicate whether to Kerberize the cluster.
539+
// Optional. Flag to indicate whether to Kerberize the cluster (default: false). Set
540+
// this field to true to enable Kerberos on a cluster.
523541
bool enable_kerberos = 1 [(google.api.field_behavior) = OPTIONAL];
524542

525543
// Required. The Cloud Storage URI of a KMS encrypted file containing the root
@@ -590,10 +608,10 @@ message KerberosConfig {
590608
message SoftwareConfig {
591609
// Optional. The version of software inside the cluster. It must be one of the
592610
// supported [Dataproc
593-
// Versions](/dataproc/docs/concepts/versioning/dataproc-versions#supported_cloud_dataproc_versions),
611+
// Versions](https://cloud.google.com/dataproc/docs/concepts/versioning/dataproc-versions#supported_cloud_dataproc_versions),
594612
// such as "1.2" (including a subminor version, such as "1.2.29"), or the
595613
// ["preview"
596-
// version](/dataproc/docs/concepts/versioning/dataproc-versions#other_versions).
614+
// version](https://cloud.google.com/dataproc/docs/concepts/versioning/dataproc-versions#other_versions).
597615
// If unspecified, it defaults to the latest Debian version.
598616
string image_version = 1 [(google.api.field_behavior) = OPTIONAL];
599617

@@ -613,8 +631,8 @@ message SoftwareConfig {
613631
// * spark: `spark-defaults.conf`
614632
// * yarn: `yarn-site.xml`
615633
//
616-
// For more information, see
617-
// [Cluster properties](/dataproc/docs/concepts/cluster-properties).
634+
// For more information, see [Cluster
635+
// properties](https://cloud.google.com/dataproc/docs/concepts/cluster-properties).
618636
map<string, string> properties = 2 [(google.api.field_behavior) = OPTIONAL];
619637

620638
// Optional. The set of components to activate on the cluster.

0 commit comments

Comments
 (0)