Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2025 Google LLC
// Copyright 2026 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -73,8 +73,9 @@ message Environment {
// specified in order for the job to have workers.
repeated WorkerPool worker_pools = 4;

// A description of the process that generated the request.
google.protobuf.Struct user_agent = 5;
// Optional. A description of the process that generated the request.
google.protobuf.Struct user_agent = 5
[(google.api.field_behavior) = OPTIONAL];

// A structure describing which components and their versions of the service
// are required in order to run the job.
Expand Down Expand Up @@ -139,6 +140,9 @@ message Environment {
// [Set the pipeline streaming
// mode](https://cloud.google.com/dataflow/docs/guides/streaming-modes).
StreamingMode streaming_mode = 19 [(google.api.field_behavior) = OPTIONAL];

// Optional. True when any worker pool that uses public IPs is present.
bool use_public_ips = 20 [(google.api.field_behavior) = OPTIONAL];
}

// The packages that must be installed in order for a worker to run the
Expand All @@ -161,6 +165,12 @@ message Package {
// storage.googleapis.com/{bucket}
// bucket.storage.googleapis.com/
string location = 2;

// Optional. The hex-encoded SHA256 checksum of the package.
// If the checksum is provided, the worker will verify the checksum of the
// package before using it. If the checksum does not match, the worker will
// fail to start.
string sha256 = 3 [(google.api.field_behavior) = OPTIONAL];
}

// Specifies the processing model used by a
Expand Down Expand Up @@ -494,6 +504,13 @@ message WorkerPool {
// attempt to choose a reasonable default.
string disk_type = 16;

// Optional. IOPS provisioned for the root disk for VMs.
int64 disk_provisioned_iops = 23 [(google.api.field_behavior) = OPTIONAL];

// Optional. Throughput provisioned for the root disk for VMs.
int64 disk_provisioned_throughput_mibps = 24
[(google.api.field_behavior) = OPTIONAL];

// Fully qualified source image for disks.
string disk_source_image = 8;

Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2025 Google LLC
// Copyright 2026 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -34,15 +34,15 @@ option java_package = "com.google.dataflow.v1beta3";
option php_namespace = "Google\\Cloud\\Dataflow\\V1beta3";
option ruby_package = "Google::Cloud::Dataflow::V1beta3";

// Provides a method to create and modify Google Cloud Dataflow jobs.
// A Job is a multi-stage computation graph run by the Cloud Dataflow service.
// Provides a method to create and modify Dataflow jobs.
// A Job is a multi-stage computation graph run by the Dataflow service.
service JobsV1Beta3 {
option (google.api.default_host) = "dataflow.googleapis.com";
option (google.api.oauth_scopes) =
"https://www.googleapis.com/auth/cloud-platform,"
"https://www.googleapis.com/auth/compute";

// Creates a Cloud Dataflow job.
// Creates a Dataflow job.
//
// To create a job, we recommend using `projects.locations.jobs.create` with a
// [regional endpoint]
Expand Down Expand Up @@ -315,6 +315,9 @@ message Job {
// Output only. Resources used by the Dataflow Service to run the job.
optional ServiceResources service_resources = 28
[(google.api.field_behavior) = OUTPUT_ONLY];

// Output only. Indicates whether the job can be paused.
bool pausable = 29 [(google.api.field_behavior) = OUTPUT_ONLY];
}

// Resources used by the Dataflow Service to run the job.
Expand Down Expand Up @@ -343,6 +346,16 @@ message RuntimeUpdatableParams {
// [Update an existing
// pipeline](https://cloud.google.com/dataflow/docs/guides/updating-a-pipeline).
optional double worker_utilization_hint = 3;

// Optional. Deprecated: Use `autoscaling_tier` instead.
// The backlog threshold duration in seconds for autoscaling. Value must be
// non-negative.
optional google.protobuf.Duration acceptable_backlog_duration = 4
[deprecated = true, (google.api.field_behavior) = OPTIONAL];

// Optional. The backlog threshold tier for autoscaling. Value must be one of
// "low-latency", "medium-latency", or "high-latency".
optional string autoscaling_tier = 5 [(google.api.field_behavior) = OPTIONAL];
}

// Metadata for a Datastore connector used by the job.
Expand Down Expand Up @@ -836,6 +849,12 @@ enum JobState {
// Currently, this is an opt-in feature, please reach out to Cloud support
// team if you are interested.
JOB_STATE_RESOURCE_CLEANING_UP = 12;

// `JOB_STATE_PAUSING` is not implemented yet.
JOB_STATE_PAUSING = 13;

// `JOB_STATE_PAUSED` is not implemented yet.
JOB_STATE_PAUSED = 14;
}

// Additional information about how a Cloud Dataflow job will be executed that
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2025 Google LLC
// Copyright 2026 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -29,8 +29,7 @@ option java_package = "com.google.dataflow.v1beta3";
option php_namespace = "Google\\Cloud\\Dataflow\\V1beta3";
option ruby_package = "Google::Cloud::Dataflow::V1beta3";

// The Dataflow Messages API is used for monitoring the progress of
// Dataflow jobs.
// The Dataflow Messages API is used to monitor the progress of Dataflow jobs.
service MessagesV1Beta3 {
option (google.api.default_host) = "dataflow.googleapis.com";
option (google.api.oauth_scopes) =
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2025 Google LLC
// Copyright 2026 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -30,8 +30,7 @@ option java_package = "com.google.dataflow.v1beta3";
option php_namespace = "Google\\Cloud\\Dataflow\\V1beta3";
option ruby_package = "Google::Cloud::Dataflow::V1beta3";

// The Dataflow Metrics API lets you monitor the progress of Dataflow
// jobs.
// The Dataflow Metrics API lets you monitor the progress of Dataflow jobs.
service MetricsV1Beta3 {
option (google.api.default_host) = "dataflow.googleapis.com";
option (google.api.oauth_scopes) =
Expand Down Expand Up @@ -96,7 +95,6 @@ message MetricStructuredName {
}

// Describes the state of a metric.
// Next ID: 14
message MetricUpdate {
// Name of the metric.
MetricStructuredName name = 1;
Expand Down Expand Up @@ -141,6 +139,12 @@ message MetricUpdate {
// possible value type is a BoundedTrieNode.
google.protobuf.Value trie = 13;

// Worker-computed aggregate value for the "Trie" aggregation kind. The only
// possible value type is a BoundedTrieNode.
// Introduced this field to avoid breaking older SDKs when Dataflow service
// starts to populate the `bounded_trie` field.
google.protobuf.Value bounded_trie = 14;

// A struct value describing properties of a distribution of numeric values.
google.protobuf.Value distribution = 11;

Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2025 Google LLC
// Copyright 2026 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2025 Google LLC
// Copyright 2026 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2025 Google LLC
// Copyright 2026 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -299,6 +299,10 @@ message FlexTemplateRuntimeEnvironment {
// mode](https://cloud.google.com/dataflow/docs/guides/streaming-modes).
optional StreamingMode streaming_mode = 26
[(google.api.field_behavior) = OPTIONAL];

// Optional. Additional pipeline option flags for the job.
repeated string additional_pipeline_options = 27
[(google.api.field_behavior) = OPTIONAL];
}

// A request to launch a Cloud Dataflow job from a FlexTemplate.
Expand Down Expand Up @@ -414,6 +418,10 @@ message RuntimeEnvironment {
// mode](https://cloud.google.com/dataflow/docs/guides/streaming-modes).
optional StreamingMode streaming_mode = 19
[(google.api.field_behavior) = OPTIONAL];

// Optional. Additional pipeline option flags for the job.
repeated string additional_pipeline_options = 20
[(google.api.field_behavior) = OPTIONAL];
}

// ParameterMetadataEnumOption specifies the option shown in the enum form.
Expand Down Expand Up @@ -523,6 +531,9 @@ message SDKInfo {

// Go.
GO = 3;

// YAML.
YAML = 4;
}

// Required. The SDK Language.
Expand Down
Loading
Loading