Google Cloud (GCP) Classic

Pulumi Official
Package maintained by Pulumi
v6.32.0 published on Wednesday, Jul 20, 2022 by Pulumi

WorkflowTemplate

A Workflow Template is a reusable workflow configuration. It defines a graph of jobs with information on where to run those jobs.

Example Usage

using Pulumi;
using Gcp = Pulumi.Gcp;

class MyStack : Stack
{
    public MyStack()
    {
        var template = new Gcp.Dataproc.WorkflowTemplate("template", new Gcp.Dataproc.WorkflowTemplateArgs
        {
            Jobs = 
            {
                new Gcp.Dataproc.Inputs.WorkflowTemplateJobArgs
                {
                    SparkJob = new Gcp.Dataproc.Inputs.WorkflowTemplateJobSparkJobArgs
                    {
                        MainClass = "SomeClass",
                    },
                    StepId = "someJob",
                },
                new Gcp.Dataproc.Inputs.WorkflowTemplateJobArgs
                {
                    PrerequisiteStepIds = 
                    {
                        "someJob",
                    },
                    PrestoJob = new Gcp.Dataproc.Inputs.WorkflowTemplateJobPrestoJobArgs
                    {
                        QueryFileUri = "someuri",
                    },
                    StepId = "otherJob",
                },
            },
            Location = "us-central1",
            Placement = new Gcp.Dataproc.Inputs.WorkflowTemplatePlacementArgs
            {
                ManagedCluster = new Gcp.Dataproc.Inputs.WorkflowTemplatePlacementManagedClusterArgs
                {
                    ClusterName = "my-cluster",
                    Config = new Gcp.Dataproc.Inputs.WorkflowTemplatePlacementManagedClusterConfigArgs
                    {
                        GceClusterConfig = new Gcp.Dataproc.Inputs.WorkflowTemplatePlacementManagedClusterConfigGceClusterConfigArgs
                        {
                            Tags = 
                            {
                                "foo",
                                "bar",
                            },
                            Zone = "us-central1-a",
                        },
                        MasterConfig = new Gcp.Dataproc.Inputs.WorkflowTemplatePlacementManagedClusterConfigMasterConfigArgs
                        {
                            DiskConfig = new Gcp.Dataproc.Inputs.WorkflowTemplatePlacementManagedClusterConfigMasterConfigDiskConfigArgs
                            {
                                BootDiskSizeGb = 15,
                                BootDiskType = "pd-ssd",
                            },
                            MachineType = "n1-standard-1",
                            NumInstances = 1,
                        },
                        SecondaryWorkerConfig = new Gcp.Dataproc.Inputs.WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigArgs
                        {
                            NumInstances = 2,
                        },
                        SoftwareConfig = new Gcp.Dataproc.Inputs.WorkflowTemplatePlacementManagedClusterConfigSoftwareConfigArgs
                        {
                            ImageVersion = "2.0.35-debian10",
                        },
                        WorkerConfig = new Gcp.Dataproc.Inputs.WorkflowTemplatePlacementManagedClusterConfigWorkerConfigArgs
                        {
                            DiskConfig = new Gcp.Dataproc.Inputs.WorkflowTemplatePlacementManagedClusterConfigWorkerConfigDiskConfigArgs
                            {
                                BootDiskSizeGb = 10,
                                NumLocalSsds = 2,
                            },
                            MachineType = "n1-standard-2",
                            NumInstances = 3,
                        },
                    },
                },
            },
        });
    }

}
package main

import (
	"github.com/pulumi/pulumi-gcp/sdk/v6/go/gcp/dataproc"
	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)

func main() {
	pulumi.Run(func(ctx *pulumi.Context) error {
		_, err := dataproc.NewWorkflowTemplate(ctx, "template", &dataproc.WorkflowTemplateArgs{
			Jobs: dataproc.WorkflowTemplateJobArray{
				&dataproc.WorkflowTemplateJobArgs{
					SparkJob: &dataproc.WorkflowTemplateJobSparkJobArgs{
						MainClass: pulumi.String("SomeClass"),
					},
					StepId: pulumi.String("someJob"),
				},
				&dataproc.WorkflowTemplateJobArgs{
					PrerequisiteStepIds: pulumi.StringArray{
						pulumi.String("someJob"),
					},
					PrestoJob: &dataproc.WorkflowTemplateJobPrestoJobArgs{
						QueryFileUri: pulumi.String("someuri"),
					},
					StepId: pulumi.String("otherJob"),
				},
			},
			Location: pulumi.String("us-central1"),
			Placement: &dataproc.WorkflowTemplatePlacementArgs{
				ManagedCluster: &dataproc.WorkflowTemplatePlacementManagedClusterArgs{
					ClusterName: pulumi.String("my-cluster"),
					Config: &dataproc.WorkflowTemplatePlacementManagedClusterConfigArgs{
						GceClusterConfig: &dataproc.WorkflowTemplatePlacementManagedClusterConfigGceClusterConfigArgs{
							Tags: pulumi.StringArray{
								pulumi.String("foo"),
								pulumi.String("bar"),
							},
							Zone: pulumi.String("us-central1-a"),
						},
						MasterConfig: &dataproc.WorkflowTemplatePlacementManagedClusterConfigMasterConfigArgs{
							DiskConfig: &dataproc.WorkflowTemplatePlacementManagedClusterConfigMasterConfigDiskConfigArgs{
								BootDiskSizeGb: pulumi.Int(15),
								BootDiskType:   pulumi.String("pd-ssd"),
							},
							MachineType:  pulumi.String("n1-standard-1"),
							NumInstances: pulumi.Int(1),
						},
						SecondaryWorkerConfig: &dataproc.WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigArgs{
							NumInstances: pulumi.Int(2),
						},
						SoftwareConfig: &dataproc.WorkflowTemplatePlacementManagedClusterConfigSoftwareConfigArgs{
							ImageVersion: pulumi.String("2.0.35-debian10"),
						},
						WorkerConfig: &dataproc.WorkflowTemplatePlacementManagedClusterConfigWorkerConfigArgs{
							DiskConfig: &dataproc.WorkflowTemplatePlacementManagedClusterConfigWorkerConfigDiskConfigArgs{
								BootDiskSizeGb: pulumi.Int(10),
								NumLocalSsds:   pulumi.Int(2),
							},
							MachineType:  pulumi.String("n1-standard-2"),
							NumInstances: pulumi.Int(3),
						},
					},
				},
			},
		})
		if err != nil {
			return err
		}
		return nil
	})
}
package generated_program;

import java.util.*;
import java.io.*;
import java.nio.*;
import com.pulumi.*;

public class App {
    public static void main(String[] args) {
        Pulumi.run(App::stack);
    }

    public static void stack(Context ctx) {
        var template = new WorkflowTemplate("template", WorkflowTemplateArgs.builder()        
            .jobs(            
                WorkflowTemplateJobArgs.builder()
                    .sparkJob(WorkflowTemplateJobSparkJobArgs.builder()
                        .mainClass("SomeClass")
                        .build())
                    .stepId("someJob")
                    .build(),
                WorkflowTemplateJobArgs.builder()
                    .prerequisiteStepIds("someJob")
                    .prestoJob(WorkflowTemplateJobPrestoJobArgs.builder()
                        .queryFileUri("someuri")
                        .build())
                    .stepId("otherJob")
                    .build())
            .location("us-central1")
            .placement(WorkflowTemplatePlacementArgs.builder()
                .managedCluster(WorkflowTemplatePlacementManagedClusterArgs.builder()
                    .clusterName("my-cluster")
                    .config(WorkflowTemplatePlacementManagedClusterConfigArgs.builder()
                        .gceClusterConfig(WorkflowTemplatePlacementManagedClusterConfigGceClusterConfigArgs.builder()
                            .tags(                            
                                "foo",
                                "bar")
                            .zone("us-central1-a")
                            .build())
                        .masterConfig(WorkflowTemplatePlacementManagedClusterConfigMasterConfigArgs.builder()
                            .diskConfig(WorkflowTemplatePlacementManagedClusterConfigMasterConfigDiskConfigArgs.builder()
                                .bootDiskSizeGb(15)
                                .bootDiskType("pd-ssd")
                                .build())
                            .machineType("n1-standard-1")
                            .numInstances(1)
                            .build())
                        .secondaryWorkerConfig(WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigArgs.builder()
                            .numInstances(2)
                            .build())
                        .softwareConfig(WorkflowTemplatePlacementManagedClusterConfigSoftwareConfigArgs.builder()
                            .imageVersion("2.0.35-debian10")
                            .build())
                        .workerConfig(WorkflowTemplatePlacementManagedClusterConfigWorkerConfigArgs.builder()
                            .diskConfig(WorkflowTemplatePlacementManagedClusterConfigWorkerConfigDiskConfigArgs.builder()
                                .bootDiskSizeGb(10)
                                .numLocalSsds(2)
                                .build())
                            .machineType("n1-standard-2")
                            .numInstances(3)
                            .build())
                        .build())
                    .build())
                .build())
            .build());

    }
}
import pulumi
import pulumi_gcp as gcp

template = gcp.dataproc.WorkflowTemplate("template",
    jobs=[
        gcp.dataproc.WorkflowTemplateJobArgs(
            spark_job=gcp.dataproc.WorkflowTemplateJobSparkJobArgs(
                main_class="SomeClass",
            ),
            step_id="someJob",
        ),
        gcp.dataproc.WorkflowTemplateJobArgs(
            prerequisite_step_ids=["someJob"],
            presto_job=gcp.dataproc.WorkflowTemplateJobPrestoJobArgs(
                query_file_uri="someuri",
            ),
            step_id="otherJob",
        ),
    ],
    location="us-central1",
    placement=gcp.dataproc.WorkflowTemplatePlacementArgs(
        managed_cluster=gcp.dataproc.WorkflowTemplatePlacementManagedClusterArgs(
            cluster_name="my-cluster",
            config=gcp.dataproc.WorkflowTemplatePlacementManagedClusterConfigArgs(
                gce_cluster_config=gcp.dataproc.WorkflowTemplatePlacementManagedClusterConfigGceClusterConfigArgs(
                    tags=[
                        "foo",
                        "bar",
                    ],
                    zone="us-central1-a",
                ),
                master_config=gcp.dataproc.WorkflowTemplatePlacementManagedClusterConfigMasterConfigArgs(
                    disk_config=gcp.dataproc.WorkflowTemplatePlacementManagedClusterConfigMasterConfigDiskConfigArgs(
                        boot_disk_size_gb=15,
                        boot_disk_type="pd-ssd",
                    ),
                    machine_type="n1-standard-1",
                    num_instances=1,
                ),
                secondary_worker_config=gcp.dataproc.WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigArgs(
                    num_instances=2,
                ),
                software_config=gcp.dataproc.WorkflowTemplatePlacementManagedClusterConfigSoftwareConfigArgs(
                    image_version="2.0.35-debian10",
                ),
                worker_config=gcp.dataproc.WorkflowTemplatePlacementManagedClusterConfigWorkerConfigArgs(
                    disk_config=gcp.dataproc.WorkflowTemplatePlacementManagedClusterConfigWorkerConfigDiskConfigArgs(
                        boot_disk_size_gb=10,
                        num_local_ssds=2,
                    ),
                    machine_type="n1-standard-2",
                    num_instances=3,
                ),
            ),
        ),
    ))
import * as pulumi from "@pulumi/pulumi";
import * as gcp from "@pulumi/gcp";

const template = new gcp.dataproc.WorkflowTemplate("template", {
    jobs: [
        {
            sparkJob: {
                mainClass: "SomeClass",
            },
            stepId: "someJob",
        },
        {
            prerequisiteStepIds: ["someJob"],
            prestoJob: {
                queryFileUri: "someuri",
            },
            stepId: "otherJob",
        },
    ],
    location: "us-central1",
    placement: {
        managedCluster: {
            clusterName: "my-cluster",
            config: {
                gceClusterConfig: {
                    tags: [
                        "foo",
                        "bar",
                    ],
                    zone: "us-central1-a",
                },
                masterConfig: {
                    diskConfig: {
                        bootDiskSizeGb: 15,
                        bootDiskType: "pd-ssd",
                    },
                    machineType: "n1-standard-1",
                    numInstances: 1,
                },
                secondaryWorkerConfig: {
                    numInstances: 2,
                },
                softwareConfig: {
                    imageVersion: "2.0.35-debian10",
                },
                workerConfig: {
                    diskConfig: {
                        bootDiskSizeGb: 10,
                        numLocalSsds: 2,
                    },
                    machineType: "n1-standard-2",
                    numInstances: 3,
                },
            },
        },
    },
});
resources:
  template:
    type: gcp:dataproc:WorkflowTemplate
    properties:
      jobs:
        - sparkJob:
            mainClass: SomeClass
          stepId: someJob
        - prerequisiteStepIds:
            - someJob
          prestoJob:
            queryFileUri: someuri
          stepId: otherJob
      location: us-central1
      placement:
        managedCluster:
          clusterName: my-cluster
          config:
            gceClusterConfig:
              tags:
                - foo
                - bar
              zone: us-central1-a
            masterConfig:
              diskConfig:
                bootDiskSizeGb: 15
                bootDiskType: pd-ssd
              machineType: n1-standard-1
              numInstances: 1
            secondaryWorkerConfig:
              numInstances: 2
            softwareConfig:
              imageVersion: 2.0.35-debian10
            workerConfig:
              diskConfig:
                bootDiskSizeGb: 10
                numLocalSsds: 2
              machineType: n1-standard-2
              numInstances: 3

Create a WorkflowTemplate Resource

new WorkflowTemplate(name: string, args: WorkflowTemplateArgs, opts?: CustomResourceOptions);
@overload
def WorkflowTemplate(resource_name: str,
                     opts: Optional[ResourceOptions] = None,
                     dag_timeout: Optional[str] = None,
                     jobs: Optional[Sequence[WorkflowTemplateJobArgs]] = None,
                     labels: Optional[Mapping[str, str]] = None,
                     location: Optional[str] = None,
                     name: Optional[str] = None,
                     parameters: Optional[Sequence[WorkflowTemplateParameterArgs]] = None,
                     placement: Optional[WorkflowTemplatePlacementArgs] = None,
                     project: Optional[str] = None,
                     version: Optional[int] = None)
@overload
def WorkflowTemplate(resource_name: str,
                     args: WorkflowTemplateArgs,
                     opts: Optional[ResourceOptions] = None)
func NewWorkflowTemplate(ctx *Context, name string, args WorkflowTemplateArgs, opts ...ResourceOption) (*WorkflowTemplate, error)
public WorkflowTemplate(string name, WorkflowTemplateArgs args, CustomResourceOptions? opts = null)
public WorkflowTemplate(String name, WorkflowTemplateArgs args)
public WorkflowTemplate(String name, WorkflowTemplateArgs args, CustomResourceOptions options)
type: gcp:dataproc:WorkflowTemplate
properties: # The arguments to resource properties.
options: # Bag of options to control resource's behavior.

name string
The unique name of the resource.
args WorkflowTemplateArgs
The arguments to resource properties.
opts CustomResourceOptions
Bag of options to control resource's behavior.
resource_name str
The unique name of the resource.
args WorkflowTemplateArgs
The arguments to resource properties.
opts ResourceOptions
Bag of options to control resource's behavior.
ctx Context
Context object for the current deployment.
name string
The unique name of the resource.
args WorkflowTemplateArgs
The arguments to resource properties.
opts ResourceOption
Bag of options to control resource's behavior.
name string
The unique name of the resource.
args WorkflowTemplateArgs
The arguments to resource properties.
opts CustomResourceOptions
Bag of options to control resource's behavior.
name String
The unique name of the resource.
args WorkflowTemplateArgs
The arguments to resource properties.
options CustomResourceOptions
Bag of options to control resource's behavior.

WorkflowTemplate Resource Properties

To learn more about resource properties and how to use them, see Inputs and Outputs in the Architecture and Concepts docs.

Inputs

The WorkflowTemplate resource accepts the following input properties:

Jobs List<WorkflowTemplateJobArgs>

Required. The Directed Acyclic Graph of Jobs to submit.

Location string

The location for the resource

Placement WorkflowTemplatePlacementArgs

Required. WorkflowTemplate scheduling information.

DagTimeout string

(Beta only) Optional. Timeout duration for the DAG of jobs. You can use "s", "m", "h", and "d" suffixes for second, minute, hour, and day duration values, respectively. The timeout duration must be from 10 minutes ("10m") to 24 hours ("24h" or "1d"). The timer begins when the first job is submitted. If the workflow is running at the end of the timeout period, any remaining jobs are cancelled, the workflow is ended, and if the workflow was running on a (/dataproc/docs/concepts/workflows/using-workflows#configuring_or_selecting_a_cluster), the cluster is deleted.

Labels Dictionary<string, string>

Optional. The labels to associate with this cluster. Label keys must be between 1 and 63 characters long, and must conform to the following PCRE regular expression: {0,63} No more than 32 labels can be associated with a given cluster.

Name string

Required. Parameter name. The parameter name is used as the key, and paired with the parameter value, which are passed to the template when the template is instantiated. The name must contain only capital letters (A-Z), numbers (0-9), and underscores (_), and must not start with a number. The maximum length is 40 characters.

Parameters List<WorkflowTemplateParameterArgs>

Optional. Template parameters whose values are substituted into the template. Values for parameters must be provided when the template is instantiated.

Project string

The project for the resource

Version int

Optional. Used to perform a consistent read-modify-write. This field should be left blank for a CreateWorkflowTemplate request. It is required for an UpdateWorkflowTemplate request, and must match the current server version. A typical update template flow would fetch the current template with a GetWorkflowTemplate request, which will return the current template with the version field filled in with the current server version. The user updates other fields in the template, then returns it as part of the UpdateWorkflowTemplate request.

Deprecated:

version is not useful as a configurable field, and will be removed in the future.

Jobs []WorkflowTemplateJobArgs

Required. The Directed Acyclic Graph of Jobs to submit.

Location string

The location for the resource

Placement WorkflowTemplatePlacementArgs

Required. WorkflowTemplate scheduling information.

DagTimeout string

(Beta only) Optional. Timeout duration for the DAG of jobs. You can use "s", "m", "h", and "d" suffixes for second, minute, hour, and day duration values, respectively. The timeout duration must be from 10 minutes ("10m") to 24 hours ("24h" or "1d"). The timer begins when the first job is submitted. If the workflow is running at the end of the timeout period, any remaining jobs are cancelled, the workflow is ended, and if the workflow was running on a (/dataproc/docs/concepts/workflows/using-workflows#configuring_or_selecting_a_cluster), the cluster is deleted.

Labels map[string]string

Optional. The labels to associate with this cluster. Label keys must be between 1 and 63 characters long, and must conform to the following PCRE regular expression: {0,63} No more than 32 labels can be associated with a given cluster.

Name string

Required. Parameter name. The parameter name is used as the key, and paired with the parameter value, which are passed to the template when the template is instantiated. The name must contain only capital letters (A-Z), numbers (0-9), and underscores (_), and must not start with a number. The maximum length is 40 characters.

Parameters []WorkflowTemplateParameterArgs

Optional. Template parameters whose values are substituted into the template. Values for parameters must be provided when the template is instantiated.

Project string

The project for the resource

Version int

Optional. Used to perform a consistent read-modify-write. This field should be left blank for a CreateWorkflowTemplate request. It is required for an UpdateWorkflowTemplate request, and must match the current server version. A typical update template flow would fetch the current template with a GetWorkflowTemplate request, which will return the current template with the version field filled in with the current server version. The user updates other fields in the template, then returns it as part of the UpdateWorkflowTemplate request.

Deprecated:

version is not useful as a configurable field, and will be removed in the future.

jobs List<WorkflowTemplateJobArgs>

Required. The Directed Acyclic Graph of Jobs to submit.

location String

The location for the resource

placement WorkflowTemplatePlacementArgs

Required. WorkflowTemplate scheduling information.

dagTimeout String

(Beta only) Optional. Timeout duration for the DAG of jobs. You can use "s", "m", "h", and "d" suffixes for second, minute, hour, and day duration values, respectively. The timeout duration must be from 10 minutes ("10m") to 24 hours ("24h" or "1d"). The timer begins when the first job is submitted. If the workflow is running at the end of the timeout period, any remaining jobs are cancelled, the workflow is ended, and if the workflow was running on a (/dataproc/docs/concepts/workflows/using-workflows#configuring_or_selecting_a_cluster), the cluster is deleted.

labels Map<String,String>

Optional. The labels to associate with this cluster. Label keys must be between 1 and 63 characters long, and must conform to the following PCRE regular expression: {0,63} No more than 32 labels can be associated with a given cluster.

name String

Required. Parameter name. The parameter name is used as the key, and paired with the parameter value, which are passed to the template when the template is instantiated. The name must contain only capital letters (A-Z), numbers (0-9), and underscores (_), and must not start with a number. The maximum length is 40 characters.

parameters List<WorkflowTemplateParameterArgs>

Optional. Template parameters whose values are substituted into the template. Values for parameters must be provided when the template is instantiated.

project String

The project for the resource

version Integer

Optional. Used to perform a consistent read-modify-write. This field should be left blank for a CreateWorkflowTemplate request. It is required for an UpdateWorkflowTemplate request, and must match the current server version. A typical update template flow would fetch the current template with a GetWorkflowTemplate request, which will return the current template with the version field filled in with the current server version. The user updates other fields in the template, then returns it as part of the UpdateWorkflowTemplate request.

Deprecated:

version is not useful as a configurable field, and will be removed in the future.

jobs WorkflowTemplateJobArgs[]

Required. The Directed Acyclic Graph of Jobs to submit.

location string

The location for the resource

placement WorkflowTemplatePlacementArgs

Required. WorkflowTemplate scheduling information.

dagTimeout string

(Beta only) Optional. Timeout duration for the DAG of jobs. You can use "s", "m", "h", and "d" suffixes for second, minute, hour, and day duration values, respectively. The timeout duration must be from 10 minutes ("10m") to 24 hours ("24h" or "1d"). The timer begins when the first job is submitted. If the workflow is running at the end of the timeout period, any remaining jobs are cancelled, the workflow is ended, and if the workflow was running on a (/dataproc/docs/concepts/workflows/using-workflows#configuring_or_selecting_a_cluster), the cluster is deleted.

labels {[key: string]: string}

Optional. The labels to associate with this cluster. Label keys must be between 1 and 63 characters long, and must conform to the following PCRE regular expression: {0,63} No more than 32 labels can be associated with a given cluster.

name string

Required. Parameter name. The parameter name is used as the key, and paired with the parameter value, which are passed to the template when the template is instantiated. The name must contain only capital letters (A-Z), numbers (0-9), and underscores (_), and must not start with a number. The maximum length is 40 characters.

parameters WorkflowTemplateParameterArgs[]

Optional. Template parameters whose values are substituted into the template. Values for parameters must be provided when the template is instantiated.

project string

The project for the resource

version number

Optional. Used to perform a consistent read-modify-write. This field should be left blank for a CreateWorkflowTemplate request. It is required for an UpdateWorkflowTemplate request, and must match the current server version. A typical update template flow would fetch the current template with a GetWorkflowTemplate request, which will return the current template with the version field filled in with the current server version. The user updates other fields in the template, then returns it as part of the UpdateWorkflowTemplate request.

Deprecated:

version is not useful as a configurable field, and will be removed in the future.

jobs Sequence[WorkflowTemplateJobArgs]

Required. The Directed Acyclic Graph of Jobs to submit.

location str

The location for the resource

placement WorkflowTemplatePlacementArgs

Required. WorkflowTemplate scheduling information.

dag_timeout str

(Beta only) Optional. Timeout duration for the DAG of jobs. You can use "s", "m", "h", and "d" suffixes for second, minute, hour, and day duration values, respectively. The timeout duration must be from 10 minutes ("10m") to 24 hours ("24h" or "1d"). The timer begins when the first job is submitted. If the workflow is running at the end of the timeout period, any remaining jobs are cancelled, the workflow is ended, and if the workflow was running on a (/dataproc/docs/concepts/workflows/using-workflows#configuring_or_selecting_a_cluster), the cluster is deleted.

labels Mapping[str, str]

Optional. The labels to associate with this cluster. Label keys must be between 1 and 63 characters long, and must conform to the following PCRE regular expression: {0,63} No more than 32 labels can be associated with a given cluster.

name str

Required. Parameter name. The parameter name is used as the key, and paired with the parameter value, which are passed to the template when the template is instantiated. The name must contain only capital letters (A-Z), numbers (0-9), and underscores (_), and must not start with a number. The maximum length is 40 characters.

parameters Sequence[WorkflowTemplateParameterArgs]

Optional. Template parameters whose values are substituted into the template. Values for parameters must be provided when the template is instantiated.

project str

The project for the resource

version int

Optional. Used to perform a consistent read-modify-write. This field should be left blank for a CreateWorkflowTemplate request. It is required for an UpdateWorkflowTemplate request, and must match the current server version. A typical update template flow would fetch the current template with a GetWorkflowTemplate request, which will return the current template with the version field filled in with the current server version. The user updates other fields in the template, then returns it as part of the UpdateWorkflowTemplate request.

Deprecated:

version is not useful as a configurable field, and will be removed in the future.

jobs List<Property Map>

Required. The Directed Acyclic Graph of Jobs to submit.

location String

The location for the resource

placement Property Map

Required. WorkflowTemplate scheduling information.

dagTimeout String

(Beta only) Optional. Timeout duration for the DAG of jobs. You can use "s", "m", "h", and "d" suffixes for second, minute, hour, and day duration values, respectively. The timeout duration must be from 10 minutes ("10m") to 24 hours ("24h" or "1d"). The timer begins when the first job is submitted. If the workflow is running at the end of the timeout period, any remaining jobs are cancelled, the workflow is ended, and if the workflow was running on a (/dataproc/docs/concepts/workflows/using-workflows#configuring_or_selecting_a_cluster), the cluster is deleted.

labels Map<String>

Optional. The labels to associate with this cluster. Label keys must be between 1 and 63 characters long, and must conform to the following PCRE regular expression: {0,63} No more than 32 labels can be associated with a given cluster.

name String

Required. Parameter name. The parameter name is used as the key, and paired with the parameter value, which are passed to the template when the template is instantiated. The name must contain only capital letters (A-Z), numbers (0-9), and underscores (_), and must not start with a number. The maximum length is 40 characters.

parameters List<Property Map>

Optional. Template parameters whose values are substituted into the template. Values for parameters must be provided when the template is instantiated.

project String

The project for the resource

version Number

Optional. Used to perform a consistent read-modify-write. This field should be left blank for a CreateWorkflowTemplate request. It is required for an UpdateWorkflowTemplate request, and must match the current server version. A typical update template flow would fetch the current template with a GetWorkflowTemplate request, which will return the current template with the version field filled in with the current server version. The user updates other fields in the template, then returns it as part of the UpdateWorkflowTemplate request.

Deprecated:

version is not useful as a configurable field, and will be removed in the future.

Outputs

All input properties are implicitly available as output properties. Additionally, the WorkflowTemplate resource produces the following output properties:

CreateTime string

Output only. The time template was created.

Id string

The provider-assigned unique ID for this managed resource.

UpdateTime string

Output only. The time template was last updated.

CreateTime string

Output only. The time template was created.

Id string

The provider-assigned unique ID for this managed resource.

UpdateTime string

Output only. The time template was last updated.

createTime String

Output only. The time template was created.

id String

The provider-assigned unique ID for this managed resource.

updateTime String

Output only. The time template was last updated.

createTime string

Output only. The time template was created.

id string

The provider-assigned unique ID for this managed resource.

updateTime string

Output only. The time template was last updated.

create_time str

Output only. The time template was created.

id str

The provider-assigned unique ID for this managed resource.

update_time str

Output only. The time template was last updated.

createTime String

Output only. The time template was created.

id String

The provider-assigned unique ID for this managed resource.

updateTime String

Output only. The time template was last updated.

Look up an Existing WorkflowTemplate Resource

Get an existing WorkflowTemplate resource’s state with the given name, ID, and optional extra properties used to qualify the lookup.

public static get(name: string, id: Input<ID>, state?: WorkflowTemplateState, opts?: CustomResourceOptions): WorkflowTemplate
@staticmethod
def get(resource_name: str,
        id: str,
        opts: Optional[ResourceOptions] = None,
        create_time: Optional[str] = None,
        dag_timeout: Optional[str] = None,
        jobs: Optional[Sequence[WorkflowTemplateJobArgs]] = None,
        labels: Optional[Mapping[str, str]] = None,
        location: Optional[str] = None,
        name: Optional[str] = None,
        parameters: Optional[Sequence[WorkflowTemplateParameterArgs]] = None,
        placement: Optional[WorkflowTemplatePlacementArgs] = None,
        project: Optional[str] = None,
        update_time: Optional[str] = None,
        version: Optional[int] = None) -> WorkflowTemplate
func GetWorkflowTemplate(ctx *Context, name string, id IDInput, state *WorkflowTemplateState, opts ...ResourceOption) (*WorkflowTemplate, error)
public static WorkflowTemplate Get(string name, Input<string> id, WorkflowTemplateState? state, CustomResourceOptions? opts = null)
public static WorkflowTemplate get(String name, Output<String> id, WorkflowTemplateState state, CustomResourceOptions options)
Resource lookup is not supported in YAML
name
The unique name of the resulting resource.
id
The unique provider ID of the resource to lookup.
state
Any extra arguments used during the lookup.
opts
A bag of options that control this resource's behavior.
resource_name
The unique name of the resulting resource.
id
The unique provider ID of the resource to lookup.
name
The unique name of the resulting resource.
id
The unique provider ID of the resource to lookup.
state
Any extra arguments used during the lookup.
opts
A bag of options that control this resource's behavior.
name
The unique name of the resulting resource.
id
The unique provider ID of the resource to lookup.
state
Any extra arguments used during the lookup.
opts
A bag of options that control this resource's behavior.
name
The unique name of the resulting resource.
id
The unique provider ID of the resource to lookup.
state
Any extra arguments used during the lookup.
opts
A bag of options that control this resource's behavior.
The following state arguments are supported:
CreateTime string

Output only. The time template was created.

DagTimeout string

(Beta only) Optional. Timeout duration for the DAG of jobs. You can use "s", "m", "h", and "d" suffixes for second, minute, hour, and day duration values, respectively. The timeout duration must be from 10 minutes ("10m") to 24 hours ("24h" or "1d"). The timer begins when the first job is submitted. If the workflow is running at the end of the timeout period, any remaining jobs are cancelled, the workflow is ended, and if the workflow was running on a (/dataproc/docs/concepts/workflows/using-workflows#configuring_or_selecting_a_cluster), the cluster is deleted.

Jobs List<WorkflowTemplateJobArgs>

Required. The Directed Acyclic Graph of Jobs to submit.

Labels Dictionary<string, string>

Optional. The labels to associate with this cluster. Label keys must be between 1 and 63 characters long, and must conform to the following PCRE regular expression: {0,63} No more than 32 labels can be associated with a given cluster.

Location string

The location for the resource

Name string

Required. Parameter name. The parameter name is used as the key, and paired with the parameter value, which are passed to the template when the template is instantiated. The name must contain only capital letters (A-Z), numbers (0-9), and underscores (_), and must not start with a number. The maximum length is 40 characters.

Parameters List<WorkflowTemplateParameterArgs>

Optional. Template parameters whose values are substituted into the template. Values for parameters must be provided when the template is instantiated.

Placement WorkflowTemplatePlacementArgs

Required. WorkflowTemplate scheduling information.

Project string

The project for the resource

UpdateTime string

Output only. The time template was last updated.

Version int

Optional. Used to perform a consistent read-modify-write. This field should be left blank for a CreateWorkflowTemplate request. It is required for an UpdateWorkflowTemplate request, and must match the current server version. A typical update template flow would fetch the current template with a GetWorkflowTemplate request, which will return the current template with the version field filled in with the current server version. The user updates other fields in the template, then returns it as part of the UpdateWorkflowTemplate request.

Deprecated:

version is not useful as a configurable field, and will be removed in the future.

CreateTime string

Output only. The time template was created.

DagTimeout string

(Beta only) Optional. Timeout duration for the DAG of jobs. You can use "s", "m", "h", and "d" suffixes for second, minute, hour, and day duration values, respectively. The timeout duration must be from 10 minutes ("10m") to 24 hours ("24h" or "1d"). The timer begins when the first job is submitted. If the workflow is running at the end of the timeout period, any remaining jobs are cancelled, the workflow is ended, and if the workflow was running on a (/dataproc/docs/concepts/workflows/using-workflows#configuring_or_selecting_a_cluster), the cluster is deleted.

Jobs []WorkflowTemplateJobArgs

Required. The Directed Acyclic Graph of Jobs to submit.

Labels map[string]string

Optional. The labels to associate with this cluster. Label keys must be between 1 and 63 characters long, and must conform to the following PCRE regular expression: {0,63} No more than 32 labels can be associated with a given cluster.

Location string

The location for the resource

Name string

Required. Parameter name. The parameter name is used as the key, and paired with the parameter value, which are passed to the template when the template is instantiated. The name must contain only capital letters (A-Z), numbers (0-9), and underscores (_), and must not start with a number. The maximum length is 40 characters.

Parameters []WorkflowTemplateParameterArgs

Optional. Template parameters whose values are substituted into the template. Values for parameters must be provided when the template is instantiated.

Placement WorkflowTemplatePlacementArgs

Required. WorkflowTemplate scheduling information.

Project string

The project for the resource

UpdateTime string

Output only. The time template was last updated.

Version int

Optional. Used to perform a consistent read-modify-write. This field should be left blank for a CreateWorkflowTemplate request. It is required for an UpdateWorkflowTemplate request, and must match the current server version. A typical update template flow would fetch the current template with a GetWorkflowTemplate request, which will return the current template with the version field filled in with the current server version. The user updates other fields in the template, then returns it as part of the UpdateWorkflowTemplate request.

Deprecated:

version is not useful as a configurable field, and will be removed in the future.

createTime String

Output only. The time template was created.

dagTimeout String

(Beta only) Optional. Timeout duration for the DAG of jobs. You can use "s", "m", "h", and "d" suffixes for second, minute, hour, and day duration values, respectively. The timeout duration must be from 10 minutes ("10m") to 24 hours ("24h" or "1d"). The timer begins when the first job is submitted. If the workflow is running at the end of the timeout period, any remaining jobs are cancelled, the workflow is ended, and if the workflow was running on a (/dataproc/docs/concepts/workflows/using-workflows#configuring_or_selecting_a_cluster), the cluster is deleted.

jobs List<WorkflowTemplateJobArgs>

Required. The Directed Acyclic Graph of Jobs to submit.

labels Map<String,String>

Optional. The labels to associate with this cluster. Label keys must be between 1 and 63 characters long, and must conform to the following PCRE regular expression: {0,63} No more than 32 labels can be associated with a given cluster.

location String

The location for the resource

name String

Required. Parameter name. The parameter name is used as the key, and paired with the parameter value, which are passed to the template when the template is instantiated. The name must contain only capital letters (A-Z), numbers (0-9), and underscores (_), and must not start with a number. The maximum length is 40 characters.

parameters List<WorkflowTemplateParameterArgs>

Optional. Template parameters whose values are substituted into the template. Values for parameters must be provided when the template is instantiated.

placement WorkflowTemplatePlacementArgs

Required. WorkflowTemplate scheduling information.

project String

The project for the resource

updateTime String

Output only. The time template was last updated.

version Integer

Optional. Used to perform a consistent read-modify-write. This field should be left blank for a CreateWorkflowTemplate request. It is required for an UpdateWorkflowTemplate request, and must match the current server version. A typical update template flow would fetch the current template with a GetWorkflowTemplate request, which will return the current template with the version field filled in with the current server version. The user updates other fields in the template, then returns it as part of the UpdateWorkflowTemplate request.

Deprecated:

version is not useful as a configurable field, and will be removed in the future.

createTime string

Output only. The time template was created.

dagTimeout string

(Beta only) Optional. Timeout duration for the DAG of jobs. You can use "s", "m", "h", and "d" suffixes for second, minute, hour, and day duration values, respectively. The timeout duration must be from 10 minutes ("10m") to 24 hours ("24h" or "1d"). The timer begins when the first job is submitted. If the workflow is running at the end of the timeout period, any remaining jobs are cancelled, the workflow is ended, and if the workflow was running on a (/dataproc/docs/concepts/workflows/using-workflows#configuring_or_selecting_a_cluster), the cluster is deleted.

jobs WorkflowTemplateJobArgs[]

Required. The Directed Acyclic Graph of Jobs to submit.

labels {[key: string]: string}

Optional. The labels to associate with this cluster. Label keys must be between 1 and 63 characters long, and must conform to the following PCRE regular expression: {0,63} No more than 32 labels can be associated with a given cluster.

location string

The location for the resource

name string

Required. Parameter name. The parameter name is used as the key, and paired with the parameter value, which are passed to the template when the template is instantiated. The name must contain only capital letters (A-Z), numbers (0-9), and underscores (_), and must not start with a number. The maximum length is 40 characters.

parameters WorkflowTemplateParameterArgs[]

Optional. Template parameters whose values are substituted into the template. Values for parameters must be provided when the template is instantiated.

placement WorkflowTemplatePlacementArgs

Required. WorkflowTemplate scheduling information.

project string

The project for the resource

updateTime string

Output only. The time template was last updated.

version number

Optional. Used to perform a consistent read-modify-write. This field should be left blank for a CreateWorkflowTemplate request. It is required for an UpdateWorkflowTemplate request, and must match the current server version. A typical update template flow would fetch the current template with a GetWorkflowTemplate request, which will return the current template with the version field filled in with the current server version. The user updates other fields in the template, then returns it as part of the UpdateWorkflowTemplate request.

Deprecated:

version is not useful as a configurable field, and will be removed in the future.

create_time str

Output only. The time template was created.

dag_timeout str

(Beta only) Optional. Timeout duration for the DAG of jobs. You can use "s", "m", "h", and "d" suffixes for second, minute, hour, and day duration values, respectively. The timeout duration must be from 10 minutes ("10m") to 24 hours ("24h" or "1d"). The timer begins when the first job is submitted. If the workflow is running at the end of the timeout period, any remaining jobs are cancelled, the workflow is ended, and if the workflow was running on a (/dataproc/docs/concepts/workflows/using-workflows#configuring_or_selecting_a_cluster), the cluster is deleted.

jobs Sequence[WorkflowTemplateJobArgs]

Required. The Directed Acyclic Graph of Jobs to submit.

labels Mapping[str, str]

Optional. The labels to associate with this cluster. Label keys must be between 1 and 63 characters long, and must conform to the following PCRE regular expression: {0,63} No more than 32 labels can be associated with a given cluster.

location str

The location for the resource

name str

Required. Parameter name. The parameter name is used as the key, and paired with the parameter value, which are passed to the template when the template is instantiated. The name must contain only capital letters (A-Z), numbers (0-9), and underscores (_), and must not start with a number. The maximum length is 40 characters.

parameters Sequence[WorkflowTemplateParameterArgs]

Optional. Template parameters whose values are substituted into the template. Values for parameters must be provided when the template is instantiated.

placement WorkflowTemplatePlacementArgs

Required. WorkflowTemplate scheduling information.

project str

The project for the resource

update_time str

Output only. The time template was last updated.

version int

Optional. Used to perform a consistent read-modify-write. This field should be left blank for a CreateWorkflowTemplate request. It is required for an UpdateWorkflowTemplate request, and must match the current server version. A typical update template flow would fetch the current template with a GetWorkflowTemplate request, which will return the current template with the version field filled in with the current server version. The user updates other fields in the template, then returns it as part of the UpdateWorkflowTemplate request.

Deprecated:

version is not useful as a configurable field, and will be removed in the future.

createTime String

Output only. The time template was created.

dagTimeout String

(Beta only) Optional. Timeout duration for the DAG of jobs. You can use "s", "m", "h", and "d" suffixes for second, minute, hour, and day duration values, respectively. The timeout duration must be from 10 minutes ("10m") to 24 hours ("24h" or "1d"). The timer begins when the first job is submitted. If the workflow is running at the end of the timeout period, any remaining jobs are cancelled, the workflow is ended, and if the workflow was running on a (/dataproc/docs/concepts/workflows/using-workflows#configuring_or_selecting_a_cluster), the cluster is deleted.

jobs List<Property Map>

Required. The Directed Acyclic Graph of Jobs to submit.

labels Map<String>

Optional. The labels to associate with this cluster. Label keys must be between 1 and 63 characters long, and must conform to the following PCRE regular expression: {0,63} No more than 32 labels can be associated with a given cluster.

location String

The location for the resource

name String

Required. Parameter name. The parameter name is used as the key, and paired with the parameter value, which are passed to the template when the template is instantiated. The name must contain only capital letters (A-Z), numbers (0-9), and underscores (_), and must not start with a number. The maximum length is 40 characters.

parameters List<Property Map>

Optional. Template parameters whose values are substituted into the template. Values for parameters must be provided when the template is instantiated.

placement Property Map

Required. WorkflowTemplate scheduling information.

project String

The project for the resource

updateTime String

Output only. The time template was last updated.

version Number

Optional. Used to perform a consistent read-modify-write. This field should be left blank for a CreateWorkflowTemplate request. It is required for an UpdateWorkflowTemplate request, and must match the current server version. A typical update template flow would fetch the current template with a GetWorkflowTemplate request, which will return the current template with the version field filled in with the current server version. The user updates other fields in the template, then returns it as part of the UpdateWorkflowTemplate request.

Deprecated:

version is not useful as a configurable field, and will be removed in the future.

Supporting Types

WorkflowTemplateJob

StepId string

Required. The step id. The id must be unique among all jobs within the template. The step id is used as prefix for job id, as job goog-dataproc-workflow-step-id label, and in field from other steps. The id must contain only letters (a-z, A-Z), numbers (0-9), underscores (_), and hyphens (-). Cannot begin or end with underscore or hyphen. Must consist of between 3 and 50 characters.

HadoopJob WorkflowTemplateJobHadoopJob

Optional. Job is a Hadoop job.

HiveJob WorkflowTemplateJobHiveJob

Optional. Job is a Hive job.

Labels Dictionary<string, string>

Optional. The labels to associate with this cluster. Label keys must be between 1 and 63 characters long, and must conform to the following PCRE regular expression: {0,63} No more than 32 labels can be associated with a given cluster.

PigJob WorkflowTemplateJobPigJob

Optional. Job is a Pig job.

PrerequisiteStepIds List<string>

Optional. The optional list of prerequisite job step_ids. If not specified, the job will start at the beginning of workflow.

PrestoJob WorkflowTemplateJobPrestoJob

Optional. Job is a Presto job.

PysparkJob WorkflowTemplateJobPysparkJob

Optional. Job is a PySpark job.

Scheduling WorkflowTemplateJobScheduling

Optional. Job scheduling configuration.

SparkJob WorkflowTemplateJobSparkJob

Optional. Job is a Spark job.

SparkRJob WorkflowTemplateJobSparkRJob

Optional. Job is a SparkR job.

SparkSqlJob WorkflowTemplateJobSparkSqlJob

Optional. Job is a SparkSql job.

StepId string

Required. The step id. The id must be unique among all jobs within the template. The step id is used as prefix for job id, as job goog-dataproc-workflow-step-id label, and in field from other steps. The id must contain only letters (a-z, A-Z), numbers (0-9), underscores (_), and hyphens (-). Cannot begin or end with underscore or hyphen. Must consist of between 3 and 50 characters.

HadoopJob WorkflowTemplateJobHadoopJob

Optional. Job is a Hadoop job.

HiveJob WorkflowTemplateJobHiveJob

Optional. Job is a Hive job.

Labels map[string]string

Optional. The labels to associate with this cluster. Label keys must be between 1 and 63 characters long, and must conform to the following PCRE regular expression: {0,63} No more than 32 labels can be associated with a given cluster.

PigJob WorkflowTemplateJobPigJob

Optional. Job is a Pig job.

PrerequisiteStepIds []string

Optional. The optional list of prerequisite job step_ids. If not specified, the job will start at the beginning of workflow.

PrestoJob WorkflowTemplateJobPrestoJob

Optional. Job is a Presto job.

PysparkJob WorkflowTemplateJobPysparkJob

Optional. Job is a PySpark job.

Scheduling WorkflowTemplateJobScheduling

Optional. Job scheduling configuration.

SparkJob WorkflowTemplateJobSparkJob

Optional. Job is a Spark job.

SparkRJob WorkflowTemplateJobSparkRJob

Optional. Job is a SparkR job.

SparkSqlJob WorkflowTemplateJobSparkSqlJob

Optional. Job is a SparkSql job.

stepId String

Required. The step id. The id must be unique among all jobs within the template. The step id is used as prefix for job id, as job goog-dataproc-workflow-step-id label, and in field from other steps. The id must contain only letters (a-z, A-Z), numbers (0-9), underscores (_), and hyphens (-). Cannot begin or end with underscore or hyphen. Must consist of between 3 and 50 characters.

hadoopJob WorkflowTemplateJobHadoopJob

Optional. Job is a Hadoop job.

hiveJob WorkflowTemplateJobHiveJob

Optional. Job is a Hive job.

labels Map<String,String>

Optional. The labels to associate with this cluster. Label keys must be between 1 and 63 characters long, and must conform to the following PCRE regular expression: {0,63} No more than 32 labels can be associated with a given cluster.

pigJob WorkflowTemplateJobPigJob

Optional. Job is a Pig job.

prerequisiteStepIds List<String>

Optional. The optional list of prerequisite job step_ids. If not specified, the job will start at the beginning of workflow.

prestoJob WorkflowTemplateJobPrestoJob

Optional. Job is a Presto job.

pysparkJob WorkflowTemplateJobPysparkJob

Optional. Job is a PySpark job.

scheduling WorkflowTemplateJobScheduling

Optional. Job scheduling configuration.

sparkJob WorkflowTemplateJobSparkJob

Optional. Job is a Spark job.

sparkRJob WorkflowTemplateJobSparkRJob

Optional. Job is a SparkR job.

sparkSqlJob WorkflowTemplateJobSparkSqlJob

Optional. Job is a SparkSql job.

stepId string

Required. The step id. The id must be unique among all jobs within the template. The step id is used as prefix for job id, as job goog-dataproc-workflow-step-id label, and in field from other steps. The id must contain only letters (a-z, A-Z), numbers (0-9), underscores (_), and hyphens (-). Cannot begin or end with underscore or hyphen. Must consist of between 3 and 50 characters.

hadoopJob WorkflowTemplateJobHadoopJob

Optional. Job is a Hadoop job.

hiveJob WorkflowTemplateJobHiveJob

Optional. Job is a Hive job.

labels {[key: string]: string}

Optional. The labels to associate with this cluster. Label keys must be between 1 and 63 characters long, and must conform to the following PCRE regular expression: {0,63} No more than 32 labels can be associated with a given cluster.

pigJob WorkflowTemplateJobPigJob

Optional. Job is a Pig job.

prerequisiteStepIds string[]

Optional. The optional list of prerequisite job step_ids. If not specified, the job will start at the beginning of workflow.

prestoJob WorkflowTemplateJobPrestoJob

Optional. Job is a Presto job.

pysparkJob WorkflowTemplateJobPysparkJob

Optional. Job is a PySpark job.

scheduling WorkflowTemplateJobScheduling

Optional. Job scheduling configuration.

sparkJob WorkflowTemplateJobSparkJob

Optional. Job is a Spark job.

sparkRJob WorkflowTemplateJobSparkRJob

Optional. Job is a SparkR job.

sparkSqlJob WorkflowTemplateJobSparkSqlJob

Optional. Job is a SparkSql job.

step_id str

Required. The step id. The id must be unique among all jobs within the template. The step id is used as prefix for job id, as job goog-dataproc-workflow-step-id label, and in field from other steps. The id must contain only letters (a-z, A-Z), numbers (0-9), underscores (_), and hyphens (-). Cannot begin or end with underscore or hyphen. Must consist of between 3 and 50 characters.

hadoop_job WorkflowTemplateJobHadoopJob

Optional. Job is a Hadoop job.

hive_job WorkflowTemplateJobHiveJob

Optional. Job is a Hive job.

labels Mapping[str, str]

Optional. The labels to associate with this cluster. Label keys must be between 1 and 63 characters long, and must conform to the following PCRE regular expression: {0,63} No more than 32 labels can be associated with a given cluster.

pig_job WorkflowTemplateJobPigJob

Optional. Job is a Pig job.

prerequisite_step_ids Sequence[str]

Optional. The optional list of prerequisite job step_ids. If not specified, the job will start at the beginning of workflow.

presto_job WorkflowTemplateJobPrestoJob

Optional. Job is a Presto job.

pyspark_job WorkflowTemplateJobPysparkJob

Optional. Job is a PySpark job.

scheduling WorkflowTemplateJobScheduling

Optional. Job scheduling configuration.

spark_job WorkflowTemplateJobSparkJob

Optional. Job is a Spark job.

spark_r_job WorkflowTemplateJobSparkRJob

Optional. Job is a SparkR job.

spark_sql_job WorkflowTemplateJobSparkSqlJob

Optional. Job is a SparkSql job.

stepId String

Required. The step id. The id must be unique among all jobs within the template. The step id is used as prefix for job id, as job goog-dataproc-workflow-step-id label, and in field from other steps. The id must contain only letters (a-z, A-Z), numbers (0-9), underscores (_), and hyphens (-). Cannot begin or end with underscore or hyphen. Must consist of between 3 and 50 characters.

hadoopJob Property Map

Optional. Job is a Hadoop job.

hiveJob Property Map

Optional. Job is a Hive job.

labels Map<String>

Optional. The labels to associate with this cluster. Label keys must be between 1 and 63 characters long, and must conform to the following PCRE regular expression: {0,63} No more than 32 labels can be associated with a given cluster.

pigJob Property Map

Optional. Job is a Pig job.

prerequisiteStepIds List<String>

Optional. The optional list of prerequisite job step_ids. If not specified, the job will start at the beginning of workflow.

prestoJob Property Map

Optional. Job is a Presto job.

pysparkJob Property Map

Optional. Job is a PySpark job.

scheduling Property Map

Optional. Job scheduling configuration.

sparkJob Property Map

Optional. Job is a Spark job.

sparkRJob Property Map

Optional. Job is a SparkR job.

sparkSqlJob Property Map

Optional. Job is a SparkSql job.

WorkflowTemplateJobHadoopJob

ArchiveUris List<string>

Optional. HCFS URIs of archives to be extracted into the working directory of each executor. Supported file types: .jar, .tar, .tar.gz, .tgz, and .zip.

Args List<string>

Optional. The arguments to pass to the driver. Do not include arguments, such as --conf, that can be set as job properties, since a collision may occur that causes an incorrect job submission.

FileUris List<string>

Optional. HCFS URIs of files to be placed in the working directory of each executor. Useful for naively parallel tasks.

JarFileUris List<string>

Optional. HCFS URIs of jar files to be added to the Spark CLASSPATH.

LoggingConfig WorkflowTemplateJobHadoopJobLoggingConfig

Optional. The runtime log config for job execution.

MainClass string

The name of the driver's main class. The jar file that contains the class must be in the default CLASSPATH or specified in jar_file_uris.

MainJarFileUri string

The HCFS URI of the jar file that contains the main class.

Properties Dictionary<string, string>

Optional. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. The following are supported prefixes and their mappings: * capacity-scheduler: capacity-scheduler.xml * core: core-site.xml * distcp: distcp-default.xml * hdfs: hdfs-site.xml * hive: hive-site.xml * mapred: mapred-site.xml * pig: pig.properties * spark: spark-defaults.conf * yarn: yarn-site.xml For more information, see (https://cloud.google.com/dataproc/docs/concepts/cluster-properties).

ArchiveUris []string

Optional. HCFS URIs of archives to be extracted into the working directory of each executor. Supported file types: .jar, .tar, .tar.gz, .tgz, and .zip.

Args []string

Optional. The arguments to pass to the driver. Do not include arguments, such as --conf, that can be set as job properties, since a collision may occur that causes an incorrect job submission.

FileUris []string

Optional. HCFS URIs of files to be placed in the working directory of each executor. Useful for naively parallel tasks.

JarFileUris []string

Optional. HCFS URIs of jar files to be added to the Spark CLASSPATH.

LoggingConfig WorkflowTemplateJobHadoopJobLoggingConfig

Optional. The runtime log config for job execution.

MainClass string

The name of the driver's main class. The jar file that contains the class must be in the default CLASSPATH or specified in jar_file_uris.

MainJarFileUri string

The HCFS URI of the jar file that contains the main class.

Properties map[string]string

Optional. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. The following are supported prefixes and their mappings: * capacity-scheduler: capacity-scheduler.xml * core: core-site.xml * distcp: distcp-default.xml * hdfs: hdfs-site.xml * hive: hive-site.xml * mapred: mapred-site.xml * pig: pig.properties * spark: spark-defaults.conf * yarn: yarn-site.xml For more information, see (https://cloud.google.com/dataproc/docs/concepts/cluster-properties).

archiveUris List<String>

Optional. HCFS URIs of archives to be extracted into the working directory of each executor. Supported file types: .jar, .tar, .tar.gz, .tgz, and .zip.

args List<String>

Optional. The arguments to pass to the driver. Do not include arguments, such as --conf, that can be set as job properties, since a collision may occur that causes an incorrect job submission.

fileUris List<String>

Optional. HCFS URIs of files to be placed in the working directory of each executor. Useful for naively parallel tasks.

jarFileUris List<String>

Optional. HCFS URIs of jar files to be added to the Spark CLASSPATH.

loggingConfig WorkflowTemplateJobHadoopJobLoggingConfig

Optional. The runtime log config for job execution.

mainClass String

The name of the driver's main class. The jar file that contains the class must be in the default CLASSPATH or specified in jar_file_uris.

mainJarFileUri String

The HCFS URI of the jar file that contains the main class.

properties Map<String,String>

Optional. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. The following are supported prefixes and their mappings: * capacity-scheduler: capacity-scheduler.xml * core: core-site.xml * distcp: distcp-default.xml * hdfs: hdfs-site.xml * hive: hive-site.xml * mapred: mapred-site.xml * pig: pig.properties * spark: spark-defaults.conf * yarn: yarn-site.xml For more information, see (https://cloud.google.com/dataproc/docs/concepts/cluster-properties).

archiveUris string[]

Optional. HCFS URIs of archives to be extracted into the working directory of each executor. Supported file types: .jar, .tar, .tar.gz, .tgz, and .zip.

args string[]

Optional. The arguments to pass to the driver. Do not include arguments, such as --conf, that can be set as job properties, since a collision may occur that causes an incorrect job submission.

fileUris string[]

Optional. HCFS URIs of files to be placed in the working directory of each executor. Useful for naively parallel tasks.

jarFileUris string[]

Optional. HCFS URIs of jar files to be added to the Spark CLASSPATH.

loggingConfig WorkflowTemplateJobHadoopJobLoggingConfig

Optional. The runtime log config for job execution.

mainClass string

The name of the driver's main class. The jar file that contains the class must be in the default CLASSPATH or specified in jar_file_uris.

mainJarFileUri string

The HCFS URI of the jar file that contains the main class.

properties {[key: string]: string}

Optional. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. The following are supported prefixes and their mappings: * capacity-scheduler: capacity-scheduler.xml * core: core-site.xml * distcp: distcp-default.xml * hdfs: hdfs-site.xml * hive: hive-site.xml * mapred: mapred-site.xml * pig: pig.properties * spark: spark-defaults.conf * yarn: yarn-site.xml For more information, see (https://cloud.google.com/dataproc/docs/concepts/cluster-properties).

archive_uris Sequence[str]

Optional. HCFS URIs of archives to be extracted into the working directory of each executor. Supported file types: .jar, .tar, .tar.gz, .tgz, and .zip.

args Sequence[str]

Optional. The arguments to pass to the driver. Do not include arguments, such as --conf, that can be set as job properties, since a collision may occur that causes an incorrect job submission.

file_uris Sequence[str]

Optional. HCFS URIs of files to be placed in the working directory of each executor. Useful for naively parallel tasks.

jar_file_uris Sequence[str]

Optional. HCFS URIs of jar files to be added to the Spark CLASSPATH.

logging_config WorkflowTemplateJobHadoopJobLoggingConfig

Optional. The runtime log config for job execution.

main_class str

The name of the driver's main class. The jar file that contains the class must be in the default CLASSPATH or specified in jar_file_uris.

main_jar_file_uri str

The HCFS URI of the jar file that contains the main class.

properties Mapping[str, str]

Optional. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. The following are supported prefixes and their mappings: * capacity-scheduler: capacity-scheduler.xml * core: core-site.xml * distcp: distcp-default.xml * hdfs: hdfs-site.xml * hive: hive-site.xml * mapred: mapred-site.xml * pig: pig.properties * spark: spark-defaults.conf * yarn: yarn-site.xml For more information, see (https://cloud.google.com/dataproc/docs/concepts/cluster-properties).

archiveUris List<String>

Optional. HCFS URIs of archives to be extracted into the working directory of each executor. Supported file types: .jar, .tar, .tar.gz, .tgz, and .zip.

args List<String>

Optional. The arguments to pass to the driver. Do not include arguments, such as --conf, that can be set as job properties, since a collision may occur that causes an incorrect job submission.

fileUris List<String>

Optional. HCFS URIs of files to be placed in the working directory of each executor. Useful for naively parallel tasks.

jarFileUris List<String>

Optional. HCFS URIs of jar files to be added to the Spark CLASSPATH.

loggingConfig Property Map

Optional. The runtime log config for job execution.

mainClass String

The name of the driver's main class. The jar file that contains the class must be in the default CLASSPATH or specified in jar_file_uris.

mainJarFileUri String

The HCFS URI of the jar file that contains the main class.

properties Map<String>

Optional. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. The following are supported prefixes and their mappings: * capacity-scheduler: capacity-scheduler.xml * core: core-site.xml * distcp: distcp-default.xml * hdfs: hdfs-site.xml * hive: hive-site.xml * mapred: mapred-site.xml * pig: pig.properties * spark: spark-defaults.conf * yarn: yarn-site.xml For more information, see (https://cloud.google.com/dataproc/docs/concepts/cluster-properties).

WorkflowTemplateJobHadoopJobLoggingConfig

DriverLogLevels Dictionary<string, string>

The per-package log levels for the driver. This may include "root" package name to configure rootLogger. Examples: 'com.google = FATAL', 'root = INFO', 'org.apache = DEBUG'

DriverLogLevels map[string]string

The per-package log levels for the driver. This may include "root" package name to configure rootLogger. Examples: 'com.google = FATAL', 'root = INFO', 'org.apache = DEBUG'

driverLogLevels Map<String,String>

The per-package log levels for the driver. This may include "root" package name to configure rootLogger. Examples: 'com.google = FATAL', 'root = INFO', 'org.apache = DEBUG'

driverLogLevels {[key: string]: string}

The per-package log levels for the driver. This may include "root" package name to configure rootLogger. Examples: 'com.google = FATAL', 'root = INFO', 'org.apache = DEBUG'

driver_log_levels Mapping[str, str]

The per-package log levels for the driver. This may include "root" package name to configure rootLogger. Examples: 'com.google = FATAL', 'root = INFO', 'org.apache = DEBUG'

driverLogLevels Map<String>

The per-package log levels for the driver. This may include "root" package name to configure rootLogger. Examples: 'com.google = FATAL', 'root = INFO', 'org.apache = DEBUG'

WorkflowTemplateJobHiveJob

ContinueOnFailure bool

Optional. Whether to continue executing queries if a query fails. The default value is false. Setting to true can be useful when executing independent parallel queries.

JarFileUris List<string>

Optional. HCFS URIs of jar files to be added to the Spark CLASSPATH.

Properties Dictionary<string, string>

Optional. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. The following are supported prefixes and their mappings: * capacity-scheduler: capacity-scheduler.xml * core: core-site.xml * distcp: distcp-default.xml * hdfs: hdfs-site.xml * hive: hive-site.xml * mapred: mapred-site.xml * pig: pig.properties * spark: spark-defaults.conf * yarn: yarn-site.xml For more information, see (https://cloud.google.com/dataproc/docs/concepts/cluster-properties).

QueryFileUri string

The HCFS URI of the script that contains SQL queries.

QueryList WorkflowTemplateJobHiveJobQueryList

A list of queries.

ScriptVariables Dictionary<string, string>

Optional. Mapping of query variable names to values (equivalent to the Spark SQL command: SET name="value";).

ContinueOnFailure bool

Optional. Whether to continue executing queries if a query fails. The default value is false. Setting to true can be useful when executing independent parallel queries.

JarFileUris []string

Optional. HCFS URIs of jar files to be added to the Spark CLASSPATH.

Properties map[string]string

Optional. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. The following are supported prefixes and their mappings: * capacity-scheduler: capacity-scheduler.xml * core: core-site.xml * distcp: distcp-default.xml * hdfs: hdfs-site.xml * hive: hive-site.xml * mapred: mapred-site.xml * pig: pig.properties * spark: spark-defaults.conf * yarn: yarn-site.xml For more information, see (https://cloud.google.com/dataproc/docs/concepts/cluster-properties).

QueryFileUri string

The HCFS URI of the script that contains SQL queries.

QueryList WorkflowTemplateJobHiveJobQueryList

A list of queries.

ScriptVariables map[string]string

Optional. Mapping of query variable names to values (equivalent to the Spark SQL command: SET name="value";).

continueOnFailure Boolean

Optional. Whether to continue executing queries if a query fails. The default value is false. Setting to true can be useful when executing independent parallel queries.

jarFileUris List<String>

Optional. HCFS URIs of jar files to be added to the Spark CLASSPATH.

properties Map<String,String>

Optional. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. The following are supported prefixes and their mappings: * capacity-scheduler: capacity-scheduler.xml * core: core-site.xml * distcp: distcp-default.xml * hdfs: hdfs-site.xml * hive: hive-site.xml * mapred: mapred-site.xml * pig: pig.properties * spark: spark-defaults.conf * yarn: yarn-site.xml For more information, see (https://cloud.google.com/dataproc/docs/concepts/cluster-properties).

queryFileUri String

The HCFS URI of the script that contains SQL queries.

queryList WorkflowTemplateJobHiveJobQueryList

A list of queries.

scriptVariables Map<String,String>

Optional. Mapping of query variable names to values (equivalent to the Spark SQL command: SET name="value";).

continueOnFailure boolean

Optional. Whether to continue executing queries if a query fails. The default value is false. Setting to true can be useful when executing independent parallel queries.

jarFileUris string[]

Optional. HCFS URIs of jar files to be added to the Spark CLASSPATH.

properties {[key: string]: string}

Optional. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. The following are supported prefixes and their mappings: * capacity-scheduler: capacity-scheduler.xml * core: core-site.xml * distcp: distcp-default.xml * hdfs: hdfs-site.xml * hive: hive-site.xml * mapred: mapred-site.xml * pig: pig.properties * spark: spark-defaults.conf * yarn: yarn-site.xml For more information, see (https://cloud.google.com/dataproc/docs/concepts/cluster-properties).

queryFileUri string

The HCFS URI of the script that contains SQL queries.

queryList WorkflowTemplateJobHiveJobQueryList

A list of queries.

scriptVariables {[key: string]: string}

Optional. Mapping of query variable names to values (equivalent to the Spark SQL command: SET name="value";).

continue_on_failure bool

Optional. Whether to continue executing queries if a query fails. The default value is false. Setting to true can be useful when executing independent parallel queries.

jar_file_uris Sequence[str]

Optional. HCFS URIs of jar files to be added to the Spark CLASSPATH.

properties Mapping[str, str]

Optional. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. The following are supported prefixes and their mappings: * capacity-scheduler: capacity-scheduler.xml * core: core-site.xml * distcp: distcp-default.xml * hdfs: hdfs-site.xml * hive: hive-site.xml * mapred: mapred-site.xml * pig: pig.properties * spark: spark-defaults.conf * yarn: yarn-site.xml For more information, see (https://cloud.google.com/dataproc/docs/concepts/cluster-properties).

query_file_uri str

The HCFS URI of the script that contains SQL queries.

query_list WorkflowTemplateJobHiveJobQueryList

A list of queries.

script_variables Mapping[str, str]

Optional. Mapping of query variable names to values (equivalent to the Spark SQL command: SET name="value";).

continueOnFailure Boolean

Optional. Whether to continue executing queries if a query fails. The default value is false. Setting to true can be useful when executing independent parallel queries.

jarFileUris List<String>

Optional. HCFS URIs of jar files to be added to the Spark CLASSPATH.

properties Map<String>

Optional. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. The following are supported prefixes and their mappings: * capacity-scheduler: capacity-scheduler.xml * core: core-site.xml * distcp: distcp-default.xml * hdfs: hdfs-site.xml * hive: hive-site.xml * mapred: mapred-site.xml * pig: pig.properties * spark: spark-defaults.conf * yarn: yarn-site.xml For more information, see (https://cloud.google.com/dataproc/docs/concepts/cluster-properties).

queryFileUri String

The HCFS URI of the script that contains SQL queries.

queryList Property Map

A list of queries.

scriptVariables Map<String>

Optional. Mapping of query variable names to values (equivalent to the Spark SQL command: SET name="value";).

WorkflowTemplateJobHiveJobQueryList

Queries List<string>

Required. The queries to execute. You do not need to end a query expression with a semicolon. Multiple queries can be specified in one string by separating each with a semicolon. Here is an example of a Dataproc API snippet that uses a QueryList to specify a HiveJob: "hiveJob": { "queryList": { "queries": } }

Queries []string

Required. The queries to execute. You do not need to end a query expression with a semicolon. Multiple queries can be specified in one string by separating each with a semicolon. Here is an example of a Dataproc API snippet that uses a QueryList to specify a HiveJob: "hiveJob": { "queryList": { "queries": } }

queries List<String>

Required. The queries to execute. You do not need to end a query expression with a semicolon. Multiple queries can be specified in one string by separating each with a semicolon. Here is an example of a Dataproc API snippet that uses a QueryList to specify a HiveJob: "hiveJob": { "queryList": { "queries": } }

queries string[]

Required. The queries to execute. You do not need to end a query expression with a semicolon. Multiple queries can be specified in one string by separating each with a semicolon. Here is an example of a Dataproc API snippet that uses a QueryList to specify a HiveJob: "hiveJob": { "queryList": { "queries": } }

queries Sequence[str]

Required. The queries to execute. You do not need to end a query expression with a semicolon. Multiple queries can be specified in one string by separating each with a semicolon. Here is an example of a Dataproc API snippet that uses a QueryList to specify a HiveJob: "hiveJob": { "queryList": { "queries": } }

queries List<String>

Required. The queries to execute. You do not need to end a query expression with a semicolon. Multiple queries can be specified in one string by separating each with a semicolon. Here is an example of a Dataproc API snippet that uses a QueryList to specify a HiveJob: "hiveJob": { "queryList": { "queries": } }

WorkflowTemplateJobPigJob

ContinueOnFailure bool

Optional. Whether to continue executing queries if a query fails. The default value is false. Setting to true can be useful when executing independent parallel queries.

JarFileUris List<string>

Optional. HCFS URIs of jar files to be added to the Spark CLASSPATH.

LoggingConfig WorkflowTemplateJobPigJobLoggingConfig

Optional. The runtime log config for job execution.

Properties Dictionary<string, string>

Optional. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. The following are supported prefixes and their mappings: * capacity-scheduler: capacity-scheduler.xml * core: core-site.xml * distcp: distcp-default.xml * hdfs: hdfs-site.xml * hive: hive-site.xml * mapred: mapred-site.xml * pig: pig.properties * spark: spark-defaults.conf * yarn: yarn-site.xml For more information, see (https://cloud.google.com/dataproc/docs/concepts/cluster-properties).

QueryFileUri string

The HCFS URI of the script that contains SQL queries.

QueryList WorkflowTemplateJobPigJobQueryList

A list of queries.

ScriptVariables Dictionary<string, string>

Optional. Mapping of query variable names to values (equivalent to the Spark SQL command: SET name="value";).

ContinueOnFailure bool

Optional. Whether to continue executing queries if a query fails. The default value is false. Setting to true can be useful when executing independent parallel queries.

JarFileUris []string

Optional. HCFS URIs of jar files to be added to the Spark CLASSPATH.

LoggingConfig WorkflowTemplateJobPigJobLoggingConfig

Optional. The runtime log config for job execution.

Properties map[string]string

Optional. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. The following are supported prefixes and their mappings: * capacity-scheduler: capacity-scheduler.xml * core: core-site.xml * distcp: distcp-default.xml * hdfs: hdfs-site.xml * hive: hive-site.xml * mapred: mapred-site.xml * pig: pig.properties * spark: spark-defaults.conf * yarn: yarn-site.xml For more information, see (https://cloud.google.com/dataproc/docs/concepts/cluster-properties).

QueryFileUri string

The HCFS URI of the script that contains SQL queries.

QueryList WorkflowTemplateJobPigJobQueryList

A list of queries.

ScriptVariables map[string]string

Optional. Mapping of query variable names to values (equivalent to the Spark SQL command: SET name="value";).

continueOnFailure Boolean

Optional. Whether to continue executing queries if a query fails. The default value is false. Setting to true can be useful when executing independent parallel queries.

jarFileUris List<String>

Optional. HCFS URIs of jar files to be added to the Spark CLASSPATH.

loggingConfig WorkflowTemplateJobPigJobLoggingConfig

Optional. The runtime log config for job execution.

properties Map<String,String>

Optional. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. The following are supported prefixes and their mappings: * capacity-scheduler: capacity-scheduler.xml * core: core-site.xml * distcp: distcp-default.xml * hdfs: hdfs-site.xml * hive: hive-site.xml * mapred: mapred-site.xml * pig: pig.properties * spark: spark-defaults.conf * yarn: yarn-site.xml For more information, see (https://cloud.google.com/dataproc/docs/concepts/cluster-properties).

queryFileUri String

The HCFS URI of the script that contains SQL queries.

queryList WorkflowTemplateJobPigJobQueryList

A list of queries.

scriptVariables Map<String,String>

Optional. Mapping of query variable names to values (equivalent to the Spark SQL command: SET name="value";).

continueOnFailure boolean

Optional. Whether to continue executing queries if a query fails. The default value is false. Setting to true can be useful when executing independent parallel queries.

jarFileUris string[]

Optional. HCFS URIs of jar files to be added to the Spark CLASSPATH.

loggingConfig WorkflowTemplateJobPigJobLoggingConfig

Optional. The runtime log config for job execution.

properties {[key: string]: string}

Optional. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. The following are supported prefixes and their mappings: * capacity-scheduler: capacity-scheduler.xml * core: core-site.xml * distcp: distcp-default.xml * hdfs: hdfs-site.xml * hive: hive-site.xml * mapred: mapred-site.xml * pig: pig.properties * spark: spark-defaults.conf * yarn: yarn-site.xml For more information, see (https://cloud.google.com/dataproc/docs/concepts/cluster-properties).

queryFileUri string

The HCFS URI of the script that contains SQL queries.

queryList WorkflowTemplateJobPigJobQueryList

A list of queries.

scriptVariables {[key: string]: string}

Optional. Mapping of query variable names to values (equivalent to the Spark SQL command: SET name="value";).

continue_on_failure bool

Optional. Whether to continue executing queries if a query fails. The default value is false. Setting to true can be useful when executing independent parallel queries.

jar_file_uris Sequence[str]

Optional. HCFS URIs of jar files to be added to the Spark CLASSPATH.

logging_config WorkflowTemplateJobPigJobLoggingConfig

Optional. The runtime log config for job execution.

properties Mapping[str, str]

Optional. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. The following are supported prefixes and their mappings: * capacity-scheduler: capacity-scheduler.xml * core: core-site.xml * distcp: distcp-default.xml * hdfs: hdfs-site.xml * hive: hive-site.xml * mapred: mapred-site.xml * pig: pig.properties * spark: spark-defaults.conf * yarn: yarn-site.xml For more information, see (https://cloud.google.com/dataproc/docs/concepts/cluster-properties).

query_file_uri str

The HCFS URI of the script that contains SQL queries.

query_list WorkflowTemplateJobPigJobQueryList

A list of queries.

script_variables Mapping[str, str]

Optional. Mapping of query variable names to values (equivalent to the Spark SQL command: SET name="value";).

continueOnFailure Boolean

Optional. Whether to continue executing queries if a query fails. The default value is false. Setting to true can be useful when executing independent parallel queries.

jarFileUris List<String>

Optional. HCFS URIs of jar files to be added to the Spark CLASSPATH.

loggingConfig Property Map

Optional. The runtime log config for job execution.

properties Map<String>

Optional. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. The following are supported prefixes and their mappings: * capacity-scheduler: capacity-scheduler.xml * core: core-site.xml * distcp: distcp-default.xml * hdfs: hdfs-site.xml * hive: hive-site.xml * mapred: mapred-site.xml * pig: pig.properties * spark: spark-defaults.conf * yarn: yarn-site.xml For more information, see (https://cloud.google.com/dataproc/docs/concepts/cluster-properties).

queryFileUri String

The HCFS URI of the script that contains SQL queries.

queryList Property Map

A list of queries.

scriptVariables Map<String>

Optional. Mapping of query variable names to values (equivalent to the Spark SQL command: SET name="value";).

WorkflowTemplateJobPigJobLoggingConfig

DriverLogLevels Dictionary<string, string>

The per-package log levels for the driver. This may include "root" package name to configure rootLogger. Examples: 'com.google = FATAL', 'root = INFO', 'org.apache = DEBUG'

DriverLogLevels map[string]string

The per-package log levels for the driver. This may include "root" package name to configure rootLogger. Examples: 'com.google = FATAL', 'root = INFO', 'org.apache = DEBUG'

driverLogLevels Map<String,String>

The per-package log levels for the driver. This may include "root" package name to configure rootLogger. Examples: 'com.google = FATAL', 'root = INFO', 'org.apache = DEBUG'

driverLogLevels {[key: string]: string}

The per-package log levels for the driver. This may include "root" package name to configure rootLogger. Examples: 'com.google = FATAL', 'root = INFO', 'org.apache = DEBUG'

driver_log_levels Mapping[str, str]

The per-package log levels for the driver. This may include "root" package name to configure rootLogger. Examples: 'com.google = FATAL', 'root = INFO', 'org.apache = DEBUG'

driverLogLevels Map<String>

The per-package log levels for the driver. This may include "root" package name to configure rootLogger. Examples: 'com.google = FATAL', 'root = INFO', 'org.apache = DEBUG'

WorkflowTemplateJobPigJobQueryList

Queries List<string>

Required. The queries to execute. You do not need to end a query expression with a semicolon. Multiple queries can be specified in one string by separating each with a semicolon. Here is an example of a Dataproc API snippet that uses a QueryList to specify a HiveJob: "hiveJob": { "queryList": { "queries": } }

Queries []string

Required. The queries to execute. You do not need to end a query expression with a semicolon. Multiple queries can be specified in one string by separating each with a semicolon. Here is an example of a Dataproc API snippet that uses a QueryList to specify a HiveJob: "hiveJob": { "queryList": { "queries": } }

queries List<String>

Required. The queries to execute. You do not need to end a query expression with a semicolon. Multiple queries can be specified in one string by separating each with a semicolon. Here is an example of a Dataproc API snippet that uses a QueryList to specify a HiveJob: "hiveJob": { "queryList": { "queries": } }

queries string[]

Required. The queries to execute. You do not need to end a query expression with a semicolon. Multiple queries can be specified in one string by separating each with a semicolon. Here is an example of a Dataproc API snippet that uses a QueryList to specify a HiveJob: "hiveJob": { "queryList": { "queries": } }

queries Sequence[str]

Required. The queries to execute. You do not need to end a query expression with a semicolon. Multiple queries can be specified in one string by separating each with a semicolon. Here is an example of a Dataproc API snippet that uses a QueryList to specify a HiveJob: "hiveJob": { "queryList": { "queries": } }

queries List<String>

Required. The queries to execute. You do not need to end a query expression with a semicolon. Multiple queries can be specified in one string by separating each with a semicolon. Here is an example of a Dataproc API snippet that uses a QueryList to specify a HiveJob: "hiveJob": { "queryList": { "queries": } }

WorkflowTemplateJobPrestoJob

ClientTags List<string>

Optional. Presto client tags to attach to this query

ContinueOnFailure bool

Optional. Whether to continue executing queries if a query fails. The default value is false. Setting to true can be useful when executing independent parallel queries.

LoggingConfig WorkflowTemplateJobPrestoJobLoggingConfig

Optional. The runtime log config for job execution.

OutputFormat string

Optional. The format in which query output will be displayed. See the Presto documentation for supported output formats

Properties Dictionary<string, string>

Optional. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. The following are supported prefixes and their mappings: * capacity-scheduler: capacity-scheduler.xml * core: core-site.xml * distcp: distcp-default.xml * hdfs: hdfs-site.xml * hive: hive-site.xml * mapred: mapred-site.xml * pig: pig.properties * spark: spark-defaults.conf * yarn: yarn-site.xml For more information, see (https://cloud.google.com/dataproc/docs/concepts/cluster-properties).

QueryFileUri string

The HCFS URI of the script that contains SQL queries.

QueryList WorkflowTemplateJobPrestoJobQueryList

A list of queries.

ClientTags []string

Optional. Presto client tags to attach to this query

ContinueOnFailure bool

Optional. Whether to continue executing queries if a query fails. The default value is false. Setting to true can be useful when executing independent parallel queries.

LoggingConfig WorkflowTemplateJobPrestoJobLoggingConfig

Optional. The runtime log config for job execution.

OutputFormat string

Optional. The format in which query output will be displayed. See the Presto documentation for supported output formats

Properties map[string]string

Optional. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. The following are supported prefixes and their mappings: * capacity-scheduler: capacity-scheduler.xml * core: core-site.xml * distcp: distcp-default.xml * hdfs: hdfs-site.xml * hive: hive-site.xml * mapred: mapred-site.xml * pig: pig.properties * spark: spark-defaults.conf * yarn: yarn-site.xml For more information, see (https://cloud.google.com/dataproc/docs/concepts/cluster-properties).

QueryFileUri string

The HCFS URI of the script that contains SQL queries.

QueryList WorkflowTemplateJobPrestoJobQueryList

A list of queries.

clientTags List<String>

Optional. Presto client tags to attach to this query

continueOnFailure Boolean

Optional. Whether to continue executing queries if a query fails. The default value is false. Setting to true can be useful when executing independent parallel queries.

loggingConfig WorkflowTemplateJobPrestoJobLoggingConfig

Optional. The runtime log config for job execution.

outputFormat String

Optional. The format in which query output will be displayed. See the Presto documentation for supported output formats

properties Map<String,String>

Optional. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. The following are supported prefixes and their mappings: * capacity-scheduler: capacity-scheduler.xml * core: core-site.xml * distcp: distcp-default.xml * hdfs: hdfs-site.xml * hive: hive-site.xml * mapred: mapred-site.xml * pig: pig.properties * spark: spark-defaults.conf * yarn: yarn-site.xml For more information, see (https://cloud.google.com/dataproc/docs/concepts/cluster-properties).

queryFileUri String

The HCFS URI of the script that contains SQL queries.

queryList WorkflowTemplateJobPrestoJobQueryList

A list of queries.

clientTags string[]

Optional. Presto client tags to attach to this query

continueOnFailure boolean

Optional. Whether to continue executing queries if a query fails. The default value is false. Setting to true can be useful when executing independent parallel queries.

loggingConfig WorkflowTemplateJobPrestoJobLoggingConfig

Optional. The runtime log config for job execution.

outputFormat string

Optional. The format in which query output will be displayed. See the Presto documentation for supported output formats

properties {[key: string]: string}

Optional. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. The following are supported prefixes and their mappings: * capacity-scheduler: capacity-scheduler.xml * core: core-site.xml * distcp: distcp-default.xml * hdfs: hdfs-site.xml * hive: hive-site.xml * mapred: mapred-site.xml * pig: pig.properties * spark: spark-defaults.conf * yarn: yarn-site.xml For more information, see (https://cloud.google.com/dataproc/docs/concepts/cluster-properties).

queryFileUri string

The HCFS URI of the script that contains SQL queries.

queryList WorkflowTemplateJobPrestoJobQueryList

A list of queries.

client_tags Sequence[str]

Optional. Presto client tags to attach to this query

continue_on_failure bool

Optional. Whether to continue executing queries if a query fails. The default value is false. Setting to true can be useful when executing independent parallel queries.

logging_config WorkflowTemplateJobPrestoJobLoggingConfig

Optional. The runtime log config for job execution.

output_format str

Optional. The format in which query output will be displayed. See the Presto documentation for supported output formats

properties Mapping[str, str]

Optional. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. The following are supported prefixes and their mappings: * capacity-scheduler: capacity-scheduler.xml * core: core-site.xml * distcp: distcp-default.xml * hdfs: hdfs-site.xml * hive: hive-site.xml * mapred: mapred-site.xml * pig: pig.properties * spark: spark-defaults.conf * yarn: yarn-site.xml For more information, see (https://cloud.google.com/dataproc/docs/concepts/cluster-properties).

query_file_uri str

The HCFS URI of the script that contains SQL queries.

query_list WorkflowTemplateJobPrestoJobQueryList

A list of queries.

clientTags List<String>

Optional. Presto client tags to attach to this query

continueOnFailure Boolean

Optional. Whether to continue executing queries if a query fails. The default value is false. Setting to true can be useful when executing independent parallel queries.

loggingConfig Property Map

Optional. The runtime log config for job execution.

outputFormat String

Optional. The format in which query output will be displayed. See the Presto documentation for supported output formats

properties Map<String>

Optional. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. The following are supported prefixes and their mappings: * capacity-scheduler: capacity-scheduler.xml * core: core-site.xml * distcp: distcp-default.xml * hdfs: hdfs-site.xml * hive: hive-site.xml * mapred: mapred-site.xml * pig: pig.properties * spark: spark-defaults.conf * yarn: yarn-site.xml For more information, see (https://cloud.google.com/dataproc/docs/concepts/cluster-properties).

queryFileUri String

The HCFS URI of the script that contains SQL queries.

queryList Property Map

A list of queries.

WorkflowTemplateJobPrestoJobLoggingConfig

DriverLogLevels Dictionary<string, string>

The per-package log levels for the driver. This may include "root" package name to configure rootLogger. Examples: 'com.google = FATAL', 'root = INFO', 'org.apache = DEBUG'

DriverLogLevels map[string]string

The per-package log levels for the driver. This may include "root" package name to configure rootLogger. Examples: 'com.google = FATAL', 'root = INFO', 'org.apache = DEBUG'

driverLogLevels Map<String,String>

The per-package log levels for the driver. This may include "root" package name to configure rootLogger. Examples: 'com.google = FATAL', 'root = INFO', 'org.apache = DEBUG'

driverLogLevels {[key: string]: string}

The per-package log levels for the driver. This may include "root" package name to configure rootLogger. Examples: 'com.google = FATAL', 'root = INFO', 'org.apache = DEBUG'

driver_log_levels Mapping[str, str]

The per-package log levels for the driver. This may include "root" package name to configure rootLogger. Examples: 'com.google = FATAL', 'root = INFO', 'org.apache = DEBUG'

driverLogLevels Map<String>

The per-package log levels for the driver. This may include "root" package name to configure rootLogger. Examples: 'com.google = FATAL', 'root = INFO', 'org.apache = DEBUG'

WorkflowTemplateJobPrestoJobQueryList

Queries List<string>

Required. The queries to execute. You do not need to end a query expression with a semicolon. Multiple queries can be specified in one string by separating each with a semicolon. Here is an example of a Dataproc API snippet that uses a QueryList to specify a HiveJob: "hiveJob": { "queryList": { "queries": } }

Queries []string

Required. The queries to execute. You do not need to end a query expression with a semicolon. Multiple queries can be specified in one string by separating each with a semicolon. Here is an example of a Dataproc API snippet that uses a QueryList to specify a HiveJob: "hiveJob": { "queryList": { "queries": } }

queries List<String>

Required. The queries to execute. You do not need to end a query expression with a semicolon. Multiple queries can be specified in one string by separating each with a semicolon. Here is an example of a Dataproc API snippet that uses a QueryList to specify a HiveJob: "hiveJob": { "queryList": { "queries": } }

queries string[]

Required. The queries to execute. You do not need to end a query expression with a semicolon. Multiple queries can be specified in one string by separating each with a semicolon. Here is an example of a Dataproc API snippet that uses a QueryList to specify a HiveJob: "hiveJob": { "queryList": { "queries": } }

queries Sequence[str]

Required. The queries to execute. You do not need to end a query expression with a semicolon. Multiple queries can be specified in one string by separating each with a semicolon. Here is an example of a Dataproc API snippet that uses a QueryList to specify a HiveJob: "hiveJob": { "queryList": { "queries": } }

queries List<String>

Required. The queries to execute. You do not need to end a query expression with a semicolon. Multiple queries can be specified in one string by separating each with a semicolon. Here is an example of a Dataproc API snippet that uses a QueryList to specify a HiveJob: "hiveJob": { "queryList": { "queries": } }

WorkflowTemplateJobPysparkJob

MainPythonFileUri string

Required. The HCFS URI of the main Python file to use as the driver. Must be a .py file.

ArchiveUris List<string>

Optional. HCFS URIs of archives to be extracted into the working directory of each executor. Supported file types: .jar, .tar, .tar.gz, .tgz, and .zip.

Args List<string>

Optional. The arguments to pass to the driver. Do not include arguments, such as --conf, that can be set as job properties, since a collision may occur that causes an incorrect job submission.

FileUris List<string>

Optional. HCFS URIs of files to be placed in the working directory of each executor. Useful for naively parallel tasks.

JarFileUris List<string>

Optional. HCFS URIs of jar files to be added to the Spark CLASSPATH.

LoggingConfig WorkflowTemplateJobPysparkJobLoggingConfig

Optional. The runtime log config for job execution.

Properties Dictionary<string, string>

Optional. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. The following are supported prefixes and their mappings: * capacity-scheduler: capacity-scheduler.xml * core: core-site.xml * distcp: distcp-default.xml * hdfs: hdfs-site.xml * hive: hive-site.xml * mapred: mapred-site.xml * pig: pig.properties * spark: spark-defaults.conf * yarn: yarn-site.xml For more information, see (https://cloud.google.com/dataproc/docs/concepts/cluster-properties).

PythonFileUris List<string>

Optional. HCFS file URIs of Python files to pass to the PySpark framework. Supported file types: .py, .egg, and .zip.

MainPythonFileUri string

Required. The HCFS URI of the main Python file to use as the driver. Must be a .py file.

ArchiveUris []string

Optional. HCFS URIs of archives to be extracted into the working directory of each executor. Supported file types: .jar, .tar, .tar.gz, .tgz, and .zip.

Args []string

Optional. The arguments to pass to the driver. Do not include arguments, such as --conf, that can be set as job properties, since a collision may occur that causes an incorrect job submission.

FileUris []string

Optional. HCFS URIs of files to be placed in the working directory of each executor. Useful for naively parallel tasks.

JarFileUris []string

Optional. HCFS URIs of jar files to be added to the Spark CLASSPATH.

LoggingConfig WorkflowTemplateJobPysparkJobLoggingConfig

Optional. The runtime log config for job execution.

Properties map[string]string

Optional. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. The following are supported prefixes and their mappings: * capacity-scheduler: capacity-scheduler.xml * core: core-site.xml * distcp: distcp-default.xml * hdfs: hdfs-site.xml * hive: hive-site.xml * mapred: mapred-site.xml * pig: pig.properties * spark: spark-defaults.conf * yarn: yarn-site.xml For more information, see (https://cloud.google.com/dataproc/docs/concepts/cluster-properties).

PythonFileUris []string

Optional. HCFS file URIs of Python files to pass to the PySpark framework. Supported file types: .py, .egg, and .zip.

mainPythonFileUri String

Required. The HCFS URI of the main Python file to use as the driver. Must be a .py file.

archiveUris List<String>

Optional. HCFS URIs of archives to be extracted into the working directory of each executor. Supported file types: .jar, .tar, .tar.gz, .tgz, and .zip.

args List<String>

Optional. The arguments to pass to the driver. Do not include arguments, such as --conf, that can be set as job properties, since a collision may occur that causes an incorrect job submission.

fileUris List<String>

Optional. HCFS URIs of files to be placed in the working directory of each executor. Useful for naively parallel tasks.

jarFileUris List<String>

Optional. HCFS URIs of jar files to be added to the Spark CLASSPATH.

loggingConfig WorkflowTemplateJobPysparkJobLoggingConfig

Optional. The runtime log config for job execution.

properties Map<String,String>

Optional. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. The following are supported prefixes and their mappings: * capacity-scheduler: capacity-scheduler.xml * core: core-site.xml * distcp: distcp-default.xml * hdfs: hdfs-site.xml * hive: hive-site.xml * mapred: mapred-site.xml * pig: pig.properties * spark: spark-defaults.conf * yarn: yarn-site.xml For more information, see (https://cloud.google.com/dataproc/docs/concepts/cluster-properties).

pythonFileUris List<String>

Optional. HCFS file URIs of Python files to pass to the PySpark framework. Supported file types: .py, .egg, and .zip.

mainPythonFileUri string

Required. The HCFS URI of the main Python file to use as the driver. Must be a .py file.

archiveUris string[]

Optional. HCFS URIs of archives to be extracted into the working directory of each executor. Supported file types: .jar, .tar, .tar.gz, .tgz, and .zip.

args string[]

Optional. The arguments to pass to the driver. Do not include arguments, such as --conf, that can be set as job properties, since a collision may occur that causes an incorrect job submission.

fileUris string[]

Optional. HCFS URIs of files to be placed in the working directory of each executor. Useful for naively parallel tasks.

jarFileUris string[]

Optional. HCFS URIs of jar files to be added to the Spark CLASSPATH.

loggingConfig WorkflowTemplateJobPysparkJobLoggingConfig

Optional. The runtime log config for job execution.

properties {[key: string]: string}

Optional. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. The following are supported prefixes and their mappings: * capacity-scheduler: capacity-scheduler.xml * core: core-site.xml * distcp: distcp-default.xml * hdfs: hdfs-site.xml * hive: hive-site.xml * mapred: mapred-site.xml * pig: pig.properties * spark: spark-defaults.conf * yarn: yarn-site.xml For more information, see (https://cloud.google.com/dataproc/docs/concepts/cluster-properties).

pythonFileUris string[]

Optional. HCFS file URIs of Python files to pass to the PySpark framework. Supported file types: .py, .egg, and .zip.

main_python_file_uri str

Required. The HCFS URI of the main Python file to use as the driver. Must be a .py file.

archive_uris Sequence[str]

Optional. HCFS URIs of archives to be extracted into the working directory of each executor. Supported file types: .jar, .tar, .tar.gz, .tgz, and .zip.

args Sequence[str]

Optional. The arguments to pass to the driver. Do not include arguments, such as --conf, that can be set as job properties, since a collision may occur that causes an incorrect job submission.

file_uris Sequence[str]

Optional. HCFS URIs of files to be placed in the working directory of each executor. Useful for naively parallel tasks.

jar_file_uris Sequence[str]

Optional. HCFS URIs of jar files to be added to the Spark CLASSPATH.

logging_config WorkflowTemplateJobPysparkJobLoggingConfig

Optional. The runtime log config for job execution.

properties Mapping[str, str]

Optional. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. The following are supported prefixes and their mappings: * capacity-scheduler: capacity-scheduler.xml * core: core-site.xml * distcp: distcp-default.xml * hdfs: hdfs-site.xml * hive: hive-site.xml * mapred: mapred-site.xml * pig: pig.properties * spark: spark-defaults.conf * yarn: yarn-site.xml For more information, see (https://cloud.google.com/dataproc/docs/concepts/cluster-properties).

python_file_uris Sequence[str]

Optional. HCFS file URIs of Python files to pass to the PySpark framework. Supported file types: .py, .egg, and .zip.

mainPythonFileUri String

Required. The HCFS URI of the main Python file to use as the driver. Must be a .py file.

archiveUris List<String>

Optional. HCFS URIs of archives to be extracted into the working directory of each executor. Supported file types: .jar, .tar, .tar.gz, .tgz, and .zip.

args List<String>

Optional. The arguments to pass to the driver. Do not include arguments, such as --conf, that can be set as job properties, since a collision may occur that causes an incorrect job submission.

fileUris List<String>

Optional. HCFS URIs of files to be placed in the working directory of each executor. Useful for naively parallel tasks.

jarFileUris List<String>

Optional. HCFS URIs of jar files to be added to the Spark CLASSPATH.

loggingConfig Property Map

Optional. The runtime log config for job execution.

properties Map<String>

Optional. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. The following are supported prefixes and their mappings: * capacity-scheduler: capacity-scheduler.xml * core: core-site.xml * distcp: distcp-default.xml * hdfs: hdfs-site.xml * hive: hive-site.xml * mapred: mapred-site.xml * pig: pig.properties * spark: spark-defaults.conf * yarn: yarn-site.xml For more information, see (https://cloud.google.com/dataproc/docs/concepts/cluster-properties).

pythonFileUris List<String>

Optional. HCFS file URIs of Python files to pass to the PySpark framework. Supported file types: .py, .egg, and .zip.

WorkflowTemplateJobPysparkJobLoggingConfig

DriverLogLevels Dictionary<string, string>

The per-package log levels for the driver. This may include "root" package name to configure rootLogger. Examples: 'com.google = FATAL', 'root = INFO', 'org.apache = DEBUG'

DriverLogLevels map[string]string

The per-package log levels for the driver. This may include "root" package name to configure rootLogger. Examples: 'com.google = FATAL', 'root = INFO', 'org.apache = DEBUG'

driverLogLevels Map<String,String>

The per-package log levels for the driver. This may include "root" package name to configure rootLogger. Examples: 'com.google = FATAL', 'root = INFO', 'org.apache = DEBUG'

driverLogLevels {[key: string]: string}

The per-package log levels for the driver. This may include "root" package name to configure rootLogger. Examples: 'com.google = FATAL', 'root = INFO', 'org.apache = DEBUG'

driver_log_levels Mapping[str, str]

The per-package log levels for the driver. This may include "root" package name to configure rootLogger. Examples: 'com.google = FATAL', 'root = INFO', 'org.apache = DEBUG'

driverLogLevels Map<String>

The per-package log levels for the driver. This may include "root" package name to configure rootLogger. Examples: 'com.google = FATAL', 'root = INFO', 'org.apache = DEBUG'

WorkflowTemplateJobScheduling

MaxFailuresPerHour int

Optional. Maximum number of times per hour a driver may be restarted as a result of driver exiting with non-zero code before job is reported failed. A job may be reported as thrashing if driver exits with non-zero code 4 times within 10 minute window. Maximum value is 10.

MaxFailuresTotal int

Optional. Maximum number of times in total a driver may be restarted as a result of driver exiting with non-zero code before job is reported failed. Maximum value is 240

MaxFailuresPerHour int

Optional. Maximum number of times per hour a driver may be restarted as a result of driver exiting with non-zero code before job is reported failed. A job may be reported as thrashing if driver exits with non-zero code 4 times within 10 minute window. Maximum value is 10.

MaxFailuresTotal int

Optional. Maximum number of times in total a driver may be restarted as a result of driver exiting with non-zero code before job is reported failed. Maximum value is 240

maxFailuresPerHour Integer

Optional. Maximum number of times per hour a driver may be restarted as a result of driver exiting with non-zero code before job is reported failed. A job may be reported as thrashing if driver exits with non-zero code 4 times within 10 minute window. Maximum value is 10.

maxFailuresTotal Integer

Optional. Maximum number of times in total a driver may be restarted as a result of driver exiting with non-zero code before job is reported failed. Maximum value is 240

maxFailuresPerHour number

Optional. Maximum number of times per hour a driver may be restarted as a result of driver exiting with non-zero code before job is reported failed. A job may be reported as thrashing if driver exits with non-zero code 4 times within 10 minute window. Maximum value is 10.

maxFailuresTotal number

Optional. Maximum number of times in total a driver may be restarted as a result of driver exiting with non-zero code before job is reported failed. Maximum value is 240

max_failures_per_hour int

Optional. Maximum number of times per hour a driver may be restarted as a result of driver exiting with non-zero code before job is reported failed. A job may be reported as thrashing if driver exits with non-zero code 4 times within 10 minute window. Maximum value is 10.

max_failures_total int

Optional. Maximum number of times in total a driver may be restarted as a result of driver exiting with non-zero code before job is reported failed. Maximum value is 240

maxFailuresPerHour Number

Optional. Maximum number of times per hour a driver may be restarted as a result of driver exiting with non-zero code before job is reported failed. A job may be reported as thrashing if driver exits with non-zero code 4 times within 10 minute window. Maximum value is 10.

maxFailuresTotal Number

Optional. Maximum number of times in total a driver may be restarted as a result of driver exiting with non-zero code before job is reported failed. Maximum value is 240

WorkflowTemplateJobSparkJob

ArchiveUris List<string>

Optional. HCFS URIs of archives to be extracted into the working directory of each executor. Supported file types: .jar, .tar, .tar.gz, .tgz, and .zip.

Args List<string>

Optional. The arguments to pass to the driver. Do not include arguments, such as --conf, that can be set as job properties, since a collision may occur that causes an incorrect job submission.

FileUris List<string>

Optional. HCFS URIs of files to be placed in the working directory of each executor. Useful for naively parallel tasks.

JarFileUris List<string>

Optional. HCFS URIs of jar files to be added to the Spark CLASSPATH.

LoggingConfig WorkflowTemplateJobSparkJobLoggingConfig

Optional. The runtime log config for job execution.

MainClass string

The name of the driver's main class. The jar file that contains the class must be in the default CLASSPATH or specified in jar_file_uris.

MainJarFileUri string

The HCFS URI of the jar file that contains the main class.

Properties Dictionary<string, string>

Optional. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. The following are supported prefixes and their mappings: * capacity-scheduler: capacity-scheduler.xml * core: core-site.xml * distcp: distcp-default.xml * hdfs: hdfs-site.xml * hive: hive-site.xml * mapred: mapred-site.xml * pig: pig.properties * spark: spark-defaults.conf * yarn: yarn-site.xml For more information, see (https://cloud.google.com/dataproc/docs/concepts/cluster-properties).

ArchiveUris []string

Optional. HCFS URIs of archives to be extracted into the working directory of each executor. Supported file types: .jar, .tar, .tar.gz, .tgz, and .zip.

Args []string

Optional. The arguments to pass to the driver. Do not include arguments, such as --conf, that can be set as job properties, since a collision may occur that causes an incorrect job submission.

FileUris []string

Optional. HCFS URIs of files to be placed in the working directory of each executor. Useful for naively parallel tasks.

JarFileUris []string

Optional. HCFS URIs of jar files to be added to the Spark CLASSPATH.

LoggingConfig WorkflowTemplateJobSparkJobLoggingConfig

Optional. The runtime log config for job execution.

MainClass string

The name of the driver's main class. The jar file that contains the class must be in the default CLASSPATH or specified in jar_file_uris.

MainJarFileUri string

The HCFS URI of the jar file that contains the main class.

Properties map[string]string

Optional. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. The following are supported prefixes and their mappings: * capacity-scheduler: capacity-scheduler.xml * core: core-site.xml * distcp: distcp-default.xml * hdfs: hdfs-site.xml * hive: hive-site.xml * mapred: mapred-site.xml * pig: pig.properties * spark: spark-defaults.conf * yarn: yarn-site.xml For more information, see (https://cloud.google.com/dataproc/docs/concepts/cluster-properties).

archiveUris List<String>

Optional. HCFS URIs of archives to be extracted into the working directory of each executor. Supported file types: .jar, .tar, .tar.gz, .tgz, and .zip.

args List<String>

Optional. The arguments to pass to the driver. Do not include arguments, such as --conf, that can be set as job properties, since a collision may occur that causes an incorrect job submission.

fileUris List<String>

Optional. HCFS URIs of files to be placed in the working directory of each executor. Useful for naively parallel tasks.

jarFileUris List<String>

Optional. HCFS URIs of jar files to be added to the Spark CLASSPATH.

loggingConfig WorkflowTemplateJobSparkJobLoggingConfig

Optional. The runtime log config for job execution.

mainClass String

The name of the driver's main class. The jar file that contains the class must be in the default CLASSPATH or specified in jar_file_uris.

mainJarFileUri String

The HCFS URI of the jar file that contains the main class.

properties Map<String,String>

Optional. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. The following are supported prefixes and their mappings: * capacity-scheduler: capacity-scheduler.xml * core: core-site.xml * distcp: distcp-default.xml * hdfs: hdfs-site.xml * hive: hive-site.xml * mapred: mapred-site.xml * pig: pig.properties * spark: spark-defaults.conf * yarn: yarn-site.xml For more information, see (https://cloud.google.com/dataproc/docs/concepts/cluster-properties).

archiveUris string[]

Optional. HCFS URIs of archives to be extracted into the working directory of each executor. Supported file types: .jar, .tar, .tar.gz, .tgz, and .zip.

args string[]

Optional. The arguments to pass to the driver. Do not include arguments, such as --conf, that can be set as job properties, since a collision may occur that causes an incorrect job submission.

fileUris string[]

Optional. HCFS URIs of files to be placed in the working directory of each executor. Useful for naively parallel tasks.

jarFileUris string[]

Optional. HCFS URIs of jar files to be added to the Spark CLASSPATH.

loggingConfig WorkflowTemplateJobSparkJobLoggingConfig

Optional. The runtime log config for job execution.

mainClass string

The name of the driver's main class. The jar file that contains the class must be in the default CLASSPATH or specified in jar_file_uris.

mainJarFileUri string

The HCFS URI of the jar file that contains the main class.

properties {[key: string]: string}

Optional. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. The following are supported prefixes and their mappings: * capacity-scheduler: capacity-scheduler.xml * core: core-site.xml * distcp: distcp-default.xml * hdfs: hdfs-site.xml * hive: hive-site.xml * mapred: mapred-site.xml * pig: pig.properties * spark: spark-defaults.conf * yarn: yarn-site.xml For more information, see (https://cloud.google.com/dataproc/docs/concepts/cluster-properties).

archive_uris Sequence[str]

Optional. HCFS URIs of archives to be extracted into the working directory of each executor. Supported file types: .jar, .tar, .tar.gz, .tgz, and .zip.

args Sequence[str]

Optional. The arguments to pass to the driver. Do not include arguments, such as --conf, that can be set as job properties, since a collision may occur that causes an incorrect job submission.

file_uris Sequence[str]

Optional. HCFS URIs of files to be placed in the working directory of each executor. Useful for naively parallel tasks.

jar_file_uris Sequence[str]

Optional. HCFS URIs of jar files to be added to the Spark CLASSPATH.

logging_config WorkflowTemplateJobSparkJobLoggingConfig

Optional. The runtime log config for job execution.

main_class str

The name of the driver's main class. The jar file that contains the class must be in the default CLASSPATH or specified in jar_file_uris.

main_jar_file_uri str

The HCFS URI of the jar file that contains the main class.

properties Mapping[str, str]

Optional. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. The following are supported prefixes and their mappings: * capacity-scheduler: capacity-scheduler.xml * core: core-site.xml * distcp: distcp-default.xml * hdfs: hdfs-site.xml * hive: hive-site.xml * mapred: mapred-site.xml * pig: pig.properties * spark: spark-defaults.conf * yarn: yarn-site.xml For more information, see (https://cloud.google.com/dataproc/docs/concepts/cluster-properties).

archiveUris List<String>

Optional. HCFS URIs of archives to be extracted into the working directory of each executor. Supported file types: .jar, .tar, .tar.gz, .tgz, and .zip.

args List<String>

Optional. The arguments to pass to the driver. Do not include arguments, such as --conf, that can be set as job properties, since a collision may occur that causes an incorrect job submission.

fileUris List<String>

Optional. HCFS URIs of files to be placed in the working directory of each executor. Useful for naively parallel tasks.

jarFileUris List<String>

Optional. HCFS URIs of jar files to be added to the Spark CLASSPATH.

loggingConfig Property Map

Optional. The runtime log config for job execution.

mainClass String

The name of the driver's main class. The jar file that contains the class must be in the default CLASSPATH or specified in jar_file_uris.

mainJarFileUri String

The HCFS URI of the jar file that contains the main class.

properties Map<String>

Optional. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. The following are supported prefixes and their mappings: * capacity-scheduler: capacity-scheduler.xml * core: core-site.xml * distcp: distcp-default.xml * hdfs: hdfs-site.xml * hive: hive-site.xml * mapred: mapred-site.xml * pig: pig.properties * spark: spark-defaults.conf * yarn: yarn-site.xml For more information, see (https://cloud.google.com/dataproc/docs/concepts/cluster-properties).

WorkflowTemplateJobSparkJobLoggingConfig

DriverLogLevels Dictionary<string, string>

The per-package log levels for the driver. This may include "root" package name to configure rootLogger. Examples: 'com.google = FATAL', 'root = INFO', 'org.apache = DEBUG'

DriverLogLevels map[string]string

The per-package log levels for the driver. This may include "root" package name to configure rootLogger. Examples: 'com.google = FATAL', 'root = INFO', 'org.apache = DEBUG'

driverLogLevels Map<String,String>

The per-package log levels for the driver. This may include "root" package name to configure rootLogger. Examples: 'com.google = FATAL', 'root = INFO', 'org.apache = DEBUG'

driverLogLevels {[key: string]: string}

The per-package log levels for the driver. This may include "root" package name to configure rootLogger. Examples: 'com.google = FATAL', 'root = INFO', 'org.apache = DEBUG'

driver_log_levels Mapping[str, str]

The per-package log levels for the driver. This may include "root" package name to configure rootLogger. Examples: 'com.google = FATAL', 'root = INFO', 'org.apache = DEBUG'

driverLogLevels Map<String>

The per-package log levels for the driver. This may include "root" package name to configure rootLogger. Examples: 'com.google = FATAL', 'root = INFO', 'org.apache = DEBUG'

WorkflowTemplateJobSparkRJob

MainRFileUri string

Required. The HCFS URI of the main R file to use as the driver. Must be a .R file.

ArchiveUris List<string>

Optional. HCFS URIs of archives to be extracted into the working directory of each executor. Supported file types: .jar, .tar, .tar.gz, .tgz, and .zip.

Args List<string>

Optional. The arguments to pass to the driver. Do not include arguments, such as --conf, that can be set as job properties, since a collision may occur that causes an incorrect job submission.

FileUris List<string>

Optional. HCFS URIs of files to be placed in the working directory of each executor. Useful for naively parallel tasks.

LoggingConfig WorkflowTemplateJobSparkRJobLoggingConfig

Optional. The runtime log config for job execution.

Properties Dictionary<string, string>

Optional. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. The following are supported prefixes and their mappings: * capacity-scheduler: capacity-scheduler.xml * core: core-site.xml * distcp: distcp-default.xml * hdfs: hdfs-site.xml * hive: hive-site.xml * mapred: mapred-site.xml * pig: pig.properties * spark: spark-defaults.conf * yarn: yarn-site.xml For more information, see (https://cloud.google.com/dataproc/docs/concepts/cluster-properties).

MainRFileUri string

Required. The HCFS URI of the main R file to use as the driver. Must be a .R file.

ArchiveUris []string

Optional. HCFS URIs of archives to be extracted into the working directory of each executor. Supported file types: .jar, .tar, .tar.gz, .tgz, and .zip.

Args []string

Optional. The arguments to pass to the driver. Do not include arguments, such as --conf, that can be set as job properties, since a collision may occur that causes an incorrect job submission.

FileUris []string

Optional. HCFS URIs of files to be placed in the working directory of each executor. Useful for naively parallel tasks.

LoggingConfig WorkflowTemplateJobSparkRJobLoggingConfig

Optional. The runtime log config for job execution.

Properties map[string]string

Optional. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. The following are supported prefixes and their mappings: * capacity-scheduler: capacity-scheduler.xml * core: core-site.xml * distcp: distcp-default.xml * hdfs: hdfs-site.xml * hive: hive-site.xml * mapred: mapred-site.xml * pig: pig.properties * spark: spark-defaults.conf * yarn: yarn-site.xml For more information, see (https://cloud.google.com/dataproc/docs/concepts/cluster-properties).

mainRFileUri String

Required. The HCFS URI of the main R file to use as the driver. Must be a .R file.

archiveUris List<String>

Optional. HCFS URIs of archives to be extracted into the working directory of each executor. Supported file types: .jar, .tar, .tar.gz, .tgz, and .zip.

args List<String>

Optional. The arguments to pass to the driver. Do not include arguments, such as --conf, that can be set as job properties, since a collision may occur that causes an incorrect job submission.

fileUris List<String>

Optional. HCFS URIs of files to be placed in the working directory of each executor. Useful for naively parallel tasks.

loggingConfig WorkflowTemplateJobSparkRJobLoggingConfig

Optional. The runtime log config for job execution.

properties Map<String,String>

Optional. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. The following are supported prefixes and their mappings: * capacity-scheduler: capacity-scheduler.xml * core: core-site.xml * distcp: distcp-default.xml * hdfs: hdfs-site.xml * hive: hive-site.xml * mapred: mapred-site.xml * pig: pig.properties * spark: spark-defaults.conf * yarn: yarn-site.xml For more information, see (https://cloud.google.com/dataproc/docs/concepts/cluster-properties).

mainRFileUri string

Required. The HCFS URI of the main R file to use as the driver. Must be a .R file.

archiveUris string[]

Optional. HCFS URIs of archives to be extracted into the working directory of each executor. Supported file types: .jar, .tar, .tar.gz, .tgz, and .zip.

args string[]

Optional. The arguments to pass to the driver. Do not include arguments, such as --conf, that can be set as job properties, since a collision may occur that causes an incorrect job submission.

fileUris string[]

Optional. HCFS URIs of files to be placed in the working directory of each executor. Useful for naively parallel tasks.

loggingConfig WorkflowTemplateJobSparkRJobLoggingConfig

Optional. The runtime log config for job execution.

properties {[key: string]: string}

Optional. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. The following are supported prefixes and their mappings: * capacity-scheduler: capacity-scheduler.xml * core: core-site.xml * distcp: distcp-default.xml * hdfs: hdfs-site.xml * hive: hive-site.xml * mapred: mapred-site.xml * pig: pig.properties * spark: spark-defaults.conf * yarn: yarn-site.xml For more information, see (https://cloud.google.com/dataproc/docs/concepts/cluster-properties).

main_r_file_uri str

Required. The HCFS URI of the main R file to use as the driver. Must be a .R file.

archive_uris Sequence[str]

Optional. HCFS URIs of archives to be extracted into the working directory of each executor. Supported file types: .jar, .tar, .tar.gz, .tgz, and .zip.

args Sequence[str]

Optional. The arguments to pass to the driver. Do not include arguments, such as --conf, that can be set as job properties, since a collision may occur that causes an incorrect job submission.

file_uris Sequence[str]

Optional. HCFS URIs of files to be placed in the working directory of each executor. Useful for naively parallel tasks.

logging_config WorkflowTemplateJobSparkRJobLoggingConfig

Optional. The runtime log config for job execution.

properties Mapping[str, str]

Optional. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. The following are supported prefixes and their mappings: * capacity-scheduler: capacity-scheduler.xml * core: core-site.xml * distcp: distcp-default.xml * hdfs: hdfs-site.xml * hive: hive-site.xml * mapred: mapred-site.xml * pig: pig.properties * spark: spark-defaults.conf * yarn: yarn-site.xml For more information, see (https://cloud.google.com/dataproc/docs/concepts/cluster-properties).

mainRFileUri String

Required. The HCFS URI of the main R file to use as the driver. Must be a .R file.

archiveUris List<String>

Optional. HCFS URIs of archives to be extracted into the working directory of each executor. Supported file types: .jar, .tar, .tar.gz, .tgz, and .zip.

args List<String>

Optional. The arguments to pass to the driver. Do not include arguments, such as --conf, that can be set as job properties, since a collision may occur that causes an incorrect job submission.

fileUris List<String>

Optional. HCFS URIs of files to be placed in the working directory of each executor. Useful for naively parallel tasks.

loggingConfig Property Map

Optional. The runtime log config for job execution.

properties Map<String>

Optional. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. The following are supported prefixes and their mappings: * capacity-scheduler: capacity-scheduler.xml * core: core-site.xml * distcp: distcp-default.xml * hdfs: hdfs-site.xml * hive: hive-site.xml * mapred: mapred-site.xml * pig: pig.properties * spark: spark-defaults.conf * yarn: yarn-site.xml For more information, see (https://cloud.google.com/dataproc/docs/concepts/cluster-properties).

WorkflowTemplateJobSparkRJobLoggingConfig

DriverLogLevels Dictionary<string, string>

The per-package log levels for the driver. This may include "root" package name to configure rootLogger. Examples: 'com.google = FATAL', 'root = INFO', 'org.apache = DEBUG'

DriverLogLevels map[string]string

The per-package log levels for the driver. This may include "root" package name to configure rootLogger. Examples: 'com.google = FATAL', 'root = INFO', 'org.apache = DEBUG'

driverLogLevels Map<String,String>

The per-package log levels for the driver. This may include "root" package name to configure rootLogger. Examples: 'com.google = FATAL', 'root = INFO', 'org.apache = DEBUG'

driverLogLevels {[key: string]: string}

The per-package log levels for the driver. This may include "root" package name to configure rootLogger. Examples: 'com.google = FATAL', 'root = INFO', 'org.apache = DEBUG'

driver_log_levels Mapping[str, str]

The per-package log levels for the driver. This may include "root" package name to configure rootLogger. Examples: 'com.google = FATAL', 'root = INFO', 'org.apache = DEBUG'

driverLogLevels Map<String>

The per-package log levels for the driver. This may include "root" package name to configure rootLogger. Examples: 'com.google = FATAL', 'root = INFO', 'org.apache = DEBUG'

WorkflowTemplateJobSparkSqlJob

JarFileUris List<string>

Optional. HCFS URIs of jar files to be added to the Spark CLASSPATH.

LoggingConfig WorkflowTemplateJobSparkSqlJobLoggingConfig

Optional. The runtime log config for job execution.

Properties Dictionary<string, string>

Optional. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. The following are supported prefixes and their mappings: * capacity-scheduler: capacity-scheduler.xml * core: core-site.xml * distcp: distcp-default.xml * hdfs: hdfs-site.xml * hive: hive-site.xml * mapred: mapred-site.xml * pig: pig.properties * spark: spark-defaults.conf * yarn: yarn-site.xml For more information, see (https://cloud.google.com/dataproc/docs/concepts/cluster-properties).

QueryFileUri string

The HCFS URI of the script that contains SQL queries.

QueryList WorkflowTemplateJobSparkSqlJobQueryList

A list of queries.

ScriptVariables Dictionary<string, string>

Optional. Mapping of query variable names to values (equivalent to the Spark SQL command: SET name="value";).

JarFileUris []string

Optional. HCFS URIs of jar files to be added to the Spark CLASSPATH.

LoggingConfig WorkflowTemplateJobSparkSqlJobLoggingConfig

Optional. The runtime log config for job execution.

Properties map[string]string

Optional. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. The following are supported prefixes and their mappings: * capacity-scheduler: capacity-scheduler.xml * core: core-site.xml * distcp: distcp-default.xml * hdfs: hdfs-site.xml * hive: hive-site.xml * mapred: mapred-site.xml * pig: pig.properties * spark: spark-defaults.conf * yarn: yarn-site.xml For more information, see (https://cloud.google.com/dataproc/docs/concepts/cluster-properties).

QueryFileUri string

The HCFS URI of the script that contains SQL queries.

QueryList WorkflowTemplateJobSparkSqlJobQueryList

A list of queries.

ScriptVariables map[string]string

Optional. Mapping of query variable names to values (equivalent to the Spark SQL command: SET name="value";).

jarFileUris List<String>

Optional. HCFS URIs of jar files to be added to the Spark CLASSPATH.

loggingConfig WorkflowTemplateJobSparkSqlJobLoggingConfig

Optional. The runtime log config for job execution.

properties Map<String,String>

Optional. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. The following are supported prefixes and their mappings: * capacity-scheduler: capacity-scheduler.xml * core: core-site.xml * distcp: distcp-default.xml * hdfs: hdfs-site.xml * hive: hive-site.xml * mapred: mapred-site.xml * pig: pig.properties * spark: spark-defaults.conf * yarn: yarn-site.xml For more information, see (https://cloud.google.com/dataproc/docs/concepts/cluster-properties).

queryFileUri String

The HCFS URI of the script that contains SQL queries.

queryList WorkflowTemplateJobSparkSqlJobQueryList

A list of queries.

scriptVariables Map<String,String>

Optional. Mapping of query variable names to values (equivalent to the Spark SQL command: SET name="value";).

jarFileUris string[]

Optional. HCFS URIs of jar files to be added to the Spark CLASSPATH.

loggingConfig WorkflowTemplateJobSparkSqlJobLoggingConfig

Optional. The runtime log config for job execution.

properties {[key: string]: string}

Optional. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. The following are supported prefixes and their mappings: * capacity-scheduler: capacity-scheduler.xml * core: core-site.xml * distcp: distcp-default.xml * hdfs: hdfs-site.xml * hive: hive-site.xml * mapred: mapred-site.xml * pig: pig.properties * spark: spark-defaults.conf * yarn: yarn-site.xml For more information, see (https://cloud.google.com/dataproc/docs/concepts/cluster-properties).

queryFileUri string

The HCFS URI of the script that contains SQL queries.

queryList WorkflowTemplateJobSparkSqlJobQueryList

A list of queries.

scriptVariables {[key: string]: string}

Optional. Mapping of query variable names to values (equivalent to the Spark SQL command: SET name="value";).

jar_file_uris Sequence[str]

Optional. HCFS URIs of jar files to be added to the Spark CLASSPATH.

logging_config WorkflowTemplateJobSparkSqlJobLoggingConfig

Optional. The runtime log config for job execution.

properties Mapping[str, str]

Optional. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. The following are supported prefixes and their mappings: * capacity-scheduler: capacity-scheduler.xml * core: core-site.xml * distcp: distcp-default.xml * hdfs: hdfs-site.xml * hive: hive-site.xml * mapred: mapred-site.xml * pig: pig.properties * spark: spark-defaults.conf * yarn: yarn-site.xml For more information, see (https://cloud.google.com/dataproc/docs/concepts/cluster-properties).

query_file_uri str

The HCFS URI of the script that contains SQL queries.

query_list WorkflowTemplateJobSparkSqlJobQueryList

A list of queries.

script_variables Mapping[str, str]

Optional. Mapping of query variable names to values (equivalent to the Spark SQL command: SET name="value";).

jarFileUris List<String>

Optional. HCFS URIs of jar files to be added to the Spark CLASSPATH.

loggingConfig Property Map

Optional. The runtime log config for job execution.

properties Map<String>

Optional. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. The following are supported prefixes and their mappings: * capacity-scheduler: capacity-scheduler.xml * core: core-site.xml * distcp: distcp-default.xml * hdfs: hdfs-site.xml * hive: hive-site.xml * mapred: mapred-site.xml * pig: pig.properties * spark: spark-defaults.conf * yarn: yarn-site.xml For more information, see (https://cloud.google.com/dataproc/docs/concepts/cluster-properties).

queryFileUri String

The HCFS URI of the script that contains SQL queries.

queryList Property Map

A list of queries.

scriptVariables Map<String>

Optional. Mapping of query variable names to values (equivalent to the Spark SQL command: SET name="value";).

WorkflowTemplateJobSparkSqlJobLoggingConfig

DriverLogLevels Dictionary<string, string>

The per-package log levels for the driver. This may include "root" package name to configure rootLogger. Examples: 'com.google = FATAL', 'root = INFO', 'org.apache = DEBUG'

DriverLogLevels map[string]string

The per-package log levels for the driver. This may include "root" package name to configure rootLogger. Examples: 'com.google = FATAL', 'root = INFO', 'org.apache = DEBUG'

driverLogLevels Map<String,String>

The per-package log levels for the driver. This may include "root" package name to configure rootLogger. Examples: 'com.google = FATAL', 'root = INFO', 'org.apache = DEBUG'

driverLogLevels {[key: string]: string}

The per-package log levels for the driver. This may include "root" package name to configure rootLogger. Examples: 'com.google = FATAL', 'root = INFO', 'org.apache = DEBUG'

driver_log_levels Mapping[str, str]

The per-package log levels for the driver. This may include "root" package name to configure rootLogger. Examples: 'com.google = FATAL', 'root = INFO', 'org.apache = DEBUG'

driverLogLevels Map<String>

The per-package log levels for the driver. This may include "root" package name to configure rootLogger. Examples: 'com.google = FATAL', 'root = INFO', 'org.apache = DEBUG'

WorkflowTemplateJobSparkSqlJobQueryList

Queries List<string>

Required. The queries to execute. You do not need to end a query expression with a semicolon. Multiple queries can be specified in one string by separating each with a semicolon. Here is an example of a Dataproc API snippet that uses a QueryList to specify a HiveJob: "hiveJob": { "queryList": { "queries": } }

Queries []string

Required. The queries to execute. You do not need to end a query expression with a semicolon. Multiple queries can be specified in one string by separating each with a semicolon. Here is an example of a Dataproc API snippet that uses a QueryList to specify a HiveJob: "hiveJob": { "queryList": { "queries": } }

queries List<String>

Required. The queries to execute. You do not need to end a query expression with a semicolon. Multiple queries can be specified in one string by separating each with a semicolon. Here is an example of a Dataproc API snippet that uses a QueryList to specify a HiveJob: "hiveJob": { "queryList": { "queries": } }

queries string[]

Required. The queries to execute. You do not need to end a query expression with a semicolon. Multiple queries can be specified in one string by separating each with a semicolon. Here is an example of a Dataproc API snippet that uses a QueryList to specify a HiveJob: "hiveJob": { "queryList": { "queries": } }

queries Sequence[str]

Required. The queries to execute. You do not need to end a query expression with a semicolon. Multiple queries can be specified in one string by separating each with a semicolon. Here is an example of a Dataproc API snippet that uses a QueryList to specify a HiveJob: "hiveJob": { "queryList": { "queries": } }

queries List<String>

Required. The queries to execute. You do not need to end a query expression with a semicolon. Multiple queries can be specified in one string by separating each with a semicolon. Here is an example of a Dataproc API snippet that uses a QueryList to specify a HiveJob: "hiveJob": { "queryList": { "queries": } }

WorkflowTemplateParameter

Fields List<string>

Required. Paths to all fields that the parameter replaces. A field is allowed to appear in at most one parameter's list of field paths. A field path is similar in syntax to a .sparkJob.args

Name string

Required. Parameter name. The parameter name is used as the key, and paired with the parameter value, which are passed to the template when the template is instantiated. The name must contain only capital letters (A-Z), numbers (0-9), and underscores (_), and must not start with a number. The maximum length is 40 characters.

Description string

Optional. Brief description of the parameter. Must not exceed 1024 characters.

Validation WorkflowTemplateParameterValidation

Optional. Validation rules to be applied to this parameter's value.

Fields []string

Required. Paths to all fields that the parameter replaces. A field is allowed to appear in at most one parameter's list of field paths. A field path is similar in syntax to a .sparkJob.args

Name string

Required. Parameter name. The parameter name is used as the key, and paired with the parameter value, which are passed to the template when the template is instantiated. The name must contain only capital letters (A-Z), numbers (0-9), and underscores (_), and must not start with a number. The maximum length is 40 characters.

Description string

Optional. Brief description of the parameter. Must not exceed 1024 characters.

Validation WorkflowTemplateParameterValidation

Optional. Validation rules to be applied to this parameter's value.

fields List<String>

Required. Paths to all fields that the parameter replaces. A field is allowed to appear in at most one parameter's list of field paths. A field path is similar in syntax to a .sparkJob.args

name String

Required. Parameter name. The parameter name is used as the key, and paired with the parameter value, which are passed to the template when the template is instantiated. The name must contain only capital letters (A-Z), numbers (0-9), and underscores (_), and must not start with a number. The maximum length is 40 characters.

description String

Optional. Brief description of the parameter. Must not exceed 1024 characters.

validation WorkflowTemplateParameterValidation

Optional. Validation rules to be applied to this parameter's value.

fields string[]

Required. Paths to all fields that the parameter replaces. A field is allowed to appear in at most one parameter's list of field paths. A field path is similar in syntax to a .sparkJob.args

name string

Required. Parameter name. The parameter name is used as the key, and paired with the parameter value, which are passed to the template when the template is instantiated. The name must contain only capital letters (A-Z), numbers (0-9), and underscores (_), and must not start with a number. The maximum length is 40 characters.

description string

Optional. Brief description of the parameter. Must not exceed 1024 characters.

validation WorkflowTemplateParameterValidation

Optional. Validation rules to be applied to this parameter's value.

fields Sequence[str]

Required. Paths to all fields that the parameter replaces. A field is allowed to appear in at most one parameter's list of field paths. A field path is similar in syntax to a .sparkJob.args

name str

Required. Parameter name. The parameter name is used as the key, and paired with the parameter value, which are passed to the template when the template is instantiated. The name must contain only capital letters (A-Z), numbers (0-9), and underscores (_), and must not start with a number. The maximum length is 40 characters.

description str

Optional. Brief description of the parameter. Must not exceed 1024 characters.

validation WorkflowTemplateParameterValidation

Optional. Validation rules to be applied to this parameter's value.

fields List<String>

Required. Paths to all fields that the parameter replaces. A field is allowed to appear in at most one parameter's list of field paths. A field path is similar in syntax to a .sparkJob.args

name String

Required. Parameter name. The parameter name is used as the key, and paired with the parameter value, which are passed to the template when the template is instantiated. The name must contain only capital letters (A-Z), numbers (0-9), and underscores (_), and must not start with a number. The maximum length is 40 characters.

description String

Optional. Brief description of the parameter. Must not exceed 1024 characters.

validation Property Map

Optional. Validation rules to be applied to this parameter's value.

WorkflowTemplateParameterValidation

Regex WorkflowTemplateParameterValidationRegex

Validation based on regular expressions.

Values WorkflowTemplateParameterValidationValues

Optional. Corresponds to the label values of reservation resource.

Regex WorkflowTemplateParameterValidationRegex

Validation based on regular expressions.

Values WorkflowTemplateParameterValidationValues

Optional. Corresponds to the label values of reservation resource.

regex WorkflowTemplateParameterValidationRegex

Validation based on regular expressions.

values WorkflowTemplateParameterValidationValues

Optional. Corresponds to the label values of reservation resource.

regex WorkflowTemplateParameterValidationRegex

Validation based on regular expressions.

values WorkflowTemplateParameterValidationValues

Optional. Corresponds to the label values of reservation resource.

regex WorkflowTemplateParameterValidationRegex

Validation based on regular expressions.

values WorkflowTemplateParameterValidationValues

Optional. Corresponds to the label values of reservation resource.

regex Property Map

Validation based on regular expressions.

values Property Map

Optional. Corresponds to the label values of reservation resource.

WorkflowTemplateParameterValidationRegex

Regexes List<string>

Required. RE2 regular expressions used to validate the parameter's value. The value must match the regex in its entirety (substring matches are not sufficient).

Regexes []string

Required. RE2 regular expressions used to validate the parameter's value. The value must match the regex in its entirety (substring matches are not sufficient).

regexes List<String>

Required. RE2 regular expressions used to validate the parameter's value. The value must match the regex in its entirety (substring matches are not sufficient).

regexes string[]

Required. RE2 regular expressions used to validate the parameter's value. The value must match the regex in its entirety (substring matches are not sufficient).

regexes Sequence[str]

Required. RE2 regular expressions used to validate the parameter's value. The value must match the regex in its entirety (substring matches are not sufficient).

regexes List<String>

Required. RE2 regular expressions used to validate the parameter's value. The value must match the regex in its entirety (substring matches are not sufficient).

WorkflowTemplateParameterValidationValues

Values List<string>

Optional. Corresponds to the label values of reservation resource.

Values []string

Optional. Corresponds to the label values of reservation resource.

values List<String>

Optional. Corresponds to the label values of reservation resource.

values string[]

Optional. Corresponds to the label values of reservation resource.

values Sequence[str]

Optional. Corresponds to the label values of reservation resource.

values List<String>

Optional. Corresponds to the label values of reservation resource.

WorkflowTemplatePlacement

ClusterSelector WorkflowTemplatePlacementClusterSelector

Optional. A selector that chooses target cluster for jobs based on metadata. The selector is evaluated at the time each job is submitted.

ManagedCluster WorkflowTemplatePlacementManagedCluster

A cluster that is managed by the workflow.

ClusterSelector WorkflowTemplatePlacementClusterSelector

Optional. A selector that chooses target cluster for jobs based on metadata. The selector is evaluated at the time each job is submitted.

ManagedCluster WorkflowTemplatePlacementManagedCluster

A cluster that is managed by the workflow.

clusterSelector WorkflowTemplatePlacementClusterSelector

Optional. A selector that chooses target cluster for jobs based on metadata. The selector is evaluated at the time each job is submitted.

managedCluster WorkflowTemplatePlacementManagedCluster

A cluster that is managed by the workflow.

clusterSelector WorkflowTemplatePlacementClusterSelector

Optional. A selector that chooses target cluster for jobs based on metadata. The selector is evaluated at the time each job is submitted.

managedCluster WorkflowTemplatePlacementManagedCluster

A cluster that is managed by the workflow.

cluster_selector WorkflowTemplatePlacementClusterSelector

Optional. A selector that chooses target cluster for jobs based on metadata. The selector is evaluated at the time each job is submitted.

managed_cluster WorkflowTemplatePlacementManagedCluster

A cluster that is managed by the workflow.

clusterSelector Property Map

Optional. A selector that chooses target cluster for jobs based on metadata. The selector is evaluated at the time each job is submitted.

managedCluster Property Map

A cluster that is managed by the workflow.

WorkflowTemplatePlacementClusterSelector

ClusterLabels Dictionary<string, string>

Required. The cluster labels. Cluster must have all labels to match.

Zone string

Optional. The zone where the Compute Engine cluster will be located. On a create request, it is required in the "global" region. If omitted in a non-global Dataproc region, the service will pick a zone in the corresponding Compute Engine region. On a get request, zone will always be present. A full URL, partial URI, or short name are valid. Examples: * https://www.googleapis.com/compute/v1/projects/ * us-central1-f

ClusterLabels map[string]string

Required. The cluster labels. Cluster must have all labels to match.

Zone string

Optional. The zone where the Compute Engine cluster will be located. On a create request, it is required in the "global" region. If omitted in a non-global Dataproc region, the service will pick a zone in the corresponding Compute Engine region. On a get request, zone will always be present. A full URL, partial URI, or short name are valid. Examples: * https://www.googleapis.com/compute/v1/projects/ * us-central1-f

clusterLabels Map<String,String>

Required. The cluster labels. Cluster must have all labels to match.

zone String

Optional. The zone where the Compute Engine cluster will be located. On a create request, it is required in the "global" region. If omitted in a non-global Dataproc region, the service will pick a zone in the corresponding Compute Engine region. On a get request, zone will always be present. A full URL, partial URI, or short name are valid. Examples: * https://www.googleapis.com/compute/v1/projects/ * us-central1-f

clusterLabels {[key: string]: string}

Required. The cluster labels. Cluster must have all labels to match.

zone string

Optional. The zone where the Compute Engine cluster will be located. On a create request, it is required in the "global" region. If omitted in a non-global Dataproc region, the service will pick a zone in the corresponding Compute Engine region. On a get request, zone will always be present. A full URL, partial URI, or short name are valid. Examples: * https://www.googleapis.com/compute/v1/projects/ * us-central1-f

cluster_labels Mapping[str, str]

Required. The cluster labels. Cluster must have all labels to match.

zone str

Optional. The zone where the Compute Engine cluster will be located. On a create request, it is required in the "global" region. If omitted in a non-global Dataproc region, the service will pick a zone in the corresponding Compute Engine region. On a get request, zone will always be present. A full URL, partial URI, or short name are valid. Examples: * https://www.googleapis.com/compute/v1/projects/ * us-central1-f

clusterLabels Map<String>

Required. The cluster labels. Cluster must have all labels to match.

zone String

Optional. The zone where the Compute Engine cluster will be located. On a create request, it is required in the "global" region. If omitted in a non-global Dataproc region, the service will pick a zone in the corresponding Compute Engine region. On a get request, zone will always be present. A full URL, partial URI, or short name are valid. Examples: * https://www.googleapis.com/compute/v1/projects/ * us-central1-f

WorkflowTemplatePlacementManagedCluster

ClusterName string

Required. The cluster name prefix. A unique cluster name will be formed by appending a random suffix. The name must contain only lower-case letters (a-z), numbers (0-9), and hyphens (-). Must begin with a letter. Cannot begin or end with hyphen. Must consist of between 2 and 35 characters.

Config WorkflowTemplatePlacementManagedClusterConfig

Required. The cluster configuration.

Labels Dictionary<string, string>

Optional. The labels to associate with this cluster. Label keys must be between 1 and 63 characters long, and must conform to the following PCRE regular expression: {0,63} No more than 32 labels can be associated with a given cluster.

ClusterName string

Required. The cluster name prefix. A unique cluster name will be formed by appending a random suffix. The name must contain only lower-case letters (a-z), numbers (0-9), and hyphens (-). Must begin with a letter. Cannot begin or end with hyphen. Must consist of between 2 and 35 characters.

Config WorkflowTemplatePlacementManagedClusterConfig

Required. The cluster configuration.

Labels map[string]string

Optional. The labels to associate with this cluster. Label keys must be between 1 and 63 characters long, and must conform to the following PCRE regular expression: {0,63} No more than 32 labels can be associated with a given cluster.

clusterName String

Required. The cluster name prefix. A unique cluster name will be formed by appending a random suffix. The name must contain only lower-case letters (a-z), numbers (0-9), and hyphens (-). Must begin with a letter. Cannot begin or end with hyphen. Must consist of between 2 and 35 characters.

config WorkflowTemplatePlacementManagedClusterConfig

Required. The cluster configuration.

labels Map<String,String>

Optional. The labels to associate with this cluster. Label keys must be between 1 and 63 characters long, and must conform to the following PCRE regular expression: {0,63} No more than 32 labels can be associated with a given cluster.

clusterName string

Required. The cluster name prefix. A unique cluster name will be formed by appending a random suffix. The name must contain only lower-case letters (a-z), numbers (0-9), and hyphens (-). Must begin with a letter. Cannot begin or end with hyphen. Must consist of between 2 and 35 characters.

config WorkflowTemplatePlacementManagedClusterConfig

Required. The cluster configuration.

labels {[key: string]: string}

Optional. The labels to associate with this cluster. Label keys must be between 1 and 63 characters long, and must conform to the following PCRE regular expression: {0,63} No more than 32 labels can be associated with a given cluster.

cluster_name str

Required. The cluster name prefix. A unique cluster name will be formed by appending a random suffix. The name must contain only lower-case letters (a-z), numbers (0-9), and hyphens (-). Must begin with a letter. Cannot begin or end with hyphen. Must consist of between 2 and 35 characters.

config WorkflowTemplatePlacementManagedClusterConfig

Required. The cluster configuration.

labels Mapping[str, str]

Optional. The labels to associate with this cluster. Label keys must be between 1 and 63 characters long, and must conform to the following PCRE regular expression: {0,63} No more than 32 labels can be associated with a given cluster.

clusterName String

Required. The cluster name prefix. A unique cluster name will be formed by appending a random suffix. The name must contain only lower-case letters (a-z), numbers (0-9), and hyphens (-). Must begin with a letter. Cannot begin or end with hyphen. Must consist of between 2 and 35 characters.

config Property Map

Required. The cluster configuration.

labels Map<String>

Optional. The labels to associate with this cluster. Label keys must be between 1 and 63 characters long, and must conform to the following PCRE regular expression: {0,63} No more than 32 labels can be associated with a given cluster.

WorkflowTemplatePlacementManagedClusterConfig

AutoscalingConfig WorkflowTemplatePlacementManagedClusterConfigAutoscalingConfig

Optional. Autoscaling config for the policy associated with the cluster. Cluster does not autoscale if this field is unset.

EncryptionConfig WorkflowTemplatePlacementManagedClusterConfigEncryptionConfig

Optional. Encryption settings for the cluster.

EndpointConfig WorkflowTemplatePlacementManagedClusterConfigEndpointConfig

Optional. Port/endpoint configuration for this cluster

GceClusterConfig WorkflowTemplatePlacementManagedClusterConfigGceClusterConfig

Optional. The shared Compute Engine config settings for all instances in a cluster.

GkeClusterConfig WorkflowTemplatePlacementManagedClusterConfigGkeClusterConfig

Optional. The Kubernetes Engine config for Dataproc clusters deployed to Kubernetes. Setting this is considered mutually exclusive with Compute Engine-based options such as gce_cluster_config, master_config, worker_config, secondary_worker_config, and autoscaling_config.

InitializationActions List<WorkflowTemplatePlacementManagedClusterConfigInitializationAction>

Optional. Commands to execute on each node after config is completed. By default, executables are run on master and all worker nodes. You can test a node's role metadata to run an executable on a master or worker node, as shown below using curl (you can also use wget): ROLE=$(curl -H Metadata-Flavor:Google http://metadata/computeMetadata/v1/instance/attributes/dataproc-role) if ; then ... master specific actions ... else ... worker specific actions ... fi

LifecycleConfig WorkflowTemplatePlacementManagedClusterConfigLifecycleConfig

Optional. Lifecycle setting for the cluster.

MasterConfig WorkflowTemplatePlacementManagedClusterConfigMasterConfig

Optional. The Compute Engine config settings for additional worker instances in a cluster.

MetastoreConfig WorkflowTemplatePlacementManagedClusterConfigMetastoreConfig

Optional. Metastore configuration.

SecondaryWorkerConfig WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfig

Optional. The Compute Engine config settings for additional worker instances in a cluster.

SecurityConfig WorkflowTemplatePlacementManagedClusterConfigSecurityConfig

Optional. Security settings for the cluster.

SoftwareConfig WorkflowTemplatePlacementManagedClusterConfigSoftwareConfig

Optional. The config settings for software inside the cluster.

StagingBucket string

Optional. A Cloud Storage bucket used to stage job dependencies, config files, and job driver console output. If you do not specify a staging bucket, Cloud Dataproc will determine a Cloud Storage location (US, ASIA, or EU) for your cluster's staging bucket according to the Compute Engine zone where your cluster is deployed, and then create and manage this project-level, per-location bucket (see (https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/staging-bucket)).

TempBucket string

Optional. A Cloud Storage bucket used to store ephemeral cluster and jobs data, such as Spark and MapReduce history files. If you do not specify a temp bucket, Dataproc will determine a Cloud Storage location (US, ASIA, or EU) for your cluster's temp bucket according to the Compute Engine zone where your cluster is deployed, and then create and manage this project-level, per-location bucket. The default bucket has a TTL of 90 days, but you can use any TTL (or none) if you specify a bucket.

WorkerConfig WorkflowTemplatePlacementManagedClusterConfigWorkerConfig

Optional. The Compute Engine config settings for additional worker instances in a cluster.

AutoscalingConfig WorkflowTemplatePlacementManagedClusterConfigAutoscalingConfig

Optional. Autoscaling config for the policy associated with the cluster. Cluster does not autoscale if this field is unset.

EncryptionConfig WorkflowTemplatePlacementManagedClusterConfigEncryptionConfig

Optional. Encryption settings for the cluster.

EndpointConfig WorkflowTemplatePlacementManagedClusterConfigEndpointConfig

Optional. Port/endpoint configuration for this cluster

GceClusterConfig WorkflowTemplatePlacementManagedClusterConfigGceClusterConfig

Optional. The shared Compute Engine config settings for all instances in a cluster.

GkeClusterConfig WorkflowTemplatePlacementManagedClusterConfigGkeClusterConfig

Optional. The Kubernetes Engine config for Dataproc clusters deployed to Kubernetes. Setting this is considered mutually exclusive with Compute Engine-based options such as gce_cluster_config, master_config, worker_config, secondary_worker_config, and autoscaling_config.

InitializationActions []WorkflowTemplatePlacementManagedClusterConfigInitializationAction

Optional. Commands to execute on each node after config is completed. By default, executables are run on master and all worker nodes. You can test a node's role metadata to run an executable on a master or worker node, as shown below using curl (you can also use wget): ROLE=$(curl -H Metadata-Flavor:Google http://metadata/computeMetadata/v1/instance/attributes/dataproc-role) if ; then ... master specific actions ... else ... worker specific actions ... fi

LifecycleConfig WorkflowTemplatePlacementManagedClusterConfigLifecycleConfig

Optional. Lifecycle setting for the cluster.

MasterConfig WorkflowTemplatePlacementManagedClusterConfigMasterConfig

Optional. The Compute Engine config settings for additional worker instances in a cluster.

MetastoreConfig WorkflowTemplatePlacementManagedClusterConfigMetastoreConfig

Optional. Metastore configuration.

SecondaryWorkerConfig WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfig

Optional. The Compute Engine config settings for additional worker instances in a cluster.

SecurityConfig WorkflowTemplatePlacementManagedClusterConfigSecurityConfig

Optional. Security settings for the cluster.

SoftwareConfig WorkflowTemplatePlacementManagedClusterConfigSoftwareConfig

Optional. The config settings for software inside the cluster.

StagingBucket string

Optional. A Cloud Storage bucket used to stage job dependencies, config files, and job driver console output. If you do not specify a staging bucket, Cloud Dataproc will determine a Cloud Storage location (US, ASIA, or EU) for your cluster's staging bucket according to the Compute Engine zone where your cluster is deployed, and then create and manage this project-level, per-location bucket (see (https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/staging-bucket)).

TempBucket string

Optional. A Cloud Storage bucket used to store ephemeral cluster and jobs data, such as Spark and MapReduce history files. If you do not specify a temp bucket, Dataproc will determine a Cloud Storage location (US, ASIA, or EU) for your cluster's temp bucket according to the Compute Engine zone where your cluster is deployed, and then create and manage this project-level, per-location bucket. The default bucket has a TTL of 90 days, but you can use any TTL (or none) if you specify a bucket.

WorkerConfig WorkflowTemplatePlacementManagedClusterConfigWorkerConfig

Optional. The Compute Engine config settings for additional worker instances in a cluster.

autoscalingConfig WorkflowTemplatePlacementManagedClusterConfigAutoscalingConfig

Optional. Autoscaling config for the policy associated with the cluster. Cluster does not autoscale if this field is unset.

encryptionConfig WorkflowTemplatePlacementManagedClusterConfigEncryptionConfig

Optional. Encryption settings for the cluster.

endpointConfig WorkflowTemplatePlacementManagedClusterConfigEndpointConfig

Optional. Port/endpoint configuration for this cluster

gceClusterConfig WorkflowTemplatePlacementManagedClusterConfigGceClusterConfig

Optional. The shared Compute Engine config settings for all instances in a cluster.

gkeClusterConfig WorkflowTemplatePlacementManagedClusterConfigGkeClusterConfig

Optional. The Kubernetes Engine config for Dataproc clusters deployed to Kubernetes. Setting this is considered mutually exclusive with Compute Engine-based options such as gce_cluster_config, master_config, worker_config, secondary_worker_config, and autoscaling_config.

initializationActions List<WorkflowTemplatePlacementManagedClusterConfigInitializationAction>

Optional. Commands to execute on each node after config is completed. By default, executables are run on master and all worker nodes. You can test a node's role metadata to run an executable on a master or worker node, as shown below using curl (you can also use wget): ROLE=$(curl -H Metadata-Flavor:Google http://metadata/computeMetadata/v1/instance/attributes/dataproc-role) if ; then ... master specific actions ... else ... worker specific actions ... fi

lifecycleConfig WorkflowTemplatePlacementManagedClusterConfigLifecycleConfig

Optional. Lifecycle setting for the cluster.

masterConfig WorkflowTemplatePlacementManagedClusterConfigMasterConfig

Optional. The Compute Engine config settings for additional worker instances in a cluster.

metastoreConfig WorkflowTemplatePlacementManagedClusterConfigMetastoreConfig

Optional. Metastore configuration.

secondaryWorkerConfig WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfig

Optional. The Compute Engine config settings for additional worker instances in a cluster.

securityConfig WorkflowTemplatePlacementManagedClusterConfigSecurityConfig

Optional. Security settings for the cluster.

softwareConfig WorkflowTemplatePlacementManagedClusterConfigSoftwareConfig

Optional. The config settings for software inside the cluster.

stagingBucket String

Optional. A Cloud Storage bucket used to stage job dependencies, config files, and job driver console output. If you do not specify a staging bucket, Cloud Dataproc will determine a Cloud Storage location (US, ASIA, or EU) for your cluster's staging bucket according to the Compute Engine zone where your cluster is deployed, and then create and manage this project-level, per-location bucket (see (https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/staging-bucket)).

tempBucket String

Optional. A Cloud Storage bucket used to store ephemeral cluster and jobs data, such as Spark and MapReduce history files. If you do not specify a temp bucket, Dataproc will determine a Cloud Storage location (US, ASIA, or EU) for your cluster's temp bucket according to the Compute Engine zone where your cluster is deployed, and then create and manage this project-level, per-location bucket. The default bucket has a TTL of 90 days, but you can use any TTL (or none) if you specify a bucket.

workerConfig WorkflowTemplatePlacementManagedClusterConfigWorkerConfig

Optional. The Compute Engine config settings for additional worker instances in a cluster.

autoscalingConfig WorkflowTemplatePlacementManagedClusterConfigAutoscalingConfig

Optional. Autoscaling config for the policy associated with the cluster. Cluster does not autoscale if this field is unset.

encryptionConfig WorkflowTemplatePlacementManagedClusterConfigEncryptionConfig

Optional. Encryption settings for the cluster.

endpointConfig WorkflowTemplatePlacementManagedClusterConfigEndpointConfig

Optional. Port/endpoint configuration for this cluster

gceClusterConfig WorkflowTemplatePlacementManagedClusterConfigGceClusterConfig

Optional. The shared Compute Engine config settings for all instances in a cluster.

gkeClusterConfig WorkflowTemplatePlacementManagedClusterConfigGkeClusterConfig

Optional. The Kubernetes Engine config for Dataproc clusters deployed to Kubernetes. Setting this is considered mutually exclusive with Compute Engine-based options such as gce_cluster_config, master_config, worker_config, secondary_worker_config, and autoscaling_config.

initializationActions WorkflowTemplatePlacementManagedClusterConfigInitializationAction[]

Optional. Commands to execute on each node after config is completed. By default, executables are run on master and all worker nodes. You can test a node's role metadata to run an executable on a master or worker node, as shown below using curl (you can also use wget): ROLE=$(curl -H Metadata-Flavor:Google http://metadata/computeMetadata/v1/instance/attributes/dataproc-role) if ; then ... master specific actions ... else ... worker specific actions ... fi

lifecycleConfig WorkflowTemplatePlacementManagedClusterConfigLifecycleConfig

Optional. Lifecycle setting for the cluster.

masterConfig WorkflowTemplatePlacementManagedClusterConfigMasterConfig

Optional. The Compute Engine config settings for additional worker instances in a cluster.

metastoreConfig WorkflowTemplatePlacementManagedClusterConfigMetastoreConfig

Optional. Metastore configuration.

secondaryWorkerConfig WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfig

Optional. The Compute Engine config settings for additional worker instances in a cluster.

securityConfig WorkflowTemplatePlacementManagedClusterConfigSecurityConfig

Optional. Security settings for the cluster.

softwareConfig WorkflowTemplatePlacementManagedClusterConfigSoftwareConfig

Optional. The config settings for software inside the cluster.

stagingBucket string

Optional. A Cloud Storage bucket used to stage job dependencies, config files, and job driver console output. If you do not specify a staging bucket, Cloud Dataproc will determine a Cloud Storage location (US, ASIA, or EU) for your cluster's staging bucket according to the Compute Engine zone where your cluster is deployed, and then create and manage this project-level, per-location bucket (see (https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/staging-bucket)).

tempBucket string

Optional. A Cloud Storage bucket used to store ephemeral cluster and jobs data, such as Spark and MapReduce history files. If you do not specify a temp bucket, Dataproc will determine a Cloud Storage location (US, ASIA, or EU) for your cluster's temp bucket according to the Compute Engine zone where your cluster is deployed, and then create and manage this project-level, per-location bucket. The default bucket has a TTL of 90 days, but you can use any TTL (or none) if you specify a bucket.

workerConfig WorkflowTemplatePlacementManagedClusterConfigWorkerConfig

Optional. The Compute Engine config settings for additional worker instances in a cluster.

autoscaling_config WorkflowTemplatePlacementManagedClusterConfigAutoscalingConfig

Optional. Autoscaling config for the policy associated with the cluster. Cluster does not autoscale if this field is unset.

encryption_config WorkflowTemplatePlacementManagedClusterConfigEncryptionConfig

Optional. Encryption settings for the cluster.

endpoint_config WorkflowTemplatePlacementManagedClusterConfigEndpointConfig

Optional. Port/endpoint configuration for this cluster

gce_cluster_config WorkflowTemplatePlacementManagedClusterConfigGceClusterConfig

Optional. The shared Compute Engine config settings for all instances in a cluster.

gke_cluster_config WorkflowTemplatePlacementManagedClusterConfigGkeClusterConfig

Optional. The Kubernetes Engine config for Dataproc clusters deployed to Kubernetes. Setting this is considered mutually exclusive with Compute Engine-based options such as gce_cluster_config, master_config, worker_config, secondary_worker_config, and autoscaling_config.

initialization_actions Sequence[WorkflowTemplatePlacementManagedClusterConfigInitializationAction]

Optional. Commands to execute on each node after config is completed. By default, executables are run on master and all worker nodes. You can test a node's role metadata to run an executable on a master or worker node, as shown below using curl (you can also use wget): ROLE=$(curl -H Metadata-Flavor:Google http://metadata/computeMetadata/v1/instance/attributes/dataproc-role) if ; then ... master specific actions ... else ... worker specific actions ... fi

lifecycle_config WorkflowTemplatePlacementManagedClusterConfigLifecycleConfig

Optional. Lifecycle setting for the cluster.

master_config WorkflowTemplatePlacementManagedClusterConfigMasterConfig

Optional. The Compute Engine config settings for additional worker instances in a cluster.

metastore_config WorkflowTemplatePlacementManagedClusterConfigMetastoreConfig

Optional. Metastore configuration.

secondary_worker_config WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfig

Optional. The Compute Engine config settings for additional worker instances in a cluster.

security_config WorkflowTemplatePlacementManagedClusterConfigSecurityConfig

Optional. Security settings for the cluster.

software_config WorkflowTemplatePlacementManagedClusterConfigSoftwareConfig

Optional. The config settings for software inside the cluster.

staging_bucket str

Optional. A Cloud Storage bucket used to stage job dependencies, config files, and job driver console output. If you do not specify a staging bucket, Cloud Dataproc will determine a Cloud Storage location (US, ASIA, or EU) for your cluster's staging bucket according to the Compute Engine zone where your cluster is deployed, and then create and manage this project-level, per-location bucket (see (https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/staging-bucket)).

temp_bucket str

Optional. A Cloud Storage bucket used to store ephemeral cluster and jobs data, such as Spark and MapReduce history files. If you do not specify a temp bucket, Dataproc will determine a Cloud Storage location (US, ASIA, or EU) for your cluster's temp bucket according to the Compute Engine zone where your cluster is deployed, and then create and manage this project-level, per-location bucket. The default bucket has a TTL of 90 days, but you can use any TTL (or none) if you specify a bucket.

worker_config WorkflowTemplatePlacementManagedClusterConfigWorkerConfig

Optional. The Compute Engine config settings for additional worker instances in a cluster.

autoscalingConfig Property Map

Optional. Autoscaling config for the policy associated with the cluster. Cluster does not autoscale if this field is unset.

encryptionConfig Property Map

Optional. Encryption settings for the cluster.

endpointConfig Property Map

Optional. Port/endpoint configuration for this cluster

gceClusterConfig Property Map

Optional. The shared Compute Engine config settings for all instances in a cluster.

gkeClusterConfig Property Map

Optional. The Kubernetes Engine config for Dataproc clusters deployed to Kubernetes. Setting this is considered mutually exclusive with Compute Engine-based options such as gce_cluster_config, master_config, worker_config, secondary_worker_config, and autoscaling_config.

initializationActions List<Property Map>

Optional. Commands to execute on each node after config is completed. By default, executables are run on master and all worker nodes. You can test a node's role metadata to run an executable on a master or worker node, as shown below using curl (you can also use wget): ROLE=$(curl -H Metadata-Flavor:Google http://metadata/computeMetadata/v1/instance/attributes/dataproc-role) if ; then ... master specific actions ... else ... worker specific actions ... fi

lifecycleConfig Property Map

Optional. Lifecycle setting for the cluster.

masterConfig Property Map

Optional. The Compute Engine config settings for additional worker instances in a cluster.

metastoreConfig Property Map

Optional. Metastore configuration.

secondaryWorkerConfig Property Map

Optional. The Compute Engine config settings for additional worker instances in a cluster.

securityConfig Property Map

Optional. Security settings for the cluster.

softwareConfig Property Map

Optional. The config settings for software inside the cluster.

stagingBucket String

Optional. A Cloud Storage bucket used to stage job dependencies, config files, and job driver console output. If you do not specify a staging bucket, Cloud Dataproc will determine a Cloud Storage location (US, ASIA, or EU) for your cluster's staging bucket according to the Compute Engine zone where your cluster is deployed, and then create and manage this project-level, per-location bucket (see (https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/staging-bucket)).

tempBucket String

Optional. A Cloud Storage bucket used to store ephemeral cluster and jobs data, such as Spark and MapReduce history files. If you do not specify a temp bucket, Dataproc will determine a Cloud Storage location (US, ASIA, or EU) for your cluster's temp bucket according to the Compute Engine zone where your cluster is deployed, and then create and manage this project-level, per-location bucket. The default bucket has a TTL of 90 days, but you can use any TTL (or none) if you specify a bucket.

workerConfig Property Map

Optional. The Compute Engine config settings for additional worker instances in a cluster.

WorkflowTemplatePlacementManagedClusterConfigAutoscalingConfig

Policy string

Optional. The autoscaling policy used by the cluster. Only resource names including projectid and location (region) are valid. Examples: * https://www.googleapis.com/compute/v1/projects/ Note that the policy must be in the same project and Dataproc region.

Policy string

Optional. The autoscaling policy used by the cluster. Only resource names including projectid and location (region) are valid. Examples: * https://www.googleapis.com/compute/v1/projects/ Note that the policy must be in the same project and Dataproc region.

policy String

Optional. The autoscaling policy used by the cluster. Only resource names including projectid and location (region) are valid. Examples: * https://www.googleapis.com/compute/v1/projects/ Note that the policy must be in the same project and Dataproc region.

policy string

Optional. The autoscaling policy used by the cluster. Only resource names including projectid and location (region) are valid. Examples: * https://www.googleapis.com/compute/v1/projects/ Note that the policy must be in the same project and Dataproc region.

policy str

Optional. The autoscaling policy used by the cluster. Only resource names including projectid and location (region) are valid. Examples: * https://www.googleapis.com/compute/v1/projects/ Note that the policy must be in the same project and Dataproc region.

policy String

Optional. The autoscaling policy used by the cluster. Only resource names including projectid and location (region) are valid. Examples: * https://www.googleapis.com/compute/v1/projects/ Note that the policy must be in the same project and Dataproc region.

WorkflowTemplatePlacementManagedClusterConfigEncryptionConfig

GcePdKmsKeyName string

Optional. The Cloud KMS key name to use for PD disk encryption for all instances in the cluster.

GcePdKmsKeyName string

Optional. The Cloud KMS key name to use for PD disk encryption for all instances in the cluster.

gcePdKmsKeyName String

Optional. The Cloud KMS key name to use for PD disk encryption for all instances in the cluster.

gcePdKmsKeyName string

Optional. The Cloud KMS key name to use for PD disk encryption for all instances in the cluster.

gce_pd_kms_key_name str

Optional. The Cloud KMS key name to use for PD disk encryption for all instances in the cluster.

gcePdKmsKeyName String

Optional. The Cloud KMS key name to use for PD disk encryption for all instances in the cluster.

WorkflowTemplatePlacementManagedClusterConfigEndpointConfig

EnableHttpPortAccess bool

Optional. If true, enable http access to specific ports on the cluster from external sources. Defaults to false.

HttpPorts Dictionary<string, string>

Output only. The map of port descriptions to URLs. Will only be populated if enable_http_port_access is true.

EnableHttpPortAccess bool

Optional. If true, enable http access to specific ports on the cluster from external sources. Defaults to false.

HttpPorts map[string]string

Output only. The map of port descriptions to URLs. Will only be populated if enable_http_port_access is true.

enableHttpPortAccess Boolean

Optional. If true, enable http access to specific ports on the cluster from external sources. Defaults to false.

httpPorts Map<String,String>

Output only. The map of port descriptions to URLs. Will only be populated if enable_http_port_access is true.

enableHttpPortAccess boolean

Optional. If true, enable http access to specific ports on the cluster from external sources. Defaults to false.

httpPorts {[key: string]: string}

Output only. The map of port descriptions to URLs. Will only be populated if enable_http_port_access is true.

enable_http_port_access bool

Optional. If true, enable http access to specific ports on the cluster from external sources. Defaults to false.

http_ports Mapping[str, str]

Output only. The map of port descriptions to URLs. Will only be populated if enable_http_port_access is true.

enableHttpPortAccess Boolean

Optional. If true, enable http access to specific ports on the cluster from external sources. Defaults to false.

httpPorts Map<String>

Output only. The map of port descriptions to URLs. Will only be populated if enable_http_port_access is true.

WorkflowTemplatePlacementManagedClusterConfigGceClusterConfig

InternalIpOnly bool

Optional. If true, all instances in the cluster will only have internal IP addresses. By default, clusters are not restricted to internal IP addresses, and will have ephemeral external IP addresses assigned to each instance. This internal_ip_only restriction can only be enabled for subnetwork enabled networks, and all off-cluster dependencies must be configured to be accessible without external IP addresses.

Metadata Dictionary<string, string>

The Compute Engine metadata entries to add to all instances (see (https://cloud.google.com/compute/docs/storing-retrieving-metadata#project_and_instance_metadata)).

Network string

Optional. The Compute Engine network to be used for machine communications. Cannot be specified with subnetwork_uri. If neither network_uri nor subnetwork_uri is specified, the "default" network of the project is used, if it exists. Cannot be a "Custom Subnet Network" (see /regions/global/default*default`

NodeGroupAffinity WorkflowTemplatePlacementManagedClusterConfigGceClusterConfigNodeGroupAffinity

Optional. Node Group Affinity for sole-tenant clusters.

PrivateIpv6GoogleAccess string

Optional. The type of IPv6 access for a cluster. Possible values: PRIVATE_IPV6_GOOGLE_ACCESS_UNSPECIFIED, INHERIT_FROM_SUBNETWORK, OUTBOUND, BIDIRECTIONAL

ReservationAffinity WorkflowTemplatePlacementManagedClusterConfigGceClusterConfigReservationAffinity

Optional. Reservation Affinity for consuming Zonal reservation.

ServiceAccount string

Optional. The (https://cloud.google.com/compute/docs/access/service-accounts#default_service_account) is used.

ServiceAccountScopes List<string>

Optional. The URIs of service account scopes to be included in Compute Engine instances. The following base set of scopes is always included: * https://www.googleapis.com/auth/cloud.useraccounts.readonly * https://www.googleapis.com/auth/devstorage.read_write * https://www.googleapis.com/auth/logging.write If no scopes are specified, the following defaults are also provided: * https://www.googleapis.com/auth/bigquery * https://www.googleapis.com/auth/bigtable.admin.table * https://www.googleapis.com/auth/bigtable.data * https://www.googleapis.com/auth/devstorage.full_control

Subnetwork string

Optional. The Compute Engine subnetwork to be used for machine communications. Cannot be specified with network_uri. A full URL, partial URI, or short name are valid. Examples: * https://www.googleapis.com/compute/v1/projects//regions/us-east1/subnetworks/sub0 * sub0

Tags List<string>

The Compute Engine tags to add to all instances (see (https://cloud.google.com/compute/docs/label-or-tag-resources#tags)).

Zone string

Optional. The zone where the Compute Engine cluster will be located. On a create request, it is required in the "global" region. If omitted in a non-global Dataproc region, the service will pick a zone in the corresponding Compute Engine region. On a get request, zone will always be present. A full URL, partial URI, or short name are valid. Examples: * https://www.googleapis.com/compute/v1/projects/ * us-central1-f

InternalIpOnly bool

Optional. If true, all instances in the cluster will only have internal IP addresses. By default, clusters are not restricted to internal IP addresses, and will have ephemeral external IP addresses assigned to each instance. This internal_ip_only restriction can only be enabled for subnetwork enabled networks, and all off-cluster dependencies must be configured to be accessible without external IP addresses.

Metadata map[string]string

The Compute Engine metadata entries to add to all instances (see (https://cloud.google.com/compute/docs/storing-retrieving-metadata#project_and_instance_metadata)).

Network string

Optional. The Compute Engine network to be used for machine communications. Cannot be specified with subnetwork_uri. If neither network_uri nor subnetwork_uri is specified, the "default" network of the project is used, if it exists. Cannot be a "Custom Subnet Network" (see /regions/global/default*default`

NodeGroupAffinity WorkflowTemplatePlacementManagedClusterConfigGceClusterConfigNodeGroupAffinity

Optional. Node Group Affinity for sole-tenant clusters.

PrivateIpv6GoogleAccess string

Optional. The type of IPv6 access for a cluster. Possible values: PRIVATE_IPV6_GOOGLE_ACCESS_UNSPECIFIED, INHERIT_FROM_SUBNETWORK, OUTBOUND, BIDIRECTIONAL

ReservationAffinity WorkflowTemplatePlacementManagedClusterConfigGceClusterConfigReservationAffinity

Optional. Reservation Affinity for consuming Zonal reservation.

ServiceAccount string

Optional. The (https://cloud.google.com/compute/docs/access/service-accounts#default_service_account) is used.

ServiceAccountScopes []string

Optional. The URIs of service account scopes to be included in Compute Engine instances. The following base set of scopes is always included: * https://www.googleapis.com/auth/cloud.useraccounts.readonly * https://www.googleapis.com/auth/devstorage.read_write * https://www.googleapis.com/auth/logging.write If no scopes are specified, the following defaults are also provided: * https://www.googleapis.com/auth/bigquery * https://www.googleapis.com/auth/bigtable.admin.table * https://www.googleapis.com/auth/bigtable.data * https://www.googleapis.com/auth/devstorage.full_control

Subnetwork string

Optional. The Compute Engine subnetwork to be used for machine communications. Cannot be specified with network_uri. A full URL, partial URI, or short name are valid. Examples: * https://www.googleapis.com/compute/v1/projects//regions/us-east1/subnetworks/sub0 * sub0

Tags []string

The Compute Engine tags to add to all instances (see (https://cloud.google.com/compute/docs/label-or-tag-resources#tags)).

Zone string

Optional. The zone where the Compute Engine cluster will be located. On a create request, it is required in the "global" region. If omitted in a non-global Dataproc region, the service will pick a zone in the corresponding Compute Engine region. On a get request, zone will always be present. A full URL, partial URI, or short name are valid. Examples: * https://www.googleapis.com/compute/v1/projects/ * us-central1-f

internalIpOnly Boolean

Optional. If true, all instances in the cluster will only have internal IP addresses. By default, clusters are not restricted to internal IP addresses, and will have ephemeral external IP addresses assigned to each instance. This internal_ip_only restriction can only be enabled for subnetwork enabled networks, and all off-cluster dependencies must be configured to be accessible without external IP addresses.

metadata Map<String,String>

The Compute Engine metadata entries to add to all instances (see (https://cloud.google.com/compute/docs/storing-retrieving-metadata#project_and_instance_metadata)).

network String

Optional. The Compute Engine network to be used for machine communications. Cannot be specified with subnetwork_uri. If neither network_uri nor subnetwork_uri is specified, the "default" network of the project is used, if it exists. Cannot be a "Custom Subnet Network" (see /regions/global/default*default`

nodeGroupAffinity WorkflowTemplatePlacementManagedClusterConfigGceClusterConfigNodeGroupAffinity

Optional. Node Group Affinity for sole-tenant clusters.

privateIpv6GoogleAccess String

Optional. The type of IPv6 access for a cluster. Possible values: PRIVATE_IPV6_GOOGLE_ACCESS_UNSPECIFIED, INHERIT_FROM_SUBNETWORK, OUTBOUND, BIDIRECTIONAL

reservationAffinity WorkflowTemplatePlacementManagedClusterConfigGceClusterConfigReservationAffinity

Optional. Reservation Affinity for consuming Zonal reservation.

serviceAccount String

Optional. The (https://cloud.google.com/compute/docs/access/service-accounts#default_service_account) is used.

serviceAccountScopes List<String>

Optional. The URIs of service account scopes to be included in Compute Engine instances. The following base set of scopes is always included: * https://www.googleapis.com/auth/cloud.useraccounts.readonly * https://www.googleapis.com/auth/devstorage.read_write * https://www.googleapis.com/auth/logging.write If no scopes are specified, the following defaults are also provided: * https://www.googleapis.com/auth/bigquery * https://www.googleapis.com/auth/bigtable.admin.table * https://www.googleapis.com/auth/bigtable.data * https://www.googleapis.com/auth/devstorage.full_control

subnetwork String

Optional. The Compute Engine subnetwork to be used for machine communications. Cannot be specified with network_uri. A full URL, partial URI, or short name are valid. Examples: * https://www.googleapis.com/compute/v1/projects//regions/us-east1/subnetworks/sub0 * sub0

tags List<String>

The Compute Engine tags to add to all instances (see (https://cloud.google.com/compute/docs/label-or-tag-resources#tags)).

zone String

Optional. The zone where the Compute Engine cluster will be located. On a create request, it is required in the "global" region. If omitted in a non-global Dataproc region, the service will pick a zone in the corresponding Compute Engine region. On a get request, zone will always be present. A full URL, partial URI, or short name are valid. Examples: * https://www.googleapis.com/compute/v1/projects/ * us-central1-f

internalIpOnly boolean

Optional. If true, all instances in the cluster will only have internal IP addresses. By default, clusters are not restricted to internal IP addresses, and will have ephemeral external IP addresses assigned to each instance. This internal_ip_only restriction can only be enabled for subnetwork enabled networks, and all off-cluster dependencies must be configured to be accessible without external IP addresses.

metadata {[key: string]: string}

The Compute Engine metadata entries to add to all instances (see (https://cloud.google.com/compute/docs/storing-retrieving-metadata#project_and_instance_metadata)).

network string

Optional. The Compute Engine network to be used for machine communications. Cannot be specified with subnetwork_uri. If neither network_uri nor subnetwork_uri is specified, the "default" network of the project is used, if it exists. Cannot be a "Custom Subnet Network" (see /regions/global/default*default`

nodeGroupAffinity WorkflowTemplatePlacementManagedClusterConfigGceClusterConfigNodeGroupAffinity

Optional. Node Group Affinity for sole-tenant clusters.

privateIpv6GoogleAccess string

Optional. The type of IPv6 access for a cluster. Possible values: PRIVATE_IPV6_GOOGLE_ACCESS_UNSPECIFIED, INHERIT_FROM_SUBNETWORK, OUTBOUND, BIDIRECTIONAL

reservationAffinity WorkflowTemplatePlacementManagedClusterConfigGceClusterConfigReservationAffinity

Optional. Reservation Affinity for consuming Zonal reservation.

serviceAccount string

Optional. The (https://cloud.google.com/compute/docs/access/service-accounts#default_service_account) is used.

serviceAccountScopes string[]

Optional. The URIs of service account scopes to be included in Compute Engine instances. The following base set of scopes is always included: * https://www.googleapis.com/auth/cloud.useraccounts.readonly * https://www.googleapis.com/auth/devstorage.read_write * https://www.googleapis.com/auth/logging.write If no scopes are specified, the following defaults are also provided: * https://www.googleapis.com/auth/bigquery * https://www.googleapis.com/auth/bigtable.admin.table * https://www.googleapis.com/auth/bigtable.data * https://www.googleapis.com/auth/devstorage.full_control

subnetwork string

Optional. The Compute Engine subnetwork to be used for machine communications. Cannot be specified with network_uri. A full URL, partial URI, or short name are valid. Examples: * https://www.googleapis.com/compute/v1/projects//regions/us-east1/subnetworks/sub0 * sub0

tags string[]

The Compute Engine tags to add to all instances (see (https://cloud.google.com/compute/docs/label-or-tag-resources#tags)).

zone string

Optional. The zone where the Compute Engine cluster will be located. On a create request, it is required in the "global" region. If omitted in a non-global Dataproc region, the service will pick a zone in the corresponding Compute Engine region. On a get request, zone will always be present. A full URL, partial URI, or short name are valid. Examples: * https://www.googleapis.com/compute/v1/projects/ * us-central1-f

internal_ip_only bool

Optional. If true, all instances in the cluster will only have internal IP addresses. By default, clusters are not restricted to internal IP addresses, and will have ephemeral external IP addresses assigned to each instance. This internal_ip_only restriction can only be enabled for subnetwork enabled networks, and all off-cluster dependencies must be configured to be accessible without external IP addresses.

metadata Mapping[str, str]

The Compute Engine metadata entries to add to all instances (see (https://cloud.google.com/compute/docs/storing-retrieving-metadata#project_and_instance_metadata)).

network str

Optional. The Compute Engine network to be used for machine communications. Cannot be specified with subnetwork_uri. If neither network_uri nor subnetwork_uri is specified, the "default" network of the project is used, if it exists. Cannot be a "Custom Subnet Network" (see /regions/global/default*default`

node_group_affinity WorkflowTemplatePlacementManagedClusterConfigGceClusterConfigNodeGroupAffinity

Optional. Node Group Affinity for sole-tenant clusters.

private_ipv6_google_access str

Optional. The type of IPv6 access for a cluster. Possible values: PRIVATE_IPV6_GOOGLE_ACCESS_UNSPECIFIED, INHERIT_FROM_SUBNETWORK, OUTBOUND, BIDIRECTIONAL

reservation_affinity WorkflowTemplatePlacementManagedClusterConfigGceClusterConfigReservationAffinity

Optional. Reservation Affinity for consuming Zonal reservation.

service_account str

Optional. The (https://cloud.google.com/compute/docs/access/service-accounts#default_service_account) is used.

service_account_scopes Sequence[str]

Optional. The URIs of service account scopes to be included in Compute Engine instances. The following base set of scopes is always included: * https://www.googleapis.com/auth/cloud.useraccounts.readonly * https://www.googleapis.com/auth/devstorage.read_write * https://www.googleapis.com/auth/logging.write If no scopes are specified, the following defaults are also provided: * https://www.googleapis.com/auth/bigquery * https://www.googleapis.com/auth/bigtable.admin.table * https://www.googleapis.com/auth/bigtable.data * https://www.googleapis.com/auth/devstorage.full_control

subnetwork str

Optional. The Compute Engine subnetwork to be used for machine communications. Cannot be specified with network_uri. A full URL, partial URI, or short name are valid. Examples: * https://www.googleapis.com/compute/v1/projects//regions/us-east1/subnetworks/sub0 * sub0

tags Sequence[str]

The Compute Engine tags to add to all instances (see (https://cloud.google.com/compute/docs/label-or-tag-resources#tags)).

zone str

Optional. The zone where the Compute Engine cluster will be located. On a create request, it is required in the "global" region. If omitted in a non-global Dataproc region, the service will pick a zone in the corresponding Compute Engine region. On a get request, zone will always be present. A full URL, partial URI, or short name are valid. Examples: * https://www.googleapis.com/compute/v1/projects/ * us-central1-f

internalIpOnly Boolean

Optional. If true, all instances in the cluster will only have internal IP addresses. By default, clusters are not restricted to internal IP addresses, and will have ephemeral external IP addresses assigned to each instance. This internal_ip_only restriction can only be enabled for subnetwork enabled networks, and all off-cluster dependencies must be configured to be accessible without external IP addresses.

metadata Map<String>

The Compute Engine metadata entries to add to all instances (see (https://cloud.google.com/compute/docs/storing-retrieving-metadata#project_and_instance_metadata)).

network String

Optional. The Compute Engine network to be used for machine communications. Cannot be specified with subnetwork_uri. If neither network_uri nor subnetwork_uri is specified, the "default" network of the project is used, if it exists. Cannot be a "Custom Subnet Network" (see /regions/global/default*default`

nodeGroupAffinity Property Map

Optional. Node Group Affinity for sole-tenant clusters.

privateIpv6GoogleAccess String

Optional. The type of IPv6 access for a cluster. Possible values: PRIVATE_IPV6_GOOGLE_ACCESS_UNSPECIFIED, INHERIT_FROM_SUBNETWORK, OUTBOUND, BIDIRECTIONAL

reservationAffinity Property Map

Optional. Reservation Affinity for consuming Zonal reservation.

serviceAccount String

Optional. The (https://cloud.google.com/compute/docs/access/service-accounts#default_service_account) is used.

serviceAccountScopes List<String>

Optional. The URIs of service account scopes to be included in Compute Engine instances. The following base set of scopes is always included: * https://www.googleapis.com/auth/cloud.useraccounts.readonly * https://www.googleapis.com/auth/devstorage.read_write * https://www.googleapis.com/auth/logging.write If no scopes are specified, the following defaults are also provided: * https://www.googleapis.com/auth/bigquery * https://www.googleapis.com/auth/bigtable.admin.table * https://www.googleapis.com/auth/bigtable.data * https://www.googleapis.com/auth/devstorage.full_control

subnetwork String

Optional. The Compute Engine subnetwork to be used for machine communications. Cannot be specified with network_uri. A full URL, partial URI, or short name are valid. Examples: * https://www.googleapis.com/compute/v1/projects//regions/us-east1/subnetworks/sub0 * sub0

tags List<String>

The Compute Engine tags to add to all instances (see (https://cloud.google.com/compute/docs/label-or-tag-resources#tags)).

zone String

Optional. The zone where the Compute Engine cluster will be located. On a create request, it is required in the "global" region. If omitted in a non-global Dataproc region, the service will pick a zone in the corresponding Compute Engine region. On a get request, zone will always be present. A full URL, partial URI, or short name are valid. Examples: * https://www.googleapis.com/compute/v1/projects/ * us-central1-f

WorkflowTemplatePlacementManagedClusterConfigGceClusterConfigNodeGroupAffinity

NodeGroup string

Required. The URI of a sole-tenant /zones/us-central1-a/nodeGroups/node-group-1*node-group-1`

NodeGroup string

Required. The URI of a sole-tenant /zones/us-central1-a/nodeGroups/node-group-1*node-group-1`

nodeGroup String

Required. The URI of a sole-tenant /zones/us-central1-a/nodeGroups/node-group-1*node-group-1`

nodeGroup string

Required. The URI of a sole-tenant /zones/us-central1-a/nodeGroups/node-group-1*node-group-1`

node_group str

Required. The URI of a sole-tenant /zones/us-central1-a/nodeGroups/node-group-1*node-group-1`

nodeGroup String

Required. The URI of a sole-tenant /zones/us-central1-a/nodeGroups/node-group-1*node-group-1`

WorkflowTemplatePlacementManagedClusterConfigGceClusterConfigReservationAffinity

ConsumeReservationType string

Optional. Type of reservation to consume Possible values: TYPE_UNSPECIFIED, NO_RESERVATION, ANY_RESERVATION, SPECIFIC_RESERVATION

Key string

Optional. Corresponds to the label key of reservation resource.

Values List<string>

Optional. Corresponds to the label values of reservation resource.

ConsumeReservationType string

Optional. Type of reservation to consume Possible values: TYPE_UNSPECIFIED, NO_RESERVATION, ANY_RESERVATION, SPECIFIC_RESERVATION

Key string

Optional. Corresponds to the label key of reservation resource.

Values []string

Optional. Corresponds to the label values of reservation resource.

consumeReservationType String

Optional. Type of reservation to consume Possible values: TYPE_UNSPECIFIED, NO_RESERVATION, ANY_RESERVATION, SPECIFIC_RESERVATION

key String

Optional. Corresponds to the label key of reservation resource.

values List<String>

Optional. Corresponds to the label values of reservation resource.

consumeReservationType string

Optional. Type of reservation to consume Possible values: TYPE_UNSPECIFIED, NO_RESERVATION, ANY_RESERVATION, SPECIFIC_RESERVATION

key string

Optional. Corresponds to the label key of reservation resource.

values string[]

Optional. Corresponds to the label values of reservation resource.

consume_reservation_type str

Optional. Type of reservation to consume Possible values: TYPE_UNSPECIFIED, NO_RESERVATION, ANY_RESERVATION, SPECIFIC_RESERVATION

key str

Optional. Corresponds to the label key of reservation resource.

values Sequence[str]

Optional. Corresponds to the label values of reservation resource.

consumeReservationType String

Optional. Type of reservation to consume Possible values: TYPE_UNSPECIFIED, NO_RESERVATION, ANY_RESERVATION, SPECIFIC_RESERVATION

key String

Optional. Corresponds to the label key of reservation resource.

values List<String>

Optional. Corresponds to the label values of reservation resource.

WorkflowTemplatePlacementManagedClusterConfigGkeClusterConfig

namespacedGkeDeploymentTarget Property Map

Optional. A target for the deployment.

WorkflowTemplatePlacementManagedClusterConfigGkeClusterConfigNamespacedGkeDeploymentTarget

ClusterNamespace string

Optional. A namespace within the GKE cluster to deploy into.

TargetGkeCluster string

Optional. The target GKE cluster to deploy to. Format: 'projects/{project}/locations/{location}/clusters/{cluster_id}'

ClusterNamespace string

Optional. A namespace within the GKE cluster to deploy into.

TargetGkeCluster string

Optional. The target GKE cluster to deploy to. Format: 'projects/{project}/locations/{location}/clusters/{cluster_id}'

clusterNamespace String

Optional. A namespace within the GKE cluster to deploy into.

targetGkeCluster String

Optional. The target GKE cluster to deploy to. Format: 'projects/{project}/locations/{location}/clusters/{cluster_id}'

clusterNamespace string

Optional. A namespace within the GKE cluster to deploy into.

targetGkeCluster string

Optional. The target GKE cluster to deploy to. Format: 'projects/{project}/locations/{location}/clusters/{cluster_id}'

cluster_namespace str

Optional. A namespace within the GKE cluster to deploy into.

target_gke_cluster str

Optional. The target GKE cluster to deploy to. Format: 'projects/{project}/locations/{location}/clusters/{cluster_id}'

clusterNamespace String

Optional. A namespace within the GKE cluster to deploy into.

targetGkeCluster String

Optional. The target GKE cluster to deploy to. Format: 'projects/{project}/locations/{location}/clusters/{cluster_id}'

WorkflowTemplatePlacementManagedClusterConfigInitializationAction

ExecutableFile string

Required. Cloud Storage URI of executable file.

ExecutionTimeout string

Optional. Amount of time executable has to complete. Default is 10 minutes (see JSON representation of (https://developers.google.com/protocol-buffers/docs/proto3#json)). Cluster creation fails with an explanatory error message (the name of the executable that caused the error and the exceeded timeout period) if the executable is not completed at end of the timeout period.

ExecutableFile string

Required. Cloud Storage URI of executable file.

ExecutionTimeout string

Optional. Amount of time executable has to complete. Default is 10 minutes (see JSON representation of (https://developers.google.com/protocol-buffers/docs/proto3#json)). Cluster creation fails with an explanatory error message (the name of the executable that caused the error and the exceeded timeout period) if the executable is not completed at end of the timeout period.

executableFile String

Required. Cloud Storage URI of executable file.

executionTimeout String

Optional. Amount of time executable has to complete. Default is 10 minutes (see JSON representation of (https://developers.google.com/protocol-buffers/docs/proto3#json)). Cluster creation fails with an explanatory error message (the name of the executable that caused the error and the exceeded timeout period) if the executable is not completed at end of the timeout period.

executableFile string

Required. Cloud Storage URI of executable file.

executionTimeout string

Optional. Amount of time executable has to complete. Default is 10 minutes (see JSON representation of (https://developers.google.com/protocol-buffers/docs/proto3#json)). Cluster creation fails with an explanatory error message (the name of the executable that caused the error and the exceeded timeout period) if the executable is not completed at end of the timeout period.

executable_file str

Required. Cloud Storage URI of executable file.

execution_timeout str

Optional. Amount of time executable has to complete. Default is 10 minutes (see JSON representation of (https://developers.google.com/protocol-buffers/docs/proto3#json)). Cluster creation fails with an explanatory error message (the name of the executable that caused the error and the exceeded timeout period) if the executable is not completed at end of the timeout period.

executableFile String

Required. Cloud Storage URI of executable file.

executionTimeout String

Optional. Amount of time executable has to complete. Default is 10 minutes (see JSON representation of (https://developers.google.com/protocol-buffers/docs/proto3#json)). Cluster creation fails with an explanatory error message (the name of the executable that caused the error and the exceeded timeout period) if the executable is not completed at end of the timeout period.

WorkflowTemplatePlacementManagedClusterConfigLifecycleConfig

AutoDeleteTime string

Optional. The time when cluster will be auto-deleted (see JSON representation of (https://developers.google.com/protocol-buffers/docs/proto3#json)).

AutoDeleteTtl string

Optional. The lifetime duration of cluster. The cluster will be auto-deleted at the end of this period. Minimum value is 10 minutes; maximum value is 14 days (see JSON representation of (https://developers.google.com/protocol-buffers/docs/proto3#json)).

IdleDeleteTtl string

Optional. The duration to keep the cluster alive while idling (when no jobs are running). Passing this threshold will cause the cluster to be deleted. Minimum value is 5 minutes; maximum value is 14 days (see JSON representation of (https://developers.google.com/protocol-buffers/docs/proto3#json).

IdleStartTime string

Output only. The time when cluster became idle (most recent job finished) and became eligible for deletion due to idleness (see JSON representation of (https://developers.google.com/protocol-buffers/docs/proto3#json)).

AutoDeleteTime string

Optional. The time when cluster will be auto-deleted (see JSON representation of (https://developers.google.com/protocol-buffers/docs/proto3#json)).

AutoDeleteTtl string

Optional. The lifetime duration of cluster. The cluster will be auto-deleted at the end of this period. Minimum value is 10 minutes; maximum value is 14 days (see JSON representation of (https://developers.google.com/protocol-buffers/docs/proto3#json)).

IdleDeleteTtl string

Optional. The duration to keep the cluster alive while idling (when no jobs are running). Passing this threshold will cause the cluster to be deleted. Minimum value is 5 minutes; maximum value is 14 days (see JSON representation of (https://developers.google.com/protocol-buffers/docs/proto3#json).

IdleStartTime string

Output only. The time when cluster became idle (most recent job finished) and became eligible for deletion due to idleness (see JSON representation of (https://developers.google.com/protocol-buffers/docs/proto3#json)).

autoDeleteTime String

Optional. The time when cluster will be auto-deleted (see JSON representation of (https://developers.google.com/protocol-buffers/docs/proto3#json)).

autoDeleteTtl String

Optional. The lifetime duration of cluster. The cluster will be auto-deleted at the end of this period. Minimum value is 10 minutes; maximum value is 14 days (see JSON representation of (https://developers.google.com/protocol-buffers/docs/proto3#json)).

idleDeleteTtl String

Optional. The duration to keep the cluster alive while idling (when no jobs are running). Passing this threshold will cause the cluster to be deleted. Minimum value is 5 minutes; maximum value is 14 days (see JSON representation of (https://developers.google.com/protocol-buffers/docs/proto3#json).

idleStartTime String

Output only. The time when cluster became idle (most recent job finished) and became eligible for deletion due to idleness (see JSON representation of (https://developers.google.com/protocol-buffers/docs/proto3#json)).

autoDeleteTime string

Optional. The time when cluster will be auto-deleted (see JSON representation of (https://developers.google.com/protocol-buffers/docs/proto3#json)).

autoDeleteTtl string

Optional. The lifetime duration of cluster. The cluster will be auto-deleted at the end of this period. Minimum value is 10 minutes; maximum value is 14 days (see JSON representation of (https://developers.google.com/protocol-buffers/docs/proto3#json)).

idleDeleteTtl string

Optional. The duration to keep the cluster alive while idling (when no jobs are running). Passing this threshold will cause the cluster to be deleted. Minimum value is 5 minutes; maximum value is 14 days (see JSON representation of (https://developers.google.com/protocol-buffers/docs/proto3#json).

idleStartTime string

Output only. The time when cluster became idle (most recent job finished) and became eligible for deletion due to idleness (see JSON representation of (https://developers.google.com/protocol-buffers/docs/proto3#json)).

auto_delete_time str

Optional. The time when cluster will be auto-deleted (see JSON representation of (https://developers.google.com/protocol-buffers/docs/proto3#json)).

auto_delete_ttl str

Optional. The lifetime duration of cluster. The cluster will be auto-deleted at the end of this period. Minimum value is 10 minutes; maximum value is 14 days (see JSON representation of (https://developers.google.com/protocol-buffers/docs/proto3#json)).

idle_delete_ttl str

Optional. The duration to keep the cluster alive while idling (when no jobs are running). Passing this threshold will cause the cluster to be deleted. Minimum value is 5 minutes; maximum value is 14 days (see JSON representation of (https://developers.google.com/protocol-buffers/docs/proto3#json).

idle_start_time str

Output only. The time when cluster became idle (most recent job finished) and became eligible for deletion due to idleness (see JSON representation of (https://developers.google.com/protocol-buffers/docs/proto3#json)).

autoDeleteTime String

Optional. The time when cluster will be auto-deleted (see JSON representation of (https://developers.google.com/protocol-buffers/docs/proto3#json)).

autoDeleteTtl String

Optional. The lifetime duration of cluster. The cluster will be auto-deleted at the end of this period. Minimum value is 10 minutes; maximum value is 14 days (see JSON representation of (https://developers.google.com/protocol-buffers/docs/proto3#json)).

idleDeleteTtl String

Optional. The duration to keep the cluster alive while idling (when no jobs are running). Passing this threshold will cause the cluster to be deleted. Minimum value is 5 minutes; maximum value is 14 days (see JSON representation of (https://developers.google.com/protocol-buffers/docs/proto3#json).

idleStartTime String

Output only. The time when cluster became idle (most recent job finished) and became eligible for deletion due to idleness (see JSON representation of (https://developers.google.com/protocol-buffers/docs/proto3#json)).

WorkflowTemplatePlacementManagedClusterConfigMasterConfig

Accelerators List<WorkflowTemplatePlacementManagedClusterConfigMasterConfigAccelerator>

Optional. The Compute Engine accelerator configuration for these instances.

DiskConfig WorkflowTemplatePlacementManagedClusterConfigMasterConfigDiskConfig

Optional. Disk option config settings.

Image string

Optional. The Compute Engine image resource used for cluster instances. The URI can represent an image or image family. Image examples: * https://www.googleapis.com/compute/beta/projects/ If the URI is unspecified, it will be inferred from SoftwareConfig.image_version or the system default.

InstanceNames List<string>

Output only. The list of instance names. Dataproc derives the names from cluster_name, num_instances, and the instance group.

IsPreemptible bool

Output only. Specifies that this instance group contains preemptible instances.

MachineType string

Optional. The Compute Engine machine type used for cluster instances. A full URL, partial URI, or short name are valid. Examples: * https://www.googleapis.com/compute/v1/projects/(https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement) feature, you must use the short name of the machine type resource, for example, n1-standard-2`.

ManagedGroupConfigs List<WorkflowTemplatePlacementManagedClusterConfigMasterConfigManagedGroupConfig>

Output only. The config for Compute Engine Instance Group Manager that manages this group. This is only used for preemptible instance groups.

MinCpuPlatform string

Optional. Specifies the minimum cpu platform for the Instance Group. See (https://cloud.google.com/dataproc/docs/concepts/compute/dataproc-min-cpu).

NumInstances int

Optional. The number of VM instances in the instance group. For master instance groups, must be set to 1.

Preemptibility string

Optional. Specifies the preemptibility of the instance group. The default value for master and worker groups is NON_PREEMPTIBLE. This default cannot be changed. The default value for secondary instances is PREEMPTIBLE. Possible values: PREEMPTIBILITY_UNSPECIFIED, NON_PREEMPTIBLE, PREEMPTIBLE

Accelerators []WorkflowTemplatePlacementManagedClusterConfigMasterConfigAccelerator

Optional. The Compute Engine accelerator configuration for these instances.

DiskConfig WorkflowTemplatePlacementManagedClusterConfigMasterConfigDiskConfig

Optional. Disk option config settings.

Image string

Optional. The Compute Engine image resource used for cluster instances. The URI can represent an image or image family. Image examples: * https://www.googleapis.com/compute/beta/projects/ If the URI is unspecified, it will be inferred from SoftwareConfig.image_version or the system default.

InstanceNames []string

Output only. The list of instance names. Dataproc derives the names from cluster_name, num_instances, and the instance group.

IsPreemptible bool

Output only. Specifies that this instance group contains preemptible instances.

MachineType string

Optional. The Compute Engine machine type used for cluster instances. A full URL, partial URI, or short name are valid. Examples: * https://www.googleapis.com/compute/v1/projects/(https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement) feature, you must use the short name of the machine type resource, for example, n1-standard-2`.

ManagedGroupConfigs []WorkflowTemplatePlacementManagedClusterConfigMasterConfigManagedGroupConfig

Output only. The config for Compute Engine Instance Group Manager that manages this group. This is only used for preemptible instance groups.

MinCpuPlatform string

Optional. Specifies the minimum cpu platform for the Instance Group. See (https://cloud.google.com/dataproc/docs/concepts/compute/dataproc-min-cpu).

NumInstances int

Optional. The number of VM instances in the instance group. For master instance groups, must be set to 1.

Preemptibility string

Optional. Specifies the preemptibility of the instance group. The default value for master and worker groups is NON_PREEMPTIBLE. This default cannot be changed. The default value for secondary instances is PREEMPTIBLE. Possible values: PREEMPTIBILITY_UNSPECIFIED, NON_PREEMPTIBLE, PREEMPTIBLE

accelerators List<WorkflowTemplatePlacementManagedClusterConfigMasterConfigAccelerator>

Optional. The Compute Engine accelerator configuration for these instances.

diskConfig WorkflowTemplatePlacementManagedClusterConfigMasterConfigDiskConfig

Optional. Disk option config settings.

image String

Optional. The Compute Engine image resource used for cluster instances. The URI can represent an image or image family. Image examples: * https://www.googleapis.com/compute/beta/projects/ If the URI is unspecified, it will be inferred from SoftwareConfig.image_version or the system default.

instanceNames List<String>

Output only. The list of instance names. Dataproc derives the names from cluster_name, num_instances, and the instance group.

isPreemptible Boolean

Output only. Specifies that this instance group contains preemptible instances.

machineType String

Optional. The Compute Engine machine type used for cluster instances. A full URL, partial URI, or short name are valid. Examples: * https://www.googleapis.com/compute/v1/projects/(https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement) feature, you must use the short name of the machine type resource, for example, n1-standard-2`.

managedGroupConfigs List<WorkflowTemplatePlacementManagedClusterConfigMasterConfigManagedGroupConfig>

Output only. The config for Compute Engine Instance Group Manager that manages this group. This is only used for preemptible instance groups.

minCpuPlatform String

Optional. Specifies the minimum cpu platform for the Instance Group. See (https://cloud.google.com/dataproc/docs/concepts/compute/dataproc-min-cpu).

numInstances Integer

Optional. The number of VM instances in the instance group. For master instance groups, must be set to 1.

preemptibility String

Optional. Specifies the preemptibility of the instance group. The default value for master and worker groups is NON_PREEMPTIBLE. This default cannot be changed. The default value for secondary instances is PREEMPTIBLE. Possible values: PREEMPTIBILITY_UNSPECIFIED, NON_PREEMPTIBLE, PREEMPTIBLE

accelerators WorkflowTemplatePlacementManagedClusterConfigMasterConfigAccelerator[]

Optional. The Compute Engine accelerator configuration for these instances.

diskConfig WorkflowTemplatePlacementManagedClusterConfigMasterConfigDiskConfig

Optional. Disk option config settings.

image string

Optional. The Compute Engine image resource used for cluster instances. The URI can represent an image or image family. Image examples: * https://www.googleapis.com/compute/beta/projects/ If the URI is unspecified, it will be inferred from SoftwareConfig.image_version or the system default.

instanceNames string[]

Output only. The list of instance names. Dataproc derives the names from cluster_name, num_instances, and the instance group.

isPreemptible boolean

Output only. Specifies that this instance group contains preemptible instances.

machineType string

Optional. The Compute Engine machine type used for cluster instances. A full URL, partial URI, or short name are valid. Examples: * https://www.googleapis.com/compute/v1/projects/(https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement) feature, you must use the short name of the machine type resource, for example, n1-standard-2`.

managedGroupConfigs WorkflowTemplatePlacementManagedClusterConfigMasterConfigManagedGroupConfig[]

Output only. The config for Compute Engine Instance Group Manager that manages this group. This is only used for preemptible instance groups.

minCpuPlatform string

Optional. Specifies the minimum cpu platform for the Instance Group. See (https://cloud.google.com/dataproc/docs/concepts/compute/dataproc-min-cpu).

numInstances number

Optional. The number of VM instances in the instance group. For master instance groups, must be set to 1.

preemptibility string

Optional. Specifies the preemptibility of the instance group. The default value for master and worker groups is NON_PREEMPTIBLE. This default cannot be changed. The default value for secondary instances is PREEMPTIBLE. Possible values: PREEMPTIBILITY_UNSPECIFIED, NON_PREEMPTIBLE, PREEMPTIBLE

accelerators Sequence[WorkflowTemplatePlacementManagedClusterConfigMasterConfigAccelerator]

Optional. The Compute Engine accelerator configuration for these instances.

disk_config WorkflowTemplatePlacementManagedClusterConfigMasterConfigDiskConfig

Optional. Disk option config settings.

image str

Optional. The Compute Engine image resource used for cluster instances. The URI can represent an image or image family. Image examples: * https://www.googleapis.com/compute/beta/projects/ If the URI is unspecified, it will be inferred from SoftwareConfig.image_version or the system default.

instance_names Sequence[str]

Output only. The list of instance names. Dataproc derives the names from cluster_name, num_instances, and the instance group.

is_preemptible bool

Output only. Specifies that this instance group contains preemptible instances.

machine_type str

Optional. The Compute Engine machine type used for cluster instances. A full URL, partial URI, or short name are valid. Examples: * https://www.googleapis.com/compute/v1/projects/(https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement) feature, you must use the short name of the machine type resource, for example, n1-standard-2`.

managed_group_configs Sequence[WorkflowTemplatePlacementManagedClusterConfigMasterConfigManagedGroupConfig]

Output only. The config for Compute Engine Instance Group Manager that manages this group. This is only used for preemptible instance groups.

min_cpu_platform str

Optional. Specifies the minimum cpu platform for the Instance Group. See (https://cloud.google.com/dataproc/docs/concepts/compute/dataproc-min-cpu).

num_instances int

Optional. The number of VM instances in the instance group. For master instance groups, must be set to 1.

preemptibility str

Optional. Specifies the preemptibility of the instance group. The default value for master and worker groups is NON_PREEMPTIBLE. This default cannot be changed. The default value for secondary instances is PREEMPTIBLE. Possible values: PREEMPTIBILITY_UNSPECIFIED, NON_PREEMPTIBLE, PREEMPTIBLE

accelerators List<Property Map>

Optional. The Compute Engine accelerator configuration for these instances.

diskConfig Property Map

Optional. Disk option config settings.

image String

Optional. The Compute Engine image resource used for cluster instances. The URI can represent an image or image family. Image examples: * https://www.googleapis.com/compute/beta/projects/ If the URI is unspecified, it will be inferred from SoftwareConfig.image_version or the system default.

instanceNames List<String>

Output only. The list of instance names. Dataproc derives the names from cluster_name, num_instances, and the instance group.

isPreemptible Boolean

Output only. Specifies that this instance group contains preemptible instances.

machineType String

Optional. The Compute Engine machine type used for cluster instances. A full URL, partial URI, or short name are valid. Examples: * https://www.googleapis.com/compute/v1/projects/(https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement) feature, you must use the short name of the machine type resource, for example, n1-standard-2`.

managedGroupConfigs List<Property Map>

Output only. The config for Compute Engine Instance Group Manager that manages this group. This is only used for preemptible instance groups.

minCpuPlatform String

Optional. Specifies the minimum cpu platform for the Instance Group. See (https://cloud.google.com/dataproc/docs/concepts/compute/dataproc-min-cpu).

numInstances Number

Optional. The number of VM instances in the instance group. For master instance groups, must be set to 1.

preemptibility String

Optional. Specifies the preemptibility of the instance group. The default value for master and worker groups is NON_PREEMPTIBLE. This default cannot be changed. The default value for secondary instances is PREEMPTIBLE. Possible values: PREEMPTIBILITY_UNSPECIFIED, NON_PREEMPTIBLE, PREEMPTIBLE

WorkflowTemplatePlacementManagedClusterConfigMasterConfigAccelerator

AcceleratorCount int

The number of the accelerator cards of this type exposed to this instance.

AcceleratorType string

Full URL, partial URI, or short name of the accelerator type resource to expose to this instance. See (https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement) feature, you must use the short name of the accelerator type resource, for example, nvidia-tesla-k80.

AcceleratorCount int

The number of the accelerator cards of this type exposed to this instance.

AcceleratorType string

Full URL, partial URI, or short name of the accelerator type resource to expose to this instance. See (https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement) feature, you must use the short name of the accelerator type resource, for example, nvidia-tesla-k80.

acceleratorCount Integer

The number of the accelerator cards of this type exposed to this instance.

acceleratorType String

Full URL, partial URI, or short name of the accelerator type resource to expose to this instance. See (https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement) feature, you must use the short name of the accelerator type resource, for example, nvidia-tesla-k80.

acceleratorCount number

The number of the accelerator cards of this type exposed to this instance.

acceleratorType string

Full URL, partial URI, or short name of the accelerator type resource to expose to this instance. See (https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement) feature, you must use the short name of the accelerator type resource, for example, nvidia-tesla-k80.

accelerator_count int

The number of the accelerator cards of this type exposed to this instance.

accelerator_type str

Full URL, partial URI, or short name of the accelerator type resource to expose to this instance. See (https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement) feature, you must use the short name of the accelerator type resource, for example, nvidia-tesla-k80.

acceleratorCount Number

The number of the accelerator cards of this type exposed to this instance.

acceleratorType String

Full URL, partial URI, or short name of the accelerator type resource to expose to this instance. See (https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement) feature, you must use the short name of the accelerator type resource, for example, nvidia-tesla-k80.

WorkflowTemplatePlacementManagedClusterConfigMasterConfigDiskConfig

BootDiskSizeGb int

Optional. Size in GB of the boot disk (default is 500GB).

BootDiskType string

Optional. Type of the boot disk (default is "pd-standard"). Valid values: "pd-ssd" (Persistent Disk Solid State Drive) or "pd-standard" (Persistent Disk Hard Disk Drive).

NumLocalSsds int

Optional. Number of attached SSDs, from 0 to 4 (default is 0). If SSDs are not attached, the boot disk is used to store runtime logs and (https://hadoop.apache.org/docs/r1.2.1/hdfs_user_guide.html) data. If one or more SSDs are attached, this runtime bulk data is spread across them, and the boot disk contains only basic config and installed binaries.

BootDiskSizeGb int

Optional. Size in GB of the boot disk (default is 500GB).

BootDiskType string

Optional. Type of the boot disk (default is "pd-standard"). Valid values: "pd-ssd" (Persistent Disk Solid State Drive) or "pd-standard" (Persistent Disk Hard Disk Drive).

NumLocalSsds int

Optional. Number of attached SSDs, from 0 to 4 (default is 0). If SSDs are not attached, the boot disk is used to store runtime logs and (https://hadoop.apache.org/docs/r1.2.1/hdfs_user_guide.html) data. If one or more SSDs are attached, this runtime bulk data is spread across them, and the boot disk contains only basic config and installed binaries.

bootDiskSizeGb Integer

Optional. Size in GB of the boot disk (default is 500GB).

bootDiskType String

Optional. Type of the boot disk (default is "pd-standard"). Valid values: "pd-ssd" (Persistent Disk Solid State Drive) or "pd-standard" (Persistent Disk Hard Disk Drive).

numLocalSsds Integer

Optional. Number of attached SSDs, from 0 to 4 (default is 0). If SSDs are not attached, the boot disk is used to store runtime logs and (https://hadoop.apache.org/docs/r1.2.1/hdfs_user_guide.html) data. If one or more SSDs are attached, this runtime bulk data is spread across them, and the boot disk contains only basic config and installed binaries.

bootDiskSizeGb number

Optional. Size in GB of the boot disk (default is 500GB).

bootDiskType string

Optional. Type of the boot disk (default is "pd-standard"). Valid values: "pd-ssd" (Persistent Disk Solid State Drive) or "pd-standard" (Persistent Disk Hard Disk Drive).

numLocalSsds number

Optional. Number of attached SSDs, from 0 to 4 (default is 0). If SSDs are not attached, the boot disk is used to store runtime logs and (https://hadoop.apache.org/docs/r1.2.1/hdfs_user_guide.html) data. If one or more SSDs are attached, this runtime bulk data is spread across them, and the boot disk contains only basic config and installed binaries.

boot_disk_size_gb int

Optional. Size in GB of the boot disk (default is 500GB).

boot_disk_type str

Optional. Type of the boot disk (default is "pd-standard"). Valid values: "pd-ssd" (Persistent Disk Solid State Drive) or "pd-standard" (Persistent Disk Hard Disk Drive).

num_local_ssds int

Optional. Number of attached SSDs, from 0 to 4 (default is 0). If SSDs are not attached, the boot disk is used to store runtime logs and (https://hadoop.apache.org/docs/r1.2.1/hdfs_user_guide.html) data. If one or more SSDs are attached, this runtime bulk data is spread across them, and the boot disk contains only basic config and installed binaries.

bootDiskSizeGb Number

Optional. Size in GB of the boot disk (default is 500GB).

bootDiskType String

Optional. Type of the boot disk (default is "pd-standard"). Valid values: "pd-ssd" (Persistent Disk Solid State Drive) or "pd-standard" (Persistent Disk Hard Disk Drive).

numLocalSsds Number

Optional. Number of attached SSDs, from 0 to 4 (default is 0). If SSDs are not attached, the boot disk is used to store runtime logs and (https://hadoop.apache.org/docs/r1.2.1/hdfs_user_guide.html) data. If one or more SSDs are attached, this runtime bulk data is spread across them, and the boot disk contains only basic config and installed binaries.

WorkflowTemplatePlacementManagedClusterConfigMasterConfigManagedGroupConfig

WorkflowTemplatePlacementManagedClusterConfigMetastoreConfig

DataprocMetastoreService string

Required. Resource name of an existing Dataproc Metastore service. Example: * projects/

DataprocMetastoreService string

Required. Resource name of an existing Dataproc Metastore service. Example: * projects/

dataprocMetastoreService String

Required. Resource name of an existing Dataproc Metastore service. Example: * projects/

dataprocMetastoreService string

Required. Resource name of an existing Dataproc Metastore service. Example: * projects/

dataproc_metastore_service str

Required. Resource name of an existing Dataproc Metastore service. Example: * projects/

dataprocMetastoreService String

Required. Resource name of an existing Dataproc Metastore service. Example: * projects/

WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfig

Accelerators List<WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigAccelerator>

Optional. The Compute Engine accelerator configuration for these instances.

DiskConfig WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigDiskConfig

Optional. Disk option config settings.

Image string

Optional. The Compute Engine image resource used for cluster instances. The URI can represent an image or image family. Image examples: * https://www.googleapis.com/compute/beta/projects/ If the URI is unspecified, it will be inferred from SoftwareConfig.image_version or the system default.

InstanceNames List<string>

Output only. The list of instance names. Dataproc derives the names from cluster_name, num_instances, and the instance group.

IsPreemptible bool

Output only. Specifies that this instance group contains preemptible instances.

MachineType string

Optional. The Compute Engine machine type used for cluster instances. A full URL, partial URI, or short name are valid. Examples: * https://www.googleapis.com/compute/v1/projects/(https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement) feature, you must use the short name of the machine type resource, for example, n1-standard-2`.

ManagedGroupConfigs List<WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigManagedGroupConfig>

Output only. The config for Compute Engine Instance Group Manager that manages this group. This is only used for preemptible instance groups.

MinCpuPlatform string

Optional. Specifies the minimum cpu platform for the Instance Group. See (https://cloud.google.com/dataproc/docs/concepts/compute/dataproc-min-cpu).

NumInstances int

Optional. The number of VM instances in the instance group. For master instance groups, must be set to 1.

Preemptibility string

Optional. Specifies the preemptibility of the instance group. The default value for master and worker groups is NON_PREEMPTIBLE. This default cannot be changed. The default value for secondary instances is PREEMPTIBLE. Possible values: PREEMPTIBILITY_UNSPECIFIED, NON_PREEMPTIBLE, PREEMPTIBLE

Accelerators []WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigAccelerator

Optional. The Compute Engine accelerator configuration for these instances.

DiskConfig WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigDiskConfig

Optional. Disk option config settings.

Image string

Optional. The Compute Engine image resource used for cluster instances. The URI can represent an image or image family. Image examples: * https://www.googleapis.com/compute/beta/projects/ If the URI is unspecified, it will be inferred from SoftwareConfig.image_version or the system default.

InstanceNames []string

Output only. The list of instance names. Dataproc derives the names from cluster_name, num_instances, and the instance group.

IsPreemptible bool

Output only. Specifies that this instance group contains preemptible instances.

MachineType string

Optional. The Compute Engine machine type used for cluster instances. A full URL, partial URI, or short name are valid. Examples: * https://www.googleapis.com/compute/v1/projects/(https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement) feature, you must use the short name of the machine type resource, for example, n1-standard-2`.

ManagedGroupConfigs []WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigManagedGroupConfig

Output only. The config for Compute Engine Instance Group Manager that manages this group. This is only used for preemptible instance groups.

MinCpuPlatform string

Optional. Specifies the minimum cpu platform for the Instance Group. See (https://cloud.google.com/dataproc/docs/concepts/compute/dataproc-min-cpu).

NumInstances int

Optional. The number of VM instances in the instance group. For master instance groups, must be set to 1.

Preemptibility string

Optional. Specifies the preemptibility of the instance group. The default value for master and worker groups is NON_PREEMPTIBLE. This default cannot be changed. The default value for secondary instances is PREEMPTIBLE. Possible values: PREEMPTIBILITY_UNSPECIFIED, NON_PREEMPTIBLE, PREEMPTIBLE

accelerators List<WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigAccelerator>

Optional. The Compute Engine accelerator configuration for these instances.

diskConfig WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigDiskConfig

Optional. Disk option config settings.

image String

Optional. The Compute Engine image resource used for cluster instances. The URI can represent an image or image family. Image examples: * https://www.googleapis.com/compute/beta/projects/ If the URI is unspecified, it will be inferred from SoftwareConfig.image_version or the system default.

instanceNames List<String>

Output only. The list of instance names. Dataproc derives the names from cluster_name, num_instances, and the instance group.

isPreemptible Boolean

Output only. Specifies that this instance group contains preemptible instances.

machineType String

Optional. The Compute Engine machine type used for cluster instances. A full URL, partial URI, or short name are valid. Examples: * https://www.googleapis.com/compute/v1/projects/(https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement) feature, you must use the short name of the machine type resource, for example, n1-standard-2`.

managedGroupConfigs List<WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigManagedGroupConfig>

Output only. The config for Compute Engine Instance Group Manager that manages this group. This is only used for preemptible instance groups.

minCpuPlatform String

Optional. Specifies the minimum cpu platform for the Instance Group. See (https://cloud.google.com/dataproc/docs/concepts/compute/dataproc-min-cpu).

numInstances Integer

Optional. The number of VM instances in the instance group. For master instance groups, must be set to 1.

preemptibility String

Optional. Specifies the preemptibility of the instance group. The default value for master and worker groups is NON_PREEMPTIBLE. This default cannot be changed. The default value for secondary instances is PREEMPTIBLE. Possible values: PREEMPTIBILITY_UNSPECIFIED, NON_PREEMPTIBLE, PREEMPTIBLE

accelerators WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigAccelerator[]

Optional. The Compute Engine accelerator configuration for these instances.

diskConfig WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigDiskConfig

Optional. Disk option config settings.

image string

Optional. The Compute Engine image resource used for cluster instances. The URI can represent an image or image family. Image examples: * https://www.googleapis.com/compute/beta/projects/ If the URI is unspecified, it will be inferred from SoftwareConfig.image_version or the system default.

instanceNames string[]

Output only. The list of instance names. Dataproc derives the names from cluster_name, num_instances, and the instance group.

isPreemptible boolean

Output only. Specifies that this instance group contains preemptible instances.

machineType string

Optional. The Compute Engine machine type used for cluster instances. A full URL, partial URI, or short name are valid. Examples: * https://www.googleapis.com/compute/v1/projects/(https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement) feature, you must use the short name of the machine type resource, for example, n1-standard-2`.

managedGroupConfigs WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigManagedGroupConfig[]

Output only. The config for Compute Engine Instance Group Manager that manages this group. This is only used for preemptible instance groups.

minCpuPlatform string

Optional. Specifies the minimum cpu platform for the Instance Group. See (https://cloud.google.com/dataproc/docs/concepts/compute/dataproc-min-cpu).

numInstances number

Optional. The number of VM instances in the instance group. For master instance groups, must be set to 1.

preemptibility string

Optional. Specifies the preemptibility of the instance group. The default value for master and worker groups is NON_PREEMPTIBLE. This default cannot be changed. The default value for secondary instances is PREEMPTIBLE. Possible values: PREEMPTIBILITY_UNSPECIFIED, NON_PREEMPTIBLE, PREEMPTIBLE

accelerators Sequence[WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigAccelerator]

Optional. The Compute Engine accelerator configuration for these instances.

disk_config WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigDiskConfig

Optional. Disk option config settings.

image str

Optional. The Compute Engine image resource used for cluster instances. The URI can represent an image or image family. Image examples: * https://www.googleapis.com/compute/beta/projects/ If the URI is unspecified, it will be inferred from SoftwareConfig.image_version or the system default.

instance_names Sequence[str]

Output only. The list of instance names. Dataproc derives the names from cluster_name, num_instances, and the instance group.

is_preemptible bool

Output only. Specifies that this instance group contains preemptible instances.

machine_type str

Optional. The Compute Engine machine type used for cluster instances. A full URL, partial URI, or short name are valid. Examples: * https://www.googleapis.com/compute/v1/projects/(https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement) feature, you must use the short name of the machine type resource, for example, n1-standard-2`.

managed_group_configs Sequence[WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigManagedGroupConfig]

Output only. The config for Compute Engine Instance Group Manager that manages this group. This is only used for preemptible instance groups.

min_cpu_platform str

Optional. Specifies the minimum cpu platform for the Instance Group. See (https://cloud.google.com/dataproc/docs/concepts/compute/dataproc-min-cpu).

num_instances int

Optional. The number of VM instances in the instance group. For master instance groups, must be set to 1.

preemptibility str

Optional. Specifies the preemptibility of the instance group. The default value for master and worker groups is NON_PREEMPTIBLE. This default cannot be changed. The default value for secondary instances is PREEMPTIBLE. Possible values: PREEMPTIBILITY_UNSPECIFIED, NON_PREEMPTIBLE, PREEMPTIBLE

accelerators List<Property Map>

Optional. The Compute Engine accelerator configuration for these instances.

diskConfig Property Map

Optional. Disk option config settings.

image String

Optional. The Compute Engine image resource used for cluster instances. The URI can represent an image or image family. Image examples: * https://www.googleapis.com/compute/beta/projects/ If the URI is unspecified, it will be inferred from SoftwareConfig.image_version or the system default.

instanceNames List<String>

Output only. The list of instance names. Dataproc derives the names from cluster_name, num_instances, and the instance group.

isPreemptible Boolean

Output only. Specifies that this instance group contains preemptible instances.

machineType String

Optional. The Compute Engine machine type used for cluster instances. A full URL, partial URI, or short name are valid. Examples: * https://www.googleapis.com/compute/v1/projects/(https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement) feature, you must use the short name of the machine type resource, for example, n1-standard-2`.

managedGroupConfigs List<Property Map>

Output only. The config for Compute Engine Instance Group Manager that manages this group. This is only used for preemptible instance groups.

minCpuPlatform String

Optional. Specifies the minimum cpu platform for the Instance Group. See (https://cloud.google.com/dataproc/docs/concepts/compute/dataproc-min-cpu).

numInstances Number

Optional. The number of VM instances in the instance group. For master instance groups, must be set to 1.

preemptibility String

Optional. Specifies the preemptibility of the instance group. The default value for master and worker groups is NON_PREEMPTIBLE. This default cannot be changed. The default value for secondary instances is PREEMPTIBLE. Possible values: PREEMPTIBILITY_UNSPECIFIED, NON_PREEMPTIBLE, PREEMPTIBLE

WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigAccelerator

AcceleratorCount int

The number of the accelerator cards of this type exposed to this instance.

AcceleratorType string

Full URL, partial URI, or short name of the accelerator type resource to expose to this instance. See (https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement) feature, you must use the short name of the accelerator type resource, for example, nvidia-tesla-k80.

AcceleratorCount int

The number of the accelerator cards of this type exposed to this instance.

AcceleratorType string

Full URL, partial URI, or short name of the accelerator type resource to expose to this instance. See (https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement) feature, you must use the short name of the accelerator type resource, for example, nvidia-tesla-k80.

acceleratorCount Integer

The number of the accelerator cards of this type exposed to this instance.

acceleratorType String

Full URL, partial URI, or short name of the accelerator type resource to expose to this instance. See (https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement) feature, you must use the short name of the accelerator type resource, for example, nvidia-tesla-k80.

acceleratorCount number

The number of the accelerator cards of this type exposed to this instance.

acceleratorType string

Full URL, partial URI, or short name of the accelerator type resource to expose to this instance. See (https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement) feature, you must use the short name of the accelerator type resource, for example, nvidia-tesla-k80.

accelerator_count int

The number of the accelerator cards of this type exposed to this instance.

accelerator_type str

Full URL, partial URI, or short name of the accelerator type resource to expose to this instance. See (https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement) feature, you must use the short name of the accelerator type resource, for example, nvidia-tesla-k80.

acceleratorCount Number

The number of the accelerator cards of this type exposed to this instance.

acceleratorType String

Full URL, partial URI, or short name of the accelerator type resource to expose to this instance. See (https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement) feature, you must use the short name of the accelerator type resource, for example, nvidia-tesla-k80.

WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigDiskConfig

BootDiskSizeGb int

Optional. Size in GB of the boot disk (default is 500GB).

BootDiskType string

Optional. Type of the boot disk (default is "pd-standard"). Valid values: "pd-ssd" (Persistent Disk Solid State Drive) or "pd-standard" (Persistent Disk Hard Disk Drive).

NumLocalSsds int

Optional. Number of attached SSDs, from 0 to 4 (default is 0). If SSDs are not attached, the boot disk is used to store runtime logs and (https://hadoop.apache.org/docs/r1.2.1/hdfs_user_guide.html) data. If one or more SSDs are attached, this runtime bulk data is spread across them, and the boot disk contains only basic config and installed binaries.

BootDiskSizeGb int

Optional. Size in GB of the boot disk (default is 500GB).

BootDiskType string

Optional. Type of the boot disk (default is "pd-standard"). Valid values: "pd-ssd" (Persistent Disk Solid State Drive) or "pd-standard" (Persistent Disk Hard Disk Drive).

NumLocalSsds int

Optional. Number of attached SSDs, from 0 to 4 (default is 0). If SSDs are not attached, the boot disk is used to store runtime logs and (https://hadoop.apache.org/docs/r1.2.1/hdfs_user_guide.html) data. If one or more SSDs are attached, this runtime bulk data is spread across them, and the boot disk contains only basic config and installed binaries.

bootDiskSizeGb Integer

Optional. Size in GB of the boot disk (default is 500GB).

bootDiskType String

Optional. Type of the boot disk (default is "pd-standard"). Valid values: "pd-ssd" (Persistent Disk Solid State Drive) or "pd-standard" (Persistent Disk Hard Disk Drive).

numLocalSsds Integer

Optional. Number of attached SSDs, from 0 to 4 (default is 0). If SSDs are not attached, the boot disk is used to store runtime logs and (https://hadoop.apache.org/docs/r1.2.1/hdfs_user_guide.html) data. If one or more SSDs are attached, this runtime bulk data is spread across them, and the boot disk contains only basic config and installed binaries.

bootDiskSizeGb number

Optional. Size in GB of the boot disk (default is 500GB).

bootDiskType string

Optional. Type of the boot disk (default is "pd-standard"). Valid values: "pd-ssd" (Persistent Disk Solid State Drive) or "pd-standard" (Persistent Disk Hard Disk Drive).

numLocalSsds number

Optional. Number of attached SSDs, from 0 to 4 (default is 0). If SSDs are not attached, the boot disk is used to store runtime logs and (https://hadoop.apache.org/docs/r1.2.1/hdfs_user_guide.html) data. If one or more SSDs are attached, this runtime bulk data is spread across them, and the boot disk contains only basic config and installed binaries.

boot_disk_size_gb int

Optional. Size in GB of the boot disk (default is 500GB).

boot_disk_type str

Optional. Type of the boot disk (default is "pd-standard"). Valid values: "pd-ssd" (Persistent Disk Solid State Drive) or "pd-standard" (Persistent Disk Hard Disk Drive).

num_local_ssds int

Optional. Number of attached SSDs, from 0 to 4 (default is 0). If SSDs are not attached, the boot disk is used to store runtime logs and (https://hadoop.apache.org/docs/r1.2.1/hdfs_user_guide.html) data. If one or more SSDs are attached, this runtime bulk data is spread across them, and the boot disk contains only basic config and installed binaries.

bootDiskSizeGb Number

Optional. Size in GB of the boot disk (default is 500GB).

bootDiskType String

Optional. Type of the boot disk (default is "pd-standard"). Valid values: "pd-ssd" (Persistent Disk Solid State Drive) or "pd-standard" (Persistent Disk Hard Disk Drive).

numLocalSsds Number

Optional. Number of attached SSDs, from 0 to 4 (default is 0). If SSDs are not attached, the boot disk is used to store runtime logs and (https://hadoop.apache.org/docs/r1.2.1/hdfs_user_guide.html) data. If one or more SSDs are attached, this runtime bulk data is spread across them, and the boot disk contains only basic config and installed binaries.

WorkflowTemplatePlacementManagedClusterConfigSecondaryWorkerConfigManagedGroupConfig

WorkflowTemplatePlacementManagedClusterConfigSecurityConfig

kerberosConfig Property Map

Kerberos related configuration.

WorkflowTemplatePlacementManagedClusterConfigSecurityConfigKerberosConfig

CrossRealmTrustAdminServer string

Optional. The admin server (IP or hostname) for the remote trusted realm in a cross realm trust relationship.

CrossRealmTrustKdc string

Optional. The KDC (IP or hostname) for the remote trusted realm in a cross realm trust relationship.

CrossRealmTrustRealm string

Optional. The remote realm the Dataproc on-cluster KDC will trust, should the user enable cross realm trust.

CrossRealmTrustSharedPassword string

Optional. The Cloud Storage URI of a KMS encrypted file containing the shared password between the on-cluster Kerberos realm and the remote trusted realm, in a cross realm trust relationship.

EnableKerberos bool

Optional. Flag to indicate whether to Kerberize the cluster (default: false). Set this field to true to enable Kerberos on a cluster.

KdcDbKey string

Optional. The Cloud Storage URI of a KMS encrypted file containing the master key of the KDC database.

KeyPassword string

Optional. The Cloud Storage URI of a KMS encrypted file containing the password to the user provided key. For the self-signed certificate, this password is generated by Dataproc.

Keystore string

Optional. The Cloud Storage URI of the keystore file used for SSL encryption. If not provided, Dataproc will provide a self-signed certificate.

KeystorePassword string

Optional. The Cloud Storage URI of a KMS encrypted file containing the password to the user provided keystore. For the self-signed certificate, this password is generated by Dataproc.

KmsKey string

Optional. The uri of the KMS key used to encrypt various sensitive files.

Realm string

Optional. The name of the on-cluster Kerberos realm. If not specified, the uppercased domain of hostnames will be the realm.

RootPrincipalPassword string

Optional. The Cloud Storage URI of a KMS encrypted file containing the root principal password.

TgtLifetimeHours int

Optional. The lifetime of the ticket granting ticket, in hours. If not specified, or user specifies 0, then default value 10 will be used.

Truststore string

Optional. The Cloud Storage URI of the truststore file used for SSL encryption. If not provided, Dataproc will provide a self-signed certificate.

TruststorePassword string

Optional. The Cloud Storage URI of a KMS encrypted file containing the password to the user provided truststore. For the self-signed certificate, this password is generated by Dataproc.

CrossRealmTrustAdminServer string

Optional. The admin server (IP or hostname) for the remote trusted realm in a cross realm trust relationship.

CrossRealmTrustKdc string

Optional. The KDC (IP or hostname) for the remote trusted realm in a cross realm trust relationship.

CrossRealmTrustRealm string

Optional. The remote realm the Dataproc on-cluster KDC will trust, should the user enable cross realm trust.

CrossRealmTrustSharedPassword string

Optional. The Cloud Storage URI of a KMS encrypted file containing the shared password between the on-cluster Kerberos realm and the remote trusted realm, in a cross realm trust relationship.

EnableKerberos bool

Optional. Flag to indicate whether to Kerberize the cluster (default: false). Set this field to true to enable Kerberos on a cluster.

KdcDbKey string

Optional. The Cloud Storage URI of a KMS encrypted file containing the master key of the KDC database.

KeyPassword string

Optional. The Cloud Storage URI of a KMS encrypted file containing the password to the user provided key. For the self-signed certificate, this password is generated by Dataproc.

Keystore string

Optional. The Cloud Storage URI of the keystore file used for SSL encryption. If not provided, Dataproc will provide a self-signed certificate.

KeystorePassword string

Optional. The Cloud Storage URI of a KMS encrypted file containing the password to the user provided keystore. For the self-signed certificate, this password is generated by Dataproc.

KmsKey string

Optional. The uri of the KMS key used to encrypt various sensitive files.

Realm string

Optional. The name of the on-cluster Kerberos realm. If not specified, the uppercased domain of hostnames will be the realm.

RootPrincipalPassword string

Optional. The Cloud Storage URI of a KMS encrypted file containing the root principal password.

TgtLifetimeHours int

Optional. The lifetime of the ticket granting ticket, in hours. If not specified, or user specifies 0, then default value 10 will be used.

Truststore string

Optional. The Cloud Storage URI of the truststore file used for SSL encryption. If not provided, Dataproc will provide a self-signed certificate.

TruststorePassword string

Optional. The Cloud Storage URI of a KMS encrypted file containing the password to the user provided truststore. For the self-signed certificate, this password is generated by Dataproc.

crossRealmTrustAdminServer String

Optional. The admin server (IP or hostname) for the remote trusted realm in a cross realm trust relationship.

crossRealmTrustKdc String

Optional. The KDC (IP or hostname) for the remote trusted realm in a cross realm trust relationship.

crossRealmTrustRealm String

Optional. The remote realm the Dataproc on-cluster KDC will trust, should the user enable cross realm trust.

crossRealmTrustSharedPassword String

Optional. The Cloud Storage URI of a KMS encrypted file containing the shared password between the on-cluster Kerberos realm and the remote trusted realm, in a cross realm trust relationship.

enableKerberos Boolean

Optional. Flag to indicate whether to Kerberize the cluster (default: false). Set this field to true to enable Kerberos on a cluster.

kdcDbKey String

Optional. The Cloud Storage URI of a KMS encrypted file containing the master key of the KDC database.

keyPassword String

Optional. The Cloud Storage URI of a KMS encrypted file containing the password to the user provided key. For the self-signed certificate, this password is generated by Dataproc.

keystore String

Optional. The Cloud Storage URI of the keystore file used for SSL encryption. If not provided, Dataproc will provide a self-signed certificate.

keystorePassword String

Optional. The Cloud Storage URI of a KMS encrypted file containing the password to the user provided keystore. For the self-signed certificate, this password is generated by Dataproc.

kmsKey String

Optional. The uri of the KMS key used to encrypt various sensitive files.

realm String

Optional. The name of the on-cluster Kerberos realm. If not specified, the uppercased domain of hostnames will be the realm.

rootPrincipalPassword String

Optional. The Cloud Storage URI of a KMS encrypted file containing the root principal password.

tgtLifetimeHours Integer

Optional. The lifetime of the ticket granting ticket, in hours. If not specified, or user specifies 0, then default value 10 will be used.

truststore String

Optional. The Cloud Storage URI of the truststore file used for SSL encryption. If not provided, Dataproc will provide a self-signed certificate.

truststorePassword String

Optional. The Cloud Storage URI of a KMS encrypted file containing the password to the user provided truststore. For the self-signed certificate, this password is generated by Dataproc.

crossRealmTrustAdminServer string

Optional. The admin server (IP or hostname) for the remote trusted realm in a cross realm trust relationship.

crossRealmTrustKdc string

Optional. The KDC (IP or hostname) for the remote trusted realm in a cross realm trust relationship.

crossRealmTrustRealm string

Optional. The remote realm the Dataproc on-cluster KDC will trust, should the user enable cross realm trust.

crossRealmTrustSharedPassword string

Optional. The Cloud Storage URI of a KMS encrypted file containing the shared password between the on-cluster Kerberos realm and the remote trusted realm, in a cross realm trust relationship.

enableKerberos boolean

Optional. Flag to indicate whether to Kerberize the cluster (default: false). Set this field to true to enable Kerberos on a cluster.

kdcDbKey string

Optional. The Cloud Storage URI of a KMS encrypted file containing the master key of the KDC database.

keyPassword string

Optional. The Cloud Storage URI of a KMS encrypted file containing the password to the user provided key. For the self-signed certificate, this password is generated by Dataproc.

keystore string

Optional. The Cloud Storage URI of the keystore file used for SSL encryption. If not provided, Dataproc will provide a self-signed certificate.

keystorePassword string

Optional. The Cloud Storage URI of a KMS encrypted file containing the password to the user provided keystore. For the self-signed certificate, this password is generated by Dataproc.

kmsKey string

Optional. The uri of the KMS key used to encrypt various sensitive files.

realm string

Optional. The name of the on-cluster Kerberos realm. If not specified, the uppercased domain of hostnames will be the realm.

rootPrincipalPassword string

Optional. The Cloud Storage URI of a KMS encrypted file containing the root principal password.

tgtLifetimeHours number

Optional. The lifetime of the ticket granting ticket, in hours. If not specified, or user specifies 0, then default value 10 will be used.

truststore string

Optional. The Cloud Storage URI of the truststore file used for SSL encryption. If not provided, Dataproc will provide a self-signed certificate.

truststorePassword string

Optional. The Cloud Storage URI of a KMS encrypted file containing the password to the user provided truststore. For the self-signed certificate, this password is generated by Dataproc.

cross_realm_trust_admin_server str

Optional. The admin server (IP or hostname) for the remote trusted realm in a cross realm trust relationship.

cross_realm_trust_kdc str

Optional. The KDC (IP or hostname) for the remote trusted realm in a cross realm trust relationship.

cross_realm_trust_realm str

Optional. The remote realm the Dataproc on-cluster KDC will trust, should the user enable cross realm trust.

cross_realm_trust_shared_password str

Optional. The Cloud Storage URI of a KMS encrypted file containing the shared password between the on-cluster Kerberos realm and the remote trusted realm, in a cross realm trust relationship.

enable_kerberos bool

Optional. Flag to indicate whether to Kerberize the cluster (default: false). Set this field to true to enable Kerberos on a cluster.

kdc_db_key str

Optional. The Cloud Storage URI of a KMS encrypted file containing the master key of the KDC database.

key_password str

Optional. The Cloud Storage URI of a KMS encrypted file containing the password to the user provided key. For the self-signed certificate, this password is generated by Dataproc.

keystore str

Optional. The Cloud Storage URI of the keystore file used for SSL encryption. If not provided, Dataproc will provide a self-signed certificate.

keystore_password str

Optional. The Cloud Storage URI of a KMS encrypted file containing the password to the user provided keystore. For the self-signed certificate, this password is generated by Dataproc.

kms_key str

Optional. The uri of the KMS key used to encrypt various sensitive files.

realm str

Optional. The name of the on-cluster Kerberos realm. If not specified, the uppercased domain of hostnames will be the realm.

root_principal_password str

Optional. The Cloud Storage URI of a KMS encrypted file containing the root principal password.

tgt_lifetime_hours int

Optional. The lifetime of the ticket granting ticket, in hours. If not specified, or user specifies 0, then default value 10 will be used.

truststore str

Optional. The Cloud Storage URI of the truststore file used for SSL encryption. If not provided, Dataproc will provide a self-signed certificate.

truststore_password str

Optional. The Cloud Storage URI of a KMS encrypted file containing the password to the user provided truststore. For the self-signed certificate, this password is generated by Dataproc.

crossRealmTrustAdminServer String

Optional. The admin server (IP or hostname) for the remote trusted realm in a cross realm trust relationship.

crossRealmTrustKdc String

Optional. The KDC (IP or hostname) for the remote trusted realm in a cross realm trust relationship.

crossRealmTrustRealm String

Optional. The remote realm the Dataproc on-cluster KDC will trust, should the user enable cross realm trust.

crossRealmTrustSharedPassword String

Optional. The Cloud Storage URI of a KMS encrypted file containing the shared password between the on-cluster Kerberos realm and the remote trusted realm, in a cross realm trust relationship.

enableKerberos Boolean

Optional. Flag to indicate whether to Kerberize the cluster (default: false). Set this field to true to enable Kerberos on a cluster.

kdcDbKey String

Optional. The Cloud Storage URI of a KMS encrypted file containing the master key of the KDC database.

keyPassword String

Optional. The Cloud Storage URI of a KMS encrypted file containing the password to the user provided key. For the self-signed certificate, this password is generated by Dataproc.

keystore String

Optional. The Cloud Storage URI of the keystore file used for SSL encryption. If not provided, Dataproc will provide a self-signed certificate.

keystorePassword String

Optional. The Cloud Storage URI of a KMS encrypted file containing the password to the user provided keystore. For the self-signed certificate, this password is generated by Dataproc.

kmsKey String

Optional. The uri of the KMS key used to encrypt various sensitive files.

realm String

Optional. The name of the on-cluster Kerberos realm. If not specified, the uppercased domain of hostnames will be the realm.

rootPrincipalPassword String

Optional. The Cloud Storage URI of a KMS encrypted file containing the root principal password.

tgtLifetimeHours Number

Optional. The lifetime of the ticket granting ticket, in hours. If not specified, or user specifies 0, then default value 10 will be used.

truststore String

Optional. The Cloud Storage URI of the truststore file used for SSL encryption. If not provided, Dataproc will provide a self-signed certificate.

truststorePassword String

Optional. The Cloud Storage URI of a KMS encrypted file containing the password to the user provided truststore. For the self-signed certificate, this password is generated by Dataproc.

WorkflowTemplatePlacementManagedClusterConfigSoftwareConfig

ImageVersion string

Optional. The version of software inside the cluster. It must be one of the supported (https://cloud.google.com/dataproc/docs/concepts/versioning/dataproc-versions#other_versions). If unspecified, it defaults to the latest Debian version.

OptionalComponents List<string>
Properties Dictionary<string, string>

Optional. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. The following are supported prefixes and their mappings: * capacity-scheduler: capacity-scheduler.xml * core: core-site.xml * distcp: distcp-default.xml * hdfs: hdfs-site.xml * hive: hive-site.xml * mapred: mapred-site.xml * pig: pig.properties * spark: spark-defaults.conf * yarn: yarn-site.xml For more information, see (https://cloud.google.com/dataproc/docs/concepts/cluster-properties).

ImageVersion string

Optional. The version of software inside the cluster. It must be one of the supported (https://cloud.google.com/dataproc/docs/concepts/versioning/dataproc-versions#other_versions). If unspecified, it defaults to the latest Debian version.

OptionalComponents []string
Properties map[string]string

Optional. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. The following are supported prefixes and their mappings: * capacity-scheduler: capacity-scheduler.xml * core: core-site.xml * distcp: distcp-default.xml * hdfs: hdfs-site.xml * hive: hive-site.xml * mapred: mapred-site.xml * pig: pig.properties * spark: spark-defaults.conf * yarn: yarn-site.xml For more information, see (https://cloud.google.com/dataproc/docs/concepts/cluster-properties).

imageVersion String

Optional. The version of software inside the cluster. It must be one of the supported (https://cloud.google.com/dataproc/docs/concepts/versioning/dataproc-versions#other_versions). If unspecified, it defaults to the latest Debian version.

optionalComponents List<String>
properties Map<String,String>

Optional. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. The following are supported prefixes and their mappings: * capacity-scheduler: capacity-scheduler.xml * core: core-site.xml * distcp: distcp-default.xml * hdfs: hdfs-site.xml * hive: hive-site.xml * mapred: mapred-site.xml * pig: pig.properties * spark: spark-defaults.conf * yarn: yarn-site.xml For more information, see (https://cloud.google.com/dataproc/docs/concepts/cluster-properties).

imageVersion string

Optional. The version of software inside the cluster. It must be one of the supported (https://cloud.google.com/dataproc/docs/concepts/versioning/dataproc-versions#other_versions). If unspecified, it defaults to the latest Debian version.

optionalComponents string[]
properties {[key: string]: string}

Optional. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. The following are supported prefixes and their mappings: * capacity-scheduler: capacity-scheduler.xml * core: core-site.xml * distcp: distcp-default.xml * hdfs: hdfs-site.xml * hive: hive-site.xml * mapred: mapred-site.xml * pig: pig.properties * spark: spark-defaults.conf * yarn: yarn-site.xml For more information, see (https://cloud.google.com/dataproc/docs/concepts/cluster-properties).

image_version str

Optional. The version of software inside the cluster. It must be one of the supported (https://cloud.google.com/dataproc/docs/concepts/versioning/dataproc-versions#other_versions). If unspecified, it defaults to the latest Debian version.

optional_components Sequence[str]
properties Mapping[str, str]

Optional. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. The following are supported prefixes and their mappings: * capacity-scheduler: capacity-scheduler.xml * core: core-site.xml * distcp: distcp-default.xml * hdfs: hdfs-site.xml * hive: hive-site.xml * mapred: mapred-site.xml * pig: pig.properties * spark: spark-defaults.conf * yarn: yarn-site.xml For more information, see (https://cloud.google.com/dataproc/docs/concepts/cluster-properties).

imageVersion String

Optional. The version of software inside the cluster. It must be one of the supported (https://cloud.google.com/dataproc/docs/concepts/versioning/dataproc-versions#other_versions). If unspecified, it defaults to the latest Debian version.

optionalComponents List<String>
properties Map<String>

Optional. The properties to set on daemon config files. Property keys are specified in prefix:property format, for example core:hadoop.tmp.dir. The following are supported prefixes and their mappings: * capacity-scheduler: capacity-scheduler.xml * core: core-site.xml * distcp: distcp-default.xml * hdfs: hdfs-site.xml * hive: hive-site.xml * mapred: mapred-site.xml * pig: pig.properties * spark: spark-defaults.conf * yarn: yarn-site.xml For more information, see (https://cloud.google.com/dataproc/docs/concepts/cluster-properties).

WorkflowTemplatePlacementManagedClusterConfigWorkerConfig

Accelerators List<WorkflowTemplatePlacementManagedClusterConfigWorkerConfigAccelerator>

Optional. The Compute Engine accelerator configuration for these instances.

DiskConfig WorkflowTemplatePlacementManagedClusterConfigWorkerConfigDiskConfig

Optional. Disk option config settings.

Image string

Optional. The Compute Engine image resource used for cluster instances. The URI can represent an image or image family. Image examples: * https://www.googleapis.com/compute/beta/projects/ If the URI is unspecified, it will be inferred from SoftwareConfig.image_version or the system default.

InstanceNames List<string>

Output only. The list of instance names. Dataproc derives the names from cluster_name, num_instances, and the instance group.

IsPreemptible bool

Output only. Specifies that this instance group contains preemptible instances.

MachineType string

Optional. The Compute Engine machine type used for cluster instances. A full URL, partial URI, or short name are valid. Examples: * https://www.googleapis.com/compute/v1/projects/(https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement) feature, you must use the short name of the machine type resource, for example, n1-standard-2`.

ManagedGroupConfigs List<WorkflowTemplatePlacementManagedClusterConfigWorkerConfigManagedGroupConfig>

Output only. The config for Compute Engine Instance Group Manager that manages this group. This is only used for preemptible instance groups.

MinCpuPlatform string

Optional. Specifies the minimum cpu platform for the Instance Group. See (https://cloud.google.com/dataproc/docs/concepts/compute/dataproc-min-cpu).

NumInstances int

Optional. The number of VM instances in the instance group. For master instance groups, must be set to 1.

Preemptibility string

Optional. Specifies the preemptibility of the instance group. The default value for master and worker groups is NON_PREEMPTIBLE. This default cannot be changed. The default value for secondary instances is PREEMPTIBLE. Possible values: PREEMPTIBILITY_UNSPECIFIED, NON_PREEMPTIBLE, PREEMPTIBLE

Accelerators []WorkflowTemplatePlacementManagedClusterConfigWorkerConfigAccelerator

Optional. The Compute Engine accelerator configuration for these instances.

DiskConfig WorkflowTemplatePlacementManagedClusterConfigWorkerConfigDiskConfig

Optional. Disk option config settings.

Image string

Optional. The Compute Engine image resource used for cluster instances. The URI can represent an image or image family. Image examples: * https://www.googleapis.com/compute/beta/projects/ If the URI is unspecified, it will be inferred from SoftwareConfig.image_version or the system default.

InstanceNames []string

Output only. The list of instance names. Dataproc derives the names from cluster_name, num_instances, and the instance group.

IsPreemptible bool

Output only. Specifies that this instance group contains preemptible instances.

MachineType string

Optional. The Compute Engine machine type used for cluster instances. A full URL, partial URI, or short name are valid. Examples: * https://www.googleapis.com/compute/v1/projects/(https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement) feature, you must use the short name of the machine type resource, for example, n1-standard-2`.

ManagedGroupConfigs []WorkflowTemplatePlacementManagedClusterConfigWorkerConfigManagedGroupConfig

Output only. The config for Compute Engine Instance Group Manager that manages this group. This is only used for preemptible instance groups.

MinCpuPlatform string

Optional. Specifies the minimum cpu platform for the Instance Group. See (https://cloud.google.com/dataproc/docs/concepts/compute/dataproc-min-cpu).

NumInstances int

Optional. The number of VM instances in the instance group. For master instance groups, must be set to 1.

Preemptibility string

Optional. Specifies the preemptibility of the instance group. The default value for master and worker groups is NON_PREEMPTIBLE. This default cannot be changed. The default value for secondary instances is PREEMPTIBLE. Possible values: PREEMPTIBILITY_UNSPECIFIED, NON_PREEMPTIBLE, PREEMPTIBLE

accelerators List<WorkflowTemplatePlacementManagedClusterConfigWorkerConfigAccelerator>

Optional. The Compute Engine accelerator configuration for these instances.

diskConfig WorkflowTemplatePlacementManagedClusterConfigWorkerConfigDiskConfig

Optional. Disk option config settings.

image String

Optional. The Compute Engine image resource used for cluster instances. The URI can represent an image or image family. Image examples: * https://www.googleapis.com/compute/beta/projects/ If the URI is unspecified, it will be inferred from SoftwareConfig.image_version or the system default.

instanceNames List<String>

Output only. The list of instance names. Dataproc derives the names from cluster_name, num_instances, and the instance group.

isPreemptible Boolean

Output only. Specifies that this instance group contains preemptible instances.

machineType String

Optional. The Compute Engine machine type used for cluster instances. A full URL, partial URI, or short name are valid. Examples: * https://www.googleapis.com/compute/v1/projects/(https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement) feature, you must use the short name of the machine type resource, for example, n1-standard-2`.

managedGroupConfigs List<WorkflowTemplatePlacementManagedClusterConfigWorkerConfigManagedGroupConfig>

Output only. The config for Compute Engine Instance Group Manager that manages this group. This is only used for preemptible instance groups.

minCpuPlatform String

Optional. Specifies the minimum cpu platform for the Instance Group. See (https://cloud.google.com/dataproc/docs/concepts/compute/dataproc-min-cpu).

numInstances Integer

Optional. The number of VM instances in the instance group. For master instance groups, must be set to 1.

preemptibility String

Optional. Specifies the preemptibility of the instance group. The default value for master and worker groups is NON_PREEMPTIBLE. This default cannot be changed. The default value for secondary instances is PREEMPTIBLE. Possible values: PREEMPTIBILITY_UNSPECIFIED, NON_PREEMPTIBLE, PREEMPTIBLE

accelerators WorkflowTemplatePlacementManagedClusterConfigWorkerConfigAccelerator[]

Optional. The Compute Engine accelerator configuration for these instances.

diskConfig WorkflowTemplatePlacementManagedClusterConfigWorkerConfigDiskConfig

Optional. Disk option config settings.

image string

Optional. The Compute Engine image resource used for cluster instances. The URI can represent an image or image family. Image examples: * https://www.googleapis.com/compute/beta/projects/ If the URI is unspecified, it will be inferred from SoftwareConfig.image_version or the system default.

instanceNames string[]

Output only. The list of instance names. Dataproc derives the names from cluster_name, num_instances, and the instance group.

isPreemptible boolean

Output only. Specifies that this instance group contains preemptible instances.

machineType string

Optional. The Compute Engine machine type used for cluster instances. A full URL, partial URI, or short name are valid. Examples: * https://www.googleapis.com/compute/v1/projects/(https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement) feature, you must use the short name of the machine type resource, for example, n1-standard-2`.

managedGroupConfigs WorkflowTemplatePlacementManagedClusterConfigWorkerConfigManagedGroupConfig[]

Output only. The config for Compute Engine Instance Group Manager that manages this group. This is only used for preemptible instance groups.

minCpuPlatform string

Optional. Specifies the minimum cpu platform for the Instance Group. See (https://cloud.google.com/dataproc/docs/concepts/compute/dataproc-min-cpu).

numInstances number

Optional. The number of VM instances in the instance group. For master instance groups, must be set to 1.

preemptibility string

Optional. Specifies the preemptibility of the instance group. The default value for master and worker groups is NON_PREEMPTIBLE. This default cannot be changed. The default value for secondary instances is PREEMPTIBLE. Possible values: PREEMPTIBILITY_UNSPECIFIED, NON_PREEMPTIBLE, PREEMPTIBLE

accelerators Sequence[WorkflowTemplatePlacementManagedClusterConfigWorkerConfigAccelerator]

Optional. The Compute Engine accelerator configuration for these instances.

disk_config WorkflowTemplatePlacementManagedClusterConfigWorkerConfigDiskConfig

Optional. Disk option config settings.

image str

Optional. The Compute Engine image resource used for cluster instances. The URI can represent an image or image family. Image examples: * https://www.googleapis.com/compute/beta/projects/ If the URI is unspecified, it will be inferred from SoftwareConfig.image_version or the system default.

instance_names Sequence[str]

Output only. The list of instance names. Dataproc derives the names from cluster_name, num_instances, and the instance group.

is_preemptible bool

Output only. Specifies that this instance group contains preemptible instances.

machine_type str

Optional. The Compute Engine machine type used for cluster instances. A full URL, partial URI, or short name are valid. Examples: * https://www.googleapis.com/compute/v1/projects/(https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement) feature, you must use the short name of the machine type resource, for example, n1-standard-2`.

managed_group_configs Sequence[WorkflowTemplatePlacementManagedClusterConfigWorkerConfigManagedGroupConfig]

Output only. The config for Compute Engine Instance Group Manager that manages this group. This is only used for preemptible instance groups.

min_cpu_platform str

Optional. Specifies the minimum cpu platform for the Instance Group. See (https://cloud.google.com/dataproc/docs/concepts/compute/dataproc-min-cpu).

num_instances int

Optional. The number of VM instances in the instance group. For master instance groups, must be set to 1.

preemptibility str

Optional. Specifies the preemptibility of the instance group. The default value for master and worker groups is NON_PREEMPTIBLE. This default cannot be changed. The default value for secondary instances is PREEMPTIBLE. Possible values: PREEMPTIBILITY_UNSPECIFIED, NON_PREEMPTIBLE, PREEMPTIBLE

accelerators List<Property Map>

Optional. The Compute Engine accelerator configuration for these instances.

diskConfig Property Map

Optional. Disk option config settings.

image String

Optional. The Compute Engine image resource used for cluster instances. The URI can represent an image or image family. Image examples: * https://www.googleapis.com/compute/beta/projects/ If the URI is unspecified, it will be inferred from SoftwareConfig.image_version or the system default.

instanceNames List<String>

Output only. The list of instance names. Dataproc derives the names from cluster_name, num_instances, and the instance group.

isPreemptible Boolean

Output only. Specifies that this instance group contains preemptible instances.

machineType String

Optional. The Compute Engine machine type used for cluster instances. A full URL, partial URI, or short name are valid. Examples: * https://www.googleapis.com/compute/v1/projects/(https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement) feature, you must use the short name of the machine type resource, for example, n1-standard-2`.

managedGroupConfigs List<Property Map>

Output only. The config for Compute Engine Instance Group Manager that manages this group. This is only used for preemptible instance groups.

minCpuPlatform String

Optional. Specifies the minimum cpu platform for the Instance Group. See (https://cloud.google.com/dataproc/docs/concepts/compute/dataproc-min-cpu).

numInstances Number

Optional. The number of VM instances in the instance group. For master instance groups, must be set to 1.

preemptibility String

Optional. Specifies the preemptibility of the instance group. The default value for master and worker groups is NON_PREEMPTIBLE. This default cannot be changed. The default value for secondary instances is PREEMPTIBLE. Possible values: PREEMPTIBILITY_UNSPECIFIED, NON_PREEMPTIBLE, PREEMPTIBLE

WorkflowTemplatePlacementManagedClusterConfigWorkerConfigAccelerator

AcceleratorCount int

The number of the accelerator cards of this type exposed to this instance.

AcceleratorType string

Full URL, partial URI, or short name of the accelerator type resource to expose to this instance. See (https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement) feature, you must use the short name of the accelerator type resource, for example, nvidia-tesla-k80.

AcceleratorCount int

The number of the accelerator cards of this type exposed to this instance.

AcceleratorType string

Full URL, partial URI, or short name of the accelerator type resource to expose to this instance. See (https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement) feature, you must use the short name of the accelerator type resource, for example, nvidia-tesla-k80.

acceleratorCount Integer

The number of the accelerator cards of this type exposed to this instance.

acceleratorType String

Full URL, partial URI, or short name of the accelerator type resource to expose to this instance. See (https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement) feature, you must use the short name of the accelerator type resource, for example, nvidia-tesla-k80.

acceleratorCount number

The number of the accelerator cards of this type exposed to this instance.

acceleratorType string

Full URL, partial URI, or short name of the accelerator type resource to expose to this instance. See (https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement) feature, you must use the short name of the accelerator type resource, for example, nvidia-tesla-k80.

accelerator_count int

The number of the accelerator cards of this type exposed to this instance.

accelerator_type str

Full URL, partial URI, or short name of the accelerator type resource to expose to this instance. See (https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement) feature, you must use the short name of the accelerator type resource, for example, nvidia-tesla-k80.

acceleratorCount Number

The number of the accelerator cards of this type exposed to this instance.

acceleratorType String

Full URL, partial URI, or short name of the accelerator type resource to expose to this instance. See (https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement) feature, you must use the short name of the accelerator type resource, for example, nvidia-tesla-k80.

WorkflowTemplatePlacementManagedClusterConfigWorkerConfigDiskConfig

BootDiskSizeGb int

Optional. Size in GB of the boot disk (default is 500GB).

BootDiskType string

Optional. Type of the boot disk (default is "pd-standard"). Valid values: "pd-ssd" (Persistent Disk Solid State Drive) or "pd-standard" (Persistent Disk Hard Disk Drive).

NumLocalSsds int

Optional. Number of attached SSDs, from 0 to 4 (default is 0). If SSDs are not attached, the boot disk is used to store runtime logs and (https://hadoop.apache.org/docs/r1.2.1/hdfs_user_guide.html) data. If one or more SSDs are attached, this runtime bulk data is spread across them, and the boot disk contains only basic config and installed binaries.

BootDiskSizeGb int

Optional. Size in GB of the boot disk (default is 500GB).

BootDiskType string

Optional. Type of the boot disk (default is "pd-standard"). Valid values: "pd-ssd" (Persistent Disk Solid State Drive) or "pd-standard" (Persistent Disk Hard Disk Drive).

NumLocalSsds int

Optional. Number of attached SSDs, from 0 to 4 (default is 0). If SSDs are not attached, the boot disk is used to store runtime logs and (https://hadoop.apache.org/docs/r1.2.1/hdfs_user_guide.html) data. If one or more SSDs are attached, this runtime bulk data is spread across them, and the boot disk contains only basic config and installed binaries.

bootDiskSizeGb Integer

Optional. Size in GB of the boot disk (default is 500GB).

bootDiskType String

Optional. Type of the boot disk (default is "pd-standard"). Valid values: "pd-ssd" (Persistent Disk Solid State Drive) or "pd-standard" (Persistent Disk Hard Disk Drive).

numLocalSsds Integer

Optional. Number of attached SSDs, from 0 to 4 (default is 0). If SSDs are not attached, the boot disk is used to store runtime logs and (https://hadoop.apache.org/docs/r1.2.1/hdfs_user_guide.html) data. If one or more SSDs are attached, this runtime bulk data is spread across them, and the boot disk contains only basic config and installed binaries.

bootDiskSizeGb number

Optional. Size in GB of the boot disk (default is 500GB).

bootDiskType string

Optional. Type of the boot disk (default is "pd-standard"). Valid values: "pd-ssd" (Persistent Disk Solid State Drive) or "pd-standard" (Persistent Disk Hard Disk Drive).

numLocalSsds number

Optional. Number of attached SSDs, from 0 to 4 (default is 0). If SSDs are not attached, the boot disk is used to store runtime logs and (https://hadoop.apache.org/docs/r1.2.1/hdfs_user_guide.html) data. If one or more SSDs are attached, this runtime bulk data is spread across them, and the boot disk contains only basic config and installed binaries.

boot_disk_size_gb int

Optional. Size in GB of the boot disk (default is 500GB).

boot_disk_type str

Optional. Type of the boot disk (default is "pd-standard"). Valid values: "pd-ssd" (Persistent Disk Solid State Drive) or "pd-standard" (Persistent Disk Hard Disk Drive).

num_local_ssds int

Optional. Number of attached SSDs, from 0 to 4 (default is 0). If SSDs are not attached, the boot disk is used to store runtime logs and (https://hadoop.apache.org/docs/r1.2.1/hdfs_user_guide.html) data. If one or more SSDs are attached, this runtime bulk data is spread across them, and the boot disk contains only basic config and installed binaries.

bootDiskSizeGb Number

Optional. Size in GB of the boot disk (default is 500GB).

bootDiskType String

Optional. Type of the boot disk (default is "pd-standard"). Valid values: "pd-ssd" (Persistent Disk Solid State Drive) or "pd-standard" (Persistent Disk Hard Disk Drive).

numLocalSsds Number

Optional. Number of attached SSDs, from 0 to 4 (default is 0). If SSDs are not attached, the boot disk is used to store runtime logs and (https://hadoop.apache.org/docs/r1.2.1/hdfs_user_guide.html) data. If one or more SSDs are attached, this runtime bulk data is spread across them, and the boot disk contains only basic config and installed binaries.

WorkflowTemplatePlacementManagedClusterConfigWorkerConfigManagedGroupConfig

Import

WorkflowTemplate can be imported using any of these accepted formats

 $ pulumi import gcp:dataproc/workflowTemplate:WorkflowTemplate default projects/{{project}}/locations/{{location}}/workflowTemplates/{{name}}
 $ pulumi import gcp:dataproc/workflowTemplate:WorkflowTemplate default {{project}}/{{location}}/{{name}}
 $ pulumi import gcp:dataproc/workflowTemplate:WorkflowTemplate default {{location}}/{{name}}

Package Details

Repository
https://github.com/pulumi/pulumi-gcp
License
Apache-2.0
Notes

This Pulumi package is based on the google-beta Terraform Provider.