gcp.dataproc.AutoscalingPolicy

Describes an autoscaling policy for Dataproc cluster autoscaler.

Example Usage

Dataproc Autoscaling Policy

using System.Collections.Generic;
using Pulumi;
using Gcp = Pulumi.Gcp;

return await Deployment.RunAsync(() => 
{
    var asp = new Gcp.Dataproc.AutoscalingPolicy("asp", new()
    {
        PolicyId = "dataproc-policy",
        Location = "us-central1",
        WorkerConfig = new Gcp.Dataproc.Inputs.AutoscalingPolicyWorkerConfigArgs
        {
            MaxInstances = 3,
        },
        BasicAlgorithm = new Gcp.Dataproc.Inputs.AutoscalingPolicyBasicAlgorithmArgs
        {
            YarnConfig = new Gcp.Dataproc.Inputs.AutoscalingPolicyBasicAlgorithmYarnConfigArgs
            {
                GracefulDecommissionTimeout = "30s",
                ScaleUpFactor = 0.5,
                ScaleDownFactor = 0.5,
            },
        },
    });

    var basic = new Gcp.Dataproc.Cluster("basic", new()
    {
        Region = "us-central1",
        ClusterConfig = new Gcp.Dataproc.Inputs.ClusterClusterConfigArgs
        {
            AutoscalingConfig = new Gcp.Dataproc.Inputs.ClusterClusterConfigAutoscalingConfigArgs
            {
                PolicyUri = asp.Name,
            },
        },
    });

});
package main

import (
	"github.com/pulumi/pulumi-gcp/sdk/v6/go/gcp/dataproc"
	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)

func main() {
	pulumi.Run(func(ctx *pulumi.Context) error {
		asp, err := dataproc.NewAutoscalingPolicy(ctx, "asp", &dataproc.AutoscalingPolicyArgs{
			PolicyId: pulumi.String("dataproc-policy"),
			Location: pulumi.String("us-central1"),
			WorkerConfig: &dataproc.AutoscalingPolicyWorkerConfigArgs{
				MaxInstances: pulumi.Int(3),
			},
			BasicAlgorithm: &dataproc.AutoscalingPolicyBasicAlgorithmArgs{
				YarnConfig: &dataproc.AutoscalingPolicyBasicAlgorithmYarnConfigArgs{
					GracefulDecommissionTimeout: pulumi.String("30s"),
					ScaleUpFactor:               pulumi.Float64(0.5),
					ScaleDownFactor:             pulumi.Float64(0.5),
				},
			},
		})
		if err != nil {
			return err
		}
		_, err = dataproc.NewCluster(ctx, "basic", &dataproc.ClusterArgs{
			Region: pulumi.String("us-central1"),
			ClusterConfig: &dataproc.ClusterClusterConfigArgs{
				AutoscalingConfig: &dataproc.ClusterClusterConfigAutoscalingConfigArgs{
					PolicyUri: asp.Name,
				},
			},
		})
		if err != nil {
			return err
		}
		return nil
	})
}
package generated_program;

import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.gcp.dataproc.AutoscalingPolicy;
import com.pulumi.gcp.dataproc.AutoscalingPolicyArgs;
import com.pulumi.gcp.dataproc.inputs.AutoscalingPolicyWorkerConfigArgs;
import com.pulumi.gcp.dataproc.inputs.AutoscalingPolicyBasicAlgorithmArgs;
import com.pulumi.gcp.dataproc.inputs.AutoscalingPolicyBasicAlgorithmYarnConfigArgs;
import com.pulumi.gcp.dataproc.Cluster;
import com.pulumi.gcp.dataproc.ClusterArgs;
import com.pulumi.gcp.dataproc.inputs.ClusterClusterConfigArgs;
import com.pulumi.gcp.dataproc.inputs.ClusterClusterConfigAutoscalingConfigArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;

public class App {
    public static void main(String[] args) {
        Pulumi.run(App::stack);
    }

    public static void stack(Context ctx) {
        var asp = new AutoscalingPolicy("asp", AutoscalingPolicyArgs.builder()        
            .policyId("dataproc-policy")
            .location("us-central1")
            .workerConfig(AutoscalingPolicyWorkerConfigArgs.builder()
                .maxInstances(3)
                .build())
            .basicAlgorithm(AutoscalingPolicyBasicAlgorithmArgs.builder()
                .yarnConfig(AutoscalingPolicyBasicAlgorithmYarnConfigArgs.builder()
                    .gracefulDecommissionTimeout("30s")
                    .scaleUpFactor(0.5)
                    .scaleDownFactor(0.5)
                    .build())
                .build())
            .build());

        var basic = new Cluster("basic", ClusterArgs.builder()        
            .region("us-central1")
            .clusterConfig(ClusterClusterConfigArgs.builder()
                .autoscalingConfig(ClusterClusterConfigAutoscalingConfigArgs.builder()
                    .policyUri(asp.name())
                    .build())
                .build())
            .build());

    }
}
import pulumi
import pulumi_gcp as gcp

asp = gcp.dataproc.AutoscalingPolicy("asp",
    policy_id="dataproc-policy",
    location="us-central1",
    worker_config=gcp.dataproc.AutoscalingPolicyWorkerConfigArgs(
        max_instances=3,
    ),
    basic_algorithm=gcp.dataproc.AutoscalingPolicyBasicAlgorithmArgs(
        yarn_config=gcp.dataproc.AutoscalingPolicyBasicAlgorithmYarnConfigArgs(
            graceful_decommission_timeout="30s",
            scale_up_factor=0.5,
            scale_down_factor=0.5,
        ),
    ))
basic = gcp.dataproc.Cluster("basic",
    region="us-central1",
    cluster_config=gcp.dataproc.ClusterClusterConfigArgs(
        autoscaling_config=gcp.dataproc.ClusterClusterConfigAutoscalingConfigArgs(
            policy_uri=asp.name,
        ),
    ))
import * as pulumi from "@pulumi/pulumi";
import * as gcp from "@pulumi/gcp";

const asp = new gcp.dataproc.AutoscalingPolicy("asp", {
    policyId: "dataproc-policy",
    location: "us-central1",
    workerConfig: {
        maxInstances: 3,
    },
    basicAlgorithm: {
        yarnConfig: {
            gracefulDecommissionTimeout: "30s",
            scaleUpFactor: 0.5,
            scaleDownFactor: 0.5,
        },
    },
});
const basic = new gcp.dataproc.Cluster("basic", {
    region: "us-central1",
    clusterConfig: {
        autoscalingConfig: {
            policyUri: asp.name,
        },
    },
});
resources:
  basic:
    type: gcp:dataproc:Cluster
    properties:
      region: us-central1
      clusterConfig:
        autoscalingConfig:
          policyUri: ${asp.name}
  asp:
    type: gcp:dataproc:AutoscalingPolicy
    properties:
      policyId: dataproc-policy
      location: us-central1
      workerConfig:
        maxInstances: 3
      basicAlgorithm:
        yarnConfig:
          gracefulDecommissionTimeout: 30s
          scaleUpFactor: 0.5
          scaleDownFactor: 0.5

Create AutoscalingPolicy Resource

new AutoscalingPolicy(name: string, args: AutoscalingPolicyArgs, opts?: CustomResourceOptions);
@overload
def AutoscalingPolicy(resource_name: str,
                      opts: Optional[ResourceOptions] = None,
                      basic_algorithm: Optional[AutoscalingPolicyBasicAlgorithmArgs] = None,
                      location: Optional[str] = None,
                      policy_id: Optional[str] = None,
                      project: Optional[str] = None,
                      secondary_worker_config: Optional[AutoscalingPolicySecondaryWorkerConfigArgs] = None,
                      worker_config: Optional[AutoscalingPolicyWorkerConfigArgs] = None)
@overload
def AutoscalingPolicy(resource_name: str,
                      args: AutoscalingPolicyArgs,
                      opts: Optional[ResourceOptions] = None)
func NewAutoscalingPolicy(ctx *Context, name string, args AutoscalingPolicyArgs, opts ...ResourceOption) (*AutoscalingPolicy, error)
public AutoscalingPolicy(string name, AutoscalingPolicyArgs args, CustomResourceOptions? opts = null)
public AutoscalingPolicy(String name, AutoscalingPolicyArgs args)
public AutoscalingPolicy(String name, AutoscalingPolicyArgs args, CustomResourceOptions options)
type: gcp:dataproc:AutoscalingPolicy
properties: # The arguments to resource properties.
options: # Bag of options to control resource's behavior.

name string
The unique name of the resource.
args AutoscalingPolicyArgs
The arguments to resource properties.
opts CustomResourceOptions
Bag of options to control resource's behavior.
resource_name str
The unique name of the resource.
args AutoscalingPolicyArgs
The arguments to resource properties.
opts ResourceOptions
Bag of options to control resource's behavior.
ctx Context
Context object for the current deployment.
name string
The unique name of the resource.
args AutoscalingPolicyArgs
The arguments to resource properties.
opts ResourceOption
Bag of options to control resource's behavior.
name string
The unique name of the resource.
args AutoscalingPolicyArgs
The arguments to resource properties.
opts CustomResourceOptions
Bag of options to control resource's behavior.
name String
The unique name of the resource.
args AutoscalingPolicyArgs
The arguments to resource properties.
options CustomResourceOptions
Bag of options to control resource's behavior.

AutoscalingPolicy Resource Properties

To learn more about resource properties and how to use them, see Inputs and Outputs in the Architecture and Concepts docs.

Inputs

The AutoscalingPolicy resource accepts the following input properties:

PolicyId string

The policy id. The id must contain only letters (a-z, A-Z), numbers (0-9), underscores (_), and hyphens (-). Cannot begin or end with underscore or hyphen. Must consist of between 3 and 50 characters.

BasicAlgorithm AutoscalingPolicyBasicAlgorithmArgs

Basic algorithm for autoscaling. Structure is documented below.

Location string

The location where the autoscaling policy should reside. The default value is global.

Project string

The ID of the project in which the resource belongs. If it is not provided, the provider project is used.

SecondaryWorkerConfig AutoscalingPolicySecondaryWorkerConfigArgs

Describes how the autoscaler will operate for secondary workers. Structure is documented below.

WorkerConfig AutoscalingPolicyWorkerConfigArgs

Describes how the autoscaler will operate for primary workers. Structure is documented below.

PolicyId string

The policy id. The id must contain only letters (a-z, A-Z), numbers (0-9), underscores (_), and hyphens (-). Cannot begin or end with underscore or hyphen. Must consist of between 3 and 50 characters.

BasicAlgorithm AutoscalingPolicyBasicAlgorithmArgs

Basic algorithm for autoscaling. Structure is documented below.

Location string

The location where the autoscaling policy should reside. The default value is global.

Project string

The ID of the project in which the resource belongs. If it is not provided, the provider project is used.

SecondaryWorkerConfig AutoscalingPolicySecondaryWorkerConfigArgs

Describes how the autoscaler will operate for secondary workers. Structure is documented below.

WorkerConfig AutoscalingPolicyWorkerConfigArgs

Describes how the autoscaler will operate for primary workers. Structure is documented below.

policyId String

The policy id. The id must contain only letters (a-z, A-Z), numbers (0-9), underscores (_), and hyphens (-). Cannot begin or end with underscore or hyphen. Must consist of between 3 and 50 characters.

basicAlgorithm AutoscalingPolicyBasicAlgorithmArgs

Basic algorithm for autoscaling. Structure is documented below.

location String

The location where the autoscaling policy should reside. The default value is global.

project String

The ID of the project in which the resource belongs. If it is not provided, the provider project is used.

secondaryWorkerConfig AutoscalingPolicySecondaryWorkerConfigArgs

Describes how the autoscaler will operate for secondary workers. Structure is documented below.

workerConfig AutoscalingPolicyWorkerConfigArgs

Describes how the autoscaler will operate for primary workers. Structure is documented below.

policyId string

The policy id. The id must contain only letters (a-z, A-Z), numbers (0-9), underscores (_), and hyphens (-). Cannot begin or end with underscore or hyphen. Must consist of between 3 and 50 characters.

basicAlgorithm AutoscalingPolicyBasicAlgorithmArgs

Basic algorithm for autoscaling. Structure is documented below.

location string

The location where the autoscaling policy should reside. The default value is global.

project string

The ID of the project in which the resource belongs. If it is not provided, the provider project is used.

secondaryWorkerConfig AutoscalingPolicySecondaryWorkerConfigArgs

Describes how the autoscaler will operate for secondary workers. Structure is documented below.

workerConfig AutoscalingPolicyWorkerConfigArgs

Describes how the autoscaler will operate for primary workers. Structure is documented below.

policy_id str

The policy id. The id must contain only letters (a-z, A-Z), numbers (0-9), underscores (_), and hyphens (-). Cannot begin or end with underscore or hyphen. Must consist of between 3 and 50 characters.

basic_algorithm AutoscalingPolicyBasicAlgorithmArgs

Basic algorithm for autoscaling. Structure is documented below.

location str

The location where the autoscaling policy should reside. The default value is global.

project str

The ID of the project in which the resource belongs. If it is not provided, the provider project is used.

secondary_worker_config AutoscalingPolicySecondaryWorkerConfigArgs

Describes how the autoscaler will operate for secondary workers. Structure is documented below.

worker_config AutoscalingPolicyWorkerConfigArgs

Describes how the autoscaler will operate for primary workers. Structure is documented below.

policyId String

The policy id. The id must contain only letters (a-z, A-Z), numbers (0-9), underscores (_), and hyphens (-). Cannot begin or end with underscore or hyphen. Must consist of between 3 and 50 characters.

basicAlgorithm Property Map

Basic algorithm for autoscaling. Structure is documented below.

location String

The location where the autoscaling policy should reside. The default value is global.

project String

The ID of the project in which the resource belongs. If it is not provided, the provider project is used.

secondaryWorkerConfig Property Map

Describes how the autoscaler will operate for secondary workers. Structure is documented below.

workerConfig Property Map

Describes how the autoscaler will operate for primary workers. Structure is documented below.

Outputs

All input properties are implicitly available as output properties. Additionally, the AutoscalingPolicy resource produces the following output properties:

Id string

The provider-assigned unique ID for this managed resource.

Name string

The "resource name" of the autoscaling policy.

Id string

The provider-assigned unique ID for this managed resource.

Name string

The "resource name" of the autoscaling policy.

id String

The provider-assigned unique ID for this managed resource.

name String

The "resource name" of the autoscaling policy.

id string

The provider-assigned unique ID for this managed resource.

name string

The "resource name" of the autoscaling policy.

id str

The provider-assigned unique ID for this managed resource.

name str

The "resource name" of the autoscaling policy.

id String

The provider-assigned unique ID for this managed resource.

name String

The "resource name" of the autoscaling policy.

Look up Existing AutoscalingPolicy Resource

Get an existing AutoscalingPolicy resource’s state with the given name, ID, and optional extra properties used to qualify the lookup.

public static get(name: string, id: Input<ID>, state?: AutoscalingPolicyState, opts?: CustomResourceOptions): AutoscalingPolicy
@staticmethod
def get(resource_name: str,
        id: str,
        opts: Optional[ResourceOptions] = None,
        basic_algorithm: Optional[AutoscalingPolicyBasicAlgorithmArgs] = None,
        location: Optional[str] = None,
        name: Optional[str] = None,
        policy_id: Optional[str] = None,
        project: Optional[str] = None,
        secondary_worker_config: Optional[AutoscalingPolicySecondaryWorkerConfigArgs] = None,
        worker_config: Optional[AutoscalingPolicyWorkerConfigArgs] = None) -> AutoscalingPolicy
func GetAutoscalingPolicy(ctx *Context, name string, id IDInput, state *AutoscalingPolicyState, opts ...ResourceOption) (*AutoscalingPolicy, error)
public static AutoscalingPolicy Get(string name, Input<string> id, AutoscalingPolicyState? state, CustomResourceOptions? opts = null)
public static AutoscalingPolicy get(String name, Output<String> id, AutoscalingPolicyState state, CustomResourceOptions options)
Resource lookup is not supported in YAML
name
The unique name of the resulting resource.
id
The unique provider ID of the resource to lookup.
state
Any extra arguments used during the lookup.
opts
A bag of options that control this resource's behavior.
resource_name
The unique name of the resulting resource.
id
The unique provider ID of the resource to lookup.
name
The unique name of the resulting resource.
id
The unique provider ID of the resource to lookup.
state
Any extra arguments used during the lookup.
opts
A bag of options that control this resource's behavior.
name
The unique name of the resulting resource.
id
The unique provider ID of the resource to lookup.
state
Any extra arguments used during the lookup.
opts
A bag of options that control this resource's behavior.
name
The unique name of the resulting resource.
id
The unique provider ID of the resource to lookup.
state
Any extra arguments used during the lookup.
opts
A bag of options that control this resource's behavior.
The following state arguments are supported:
BasicAlgorithm AutoscalingPolicyBasicAlgorithmArgs

Basic algorithm for autoscaling. Structure is documented below.

Location string

The location where the autoscaling policy should reside. The default value is global.

Name string

The "resource name" of the autoscaling policy.

PolicyId string

The policy id. The id must contain only letters (a-z, A-Z), numbers (0-9), underscores (_), and hyphens (-). Cannot begin or end with underscore or hyphen. Must consist of between 3 and 50 characters.

Project string

The ID of the project in which the resource belongs. If it is not provided, the provider project is used.

SecondaryWorkerConfig AutoscalingPolicySecondaryWorkerConfigArgs

Describes how the autoscaler will operate for secondary workers. Structure is documented below.

WorkerConfig AutoscalingPolicyWorkerConfigArgs

Describes how the autoscaler will operate for primary workers. Structure is documented below.

BasicAlgorithm AutoscalingPolicyBasicAlgorithmArgs

Basic algorithm for autoscaling. Structure is documented below.

Location string

The location where the autoscaling policy should reside. The default value is global.

Name string

The "resource name" of the autoscaling policy.

PolicyId string

The policy id. The id must contain only letters (a-z, A-Z), numbers (0-9), underscores (_), and hyphens (-). Cannot begin or end with underscore or hyphen. Must consist of between 3 and 50 characters.

Project string

The ID of the project in which the resource belongs. If it is not provided, the provider project is used.

SecondaryWorkerConfig AutoscalingPolicySecondaryWorkerConfigArgs

Describes how the autoscaler will operate for secondary workers. Structure is documented below.

WorkerConfig AutoscalingPolicyWorkerConfigArgs

Describes how the autoscaler will operate for primary workers. Structure is documented below.

basicAlgorithm AutoscalingPolicyBasicAlgorithmArgs

Basic algorithm for autoscaling. Structure is documented below.

location String

The location where the autoscaling policy should reside. The default value is global.

name String

The "resource name" of the autoscaling policy.

policyId String

The policy id. The id must contain only letters (a-z, A-Z), numbers (0-9), underscores (_), and hyphens (-). Cannot begin or end with underscore or hyphen. Must consist of between 3 and 50 characters.

project String

The ID of the project in which the resource belongs. If it is not provided, the provider project is used.

secondaryWorkerConfig AutoscalingPolicySecondaryWorkerConfigArgs

Describes how the autoscaler will operate for secondary workers. Structure is documented below.

workerConfig AutoscalingPolicyWorkerConfigArgs

Describes how the autoscaler will operate for primary workers. Structure is documented below.

basicAlgorithm AutoscalingPolicyBasicAlgorithmArgs

Basic algorithm for autoscaling. Structure is documented below.

location string

The location where the autoscaling policy should reside. The default value is global.

name string

The "resource name" of the autoscaling policy.

policyId string

The policy id. The id must contain only letters (a-z, A-Z), numbers (0-9), underscores (_), and hyphens (-). Cannot begin or end with underscore or hyphen. Must consist of between 3 and 50 characters.

project string

The ID of the project in which the resource belongs. If it is not provided, the provider project is used.

secondaryWorkerConfig AutoscalingPolicySecondaryWorkerConfigArgs

Describes how the autoscaler will operate for secondary workers. Structure is documented below.

workerConfig AutoscalingPolicyWorkerConfigArgs

Describes how the autoscaler will operate for primary workers. Structure is documented below.

basic_algorithm AutoscalingPolicyBasicAlgorithmArgs

Basic algorithm for autoscaling. Structure is documented below.

location str

The location where the autoscaling policy should reside. The default value is global.

name str

The "resource name" of the autoscaling policy.

policy_id str

The policy id. The id must contain only letters (a-z, A-Z), numbers (0-9), underscores (_), and hyphens (-). Cannot begin or end with underscore or hyphen. Must consist of between 3 and 50 characters.

project str

The ID of the project in which the resource belongs. If it is not provided, the provider project is used.

secondary_worker_config AutoscalingPolicySecondaryWorkerConfigArgs

Describes how the autoscaler will operate for secondary workers. Structure is documented below.

worker_config AutoscalingPolicyWorkerConfigArgs

Describes how the autoscaler will operate for primary workers. Structure is documented below.

basicAlgorithm Property Map

Basic algorithm for autoscaling. Structure is documented below.

location String

The location where the autoscaling policy should reside. The default value is global.

name String

The "resource name" of the autoscaling policy.

policyId String

The policy id. The id must contain only letters (a-z, A-Z), numbers (0-9), underscores (_), and hyphens (-). Cannot begin or end with underscore or hyphen. Must consist of between 3 and 50 characters.

project String

The ID of the project in which the resource belongs. If it is not provided, the provider project is used.

secondaryWorkerConfig Property Map

Describes how the autoscaler will operate for secondary workers. Structure is documented below.

workerConfig Property Map

Describes how the autoscaler will operate for primary workers. Structure is documented below.

Supporting Types

AutoscalingPolicyBasicAlgorithm

YarnConfig AutoscalingPolicyBasicAlgorithmYarnConfig

YARN autoscaling configuration. Structure is documented below.

CooldownPeriod string

Duration between scaling events. A scaling period starts after the update operation from the previous event has completed. Bounds: [2m, 1d]. Default: 2m.

YarnConfig AutoscalingPolicyBasicAlgorithmYarnConfig

YARN autoscaling configuration. Structure is documented below.

CooldownPeriod string

Duration between scaling events. A scaling period starts after the update operation from the previous event has completed. Bounds: [2m, 1d]. Default: 2m.

yarnConfig AutoscalingPolicyBasicAlgorithmYarnConfig

YARN autoscaling configuration. Structure is documented below.

cooldownPeriod String

Duration between scaling events. A scaling period starts after the update operation from the previous event has completed. Bounds: [2m, 1d]. Default: 2m.

yarnConfig AutoscalingPolicyBasicAlgorithmYarnConfig

YARN autoscaling configuration. Structure is documented below.

cooldownPeriod string

Duration between scaling events. A scaling period starts after the update operation from the previous event has completed. Bounds: [2m, 1d]. Default: 2m.

yarn_config AutoscalingPolicyBasicAlgorithmYarnConfig

YARN autoscaling configuration. Structure is documented below.

cooldown_period str

Duration between scaling events. A scaling period starts after the update operation from the previous event has completed. Bounds: [2m, 1d]. Default: 2m.

yarnConfig Property Map

YARN autoscaling configuration. Structure is documented below.

cooldownPeriod String

Duration between scaling events. A scaling period starts after the update operation from the previous event has completed. Bounds: [2m, 1d]. Default: 2m.

AutoscalingPolicyBasicAlgorithmYarnConfig

GracefulDecommissionTimeout string

Timeout for YARN graceful decommissioning of Node Managers. Specifies the duration to wait for jobs to complete before forcefully removing workers (and potentially interrupting jobs). Only applicable to downscaling operations. Bounds: [0s, 1d].

ScaleDownFactor double

Fraction of average pending memory in the last cooldown period for which to remove workers. A scale-down factor of 1 will result in scaling down so that there is no available memory remaining after the update (more aggressive scaling). A scale-down factor of 0 disables removing workers, which can be beneficial for autoscaling a single job. Bounds: [0.0, 1.0].

ScaleUpFactor double

Fraction of average pending memory in the last cooldown period for which to add workers. A scale-up factor of 1.0 will result in scaling up so that there is no pending memory remaining after the update (more aggressive scaling). A scale-up factor closer to 0 will result in a smaller magnitude of scaling up (less aggressive scaling). Bounds: [0.0, 1.0].

ScaleDownMinWorkerFraction double

Minimum scale-down threshold as a fraction of total cluster size before scaling occurs. For example, in a 20-worker cluster, a threshold of 0.1 means the autoscaler must recommend at least a 2 worker scale-down for the cluster to scale. A threshold of 0 means the autoscaler will scale down on any recommended change. Bounds: [0.0, 1.0]. Default: 0.0.

ScaleUpMinWorkerFraction double

Minimum scale-up threshold as a fraction of total cluster size before scaling occurs. For example, in a 20-worker cluster, a threshold of 0.1 means the autoscaler must recommend at least a 2-worker scale-up for the cluster to scale. A threshold of 0 means the autoscaler will scale up on any recommended change. Bounds: [0.0, 1.0]. Default: 0.0.

GracefulDecommissionTimeout string

Timeout for YARN graceful decommissioning of Node Managers. Specifies the duration to wait for jobs to complete before forcefully removing workers (and potentially interrupting jobs). Only applicable to downscaling operations. Bounds: [0s, 1d].

ScaleDownFactor float64

Fraction of average pending memory in the last cooldown period for which to remove workers. A scale-down factor of 1 will result in scaling down so that there is no available memory remaining after the update (more aggressive scaling). A scale-down factor of 0 disables removing workers, which can be beneficial for autoscaling a single job. Bounds: [0.0, 1.0].

ScaleUpFactor float64

Fraction of average pending memory in the last cooldown period for which to add workers. A scale-up factor of 1.0 will result in scaling up so that there is no pending memory remaining after the update (more aggressive scaling). A scale-up factor closer to 0 will result in a smaller magnitude of scaling up (less aggressive scaling). Bounds: [0.0, 1.0].

ScaleDownMinWorkerFraction float64

Minimum scale-down threshold as a fraction of total cluster size before scaling occurs. For example, in a 20-worker cluster, a threshold of 0.1 means the autoscaler must recommend at least a 2 worker scale-down for the cluster to scale. A threshold of 0 means the autoscaler will scale down on any recommended change. Bounds: [0.0, 1.0]. Default: 0.0.

ScaleUpMinWorkerFraction float64

Minimum scale-up threshold as a fraction of total cluster size before scaling occurs. For example, in a 20-worker cluster, a threshold of 0.1 means the autoscaler must recommend at least a 2-worker scale-up for the cluster to scale. A threshold of 0 means the autoscaler will scale up on any recommended change. Bounds: [0.0, 1.0]. Default: 0.0.

gracefulDecommissionTimeout String

Timeout for YARN graceful decommissioning of Node Managers. Specifies the duration to wait for jobs to complete before forcefully removing workers (and potentially interrupting jobs). Only applicable to downscaling operations. Bounds: [0s, 1d].

scaleDownFactor Double

Fraction of average pending memory in the last cooldown period for which to remove workers. A scale-down factor of 1 will result in scaling down so that there is no available memory remaining after the update (more aggressive scaling). A scale-down factor of 0 disables removing workers, which can be beneficial for autoscaling a single job. Bounds: [0.0, 1.0].

scaleUpFactor Double

Fraction of average pending memory in the last cooldown period for which to add workers. A scale-up factor of 1.0 will result in scaling up so that there is no pending memory remaining after the update (more aggressive scaling). A scale-up factor closer to 0 will result in a smaller magnitude of scaling up (less aggressive scaling). Bounds: [0.0, 1.0].

scaleDownMinWorkerFraction Double

Minimum scale-down threshold as a fraction of total cluster size before scaling occurs. For example, in a 20-worker cluster, a threshold of 0.1 means the autoscaler must recommend at least a 2 worker scale-down for the cluster to scale. A threshold of 0 means the autoscaler will scale down on any recommended change. Bounds: [0.0, 1.0]. Default: 0.0.

scaleUpMinWorkerFraction Double

Minimum scale-up threshold as a fraction of total cluster size before scaling occurs. For example, in a 20-worker cluster, a threshold of 0.1 means the autoscaler must recommend at least a 2-worker scale-up for the cluster to scale. A threshold of 0 means the autoscaler will scale up on any recommended change. Bounds: [0.0, 1.0]. Default: 0.0.

gracefulDecommissionTimeout string

Timeout for YARN graceful decommissioning of Node Managers. Specifies the duration to wait for jobs to complete before forcefully removing workers (and potentially interrupting jobs). Only applicable to downscaling operations. Bounds: [0s, 1d].

scaleDownFactor number

Fraction of average pending memory in the last cooldown period for which to remove workers. A scale-down factor of 1 will result in scaling down so that there is no available memory remaining after the update (more aggressive scaling). A scale-down factor of 0 disables removing workers, which can be beneficial for autoscaling a single job. Bounds: [0.0, 1.0].

scaleUpFactor number

Fraction of average pending memory in the last cooldown period for which to add workers. A scale-up factor of 1.0 will result in scaling up so that there is no pending memory remaining after the update (more aggressive scaling). A scale-up factor closer to 0 will result in a smaller magnitude of scaling up (less aggressive scaling). Bounds: [0.0, 1.0].

scaleDownMinWorkerFraction number

Minimum scale-down threshold as a fraction of total cluster size before scaling occurs. For example, in a 20-worker cluster, a threshold of 0.1 means the autoscaler must recommend at least a 2 worker scale-down for the cluster to scale. A threshold of 0 means the autoscaler will scale down on any recommended change. Bounds: [0.0, 1.0]. Default: 0.0.

scaleUpMinWorkerFraction number

Minimum scale-up threshold as a fraction of total cluster size before scaling occurs. For example, in a 20-worker cluster, a threshold of 0.1 means the autoscaler must recommend at least a 2-worker scale-up for the cluster to scale. A threshold of 0 means the autoscaler will scale up on any recommended change. Bounds: [0.0, 1.0]. Default: 0.0.

graceful_decommission_timeout str

Timeout for YARN graceful decommissioning of Node Managers. Specifies the duration to wait for jobs to complete before forcefully removing workers (and potentially interrupting jobs). Only applicable to downscaling operations. Bounds: [0s, 1d].

scale_down_factor float

Fraction of average pending memory in the last cooldown period for which to remove workers. A scale-down factor of 1 will result in scaling down so that there is no available memory remaining after the update (more aggressive scaling). A scale-down factor of 0 disables removing workers, which can be beneficial for autoscaling a single job. Bounds: [0.0, 1.0].

scale_up_factor float

Fraction of average pending memory in the last cooldown period for which to add workers. A scale-up factor of 1.0 will result in scaling up so that there is no pending memory remaining after the update (more aggressive scaling). A scale-up factor closer to 0 will result in a smaller magnitude of scaling up (less aggressive scaling). Bounds: [0.0, 1.0].

scale_down_min_worker_fraction float

Minimum scale-down threshold as a fraction of total cluster size before scaling occurs. For example, in a 20-worker cluster, a threshold of 0.1 means the autoscaler must recommend at least a 2 worker scale-down for the cluster to scale. A threshold of 0 means the autoscaler will scale down on any recommended change. Bounds: [0.0, 1.0]. Default: 0.0.

scale_up_min_worker_fraction float

Minimum scale-up threshold as a fraction of total cluster size before scaling occurs. For example, in a 20-worker cluster, a threshold of 0.1 means the autoscaler must recommend at least a 2-worker scale-up for the cluster to scale. A threshold of 0 means the autoscaler will scale up on any recommended change. Bounds: [0.0, 1.0]. Default: 0.0.

gracefulDecommissionTimeout String

Timeout for YARN graceful decommissioning of Node Managers. Specifies the duration to wait for jobs to complete before forcefully removing workers (and potentially interrupting jobs). Only applicable to downscaling operations. Bounds: [0s, 1d].

scaleDownFactor Number

Fraction of average pending memory in the last cooldown period for which to remove workers. A scale-down factor of 1 will result in scaling down so that there is no available memory remaining after the update (more aggressive scaling). A scale-down factor of 0 disables removing workers, which can be beneficial for autoscaling a single job. Bounds: [0.0, 1.0].

scaleUpFactor Number

Fraction of average pending memory in the last cooldown period for which to add workers. A scale-up factor of 1.0 will result in scaling up so that there is no pending memory remaining after the update (more aggressive scaling). A scale-up factor closer to 0 will result in a smaller magnitude of scaling up (less aggressive scaling). Bounds: [0.0, 1.0].

scaleDownMinWorkerFraction Number

Minimum scale-down threshold as a fraction of total cluster size before scaling occurs. For example, in a 20-worker cluster, a threshold of 0.1 means the autoscaler must recommend at least a 2 worker scale-down for the cluster to scale. A threshold of 0 means the autoscaler will scale down on any recommended change. Bounds: [0.0, 1.0]. Default: 0.0.

scaleUpMinWorkerFraction Number

Minimum scale-up threshold as a fraction of total cluster size before scaling occurs. For example, in a 20-worker cluster, a threshold of 0.1 means the autoscaler must recommend at least a 2-worker scale-up for the cluster to scale. A threshold of 0 means the autoscaler will scale up on any recommended change. Bounds: [0.0, 1.0]. Default: 0.0.

AutoscalingPolicySecondaryWorkerConfig

MaxInstances int

Maximum number of instances for this group. Note that by default, clusters will not use secondary workers. Required for secondary workers if the minimum secondary instances is set. Bounds: [minInstances, ). Defaults to 0.

MinInstances int

Minimum number of instances for this group. Bounds: [0, maxInstances]. Defaults to 0.

Weight int

Weight for the instance group, which is used to determine the fraction of total workers in the cluster from this instance group. For example, if primary workers have weight 2, and secondary workers have weight 1, the cluster will have approximately 2 primary workers for each secondary worker. The cluster may not reach the specified balance if constrained by min/max bounds or other autoscaling settings. For example, if maxInstances for secondary workers is 0, then only primary workers will be added. The cluster can also be out of balance when created. If weight is not set on any instance group, the cluster will default to equal weight for all groups: the cluster will attempt to maintain an equal number of workers in each group within the configured size bounds for each group. If weight is set for one group only, the cluster will default to zero weight on the unset group. For example if weight is set only on primary workers, the cluster will use primary workers only and no secondary workers.

MaxInstances int

Maximum number of instances for this group. Note that by default, clusters will not use secondary workers. Required for secondary workers if the minimum secondary instances is set. Bounds: [minInstances, ). Defaults to 0.

MinInstances int

Minimum number of instances for this group. Bounds: [0, maxInstances]. Defaults to 0.

Weight int

Weight for the instance group, which is used to determine the fraction of total workers in the cluster from this instance group. For example, if primary workers have weight 2, and secondary workers have weight 1, the cluster will have approximately 2 primary workers for each secondary worker. The cluster may not reach the specified balance if constrained by min/max bounds or other autoscaling settings. For example, if maxInstances for secondary workers is 0, then only primary workers will be added. The cluster can also be out of balance when created. If weight is not set on any instance group, the cluster will default to equal weight for all groups: the cluster will attempt to maintain an equal number of workers in each group within the configured size bounds for each group. If weight is set for one group only, the cluster will default to zero weight on the unset group. For example if weight is set only on primary workers, the cluster will use primary workers only and no secondary workers.

maxInstances Integer

Maximum number of instances for this group. Note that by default, clusters will not use secondary workers. Required for secondary workers if the minimum secondary instances is set. Bounds: [minInstances, ). Defaults to 0.

minInstances Integer

Minimum number of instances for this group. Bounds: [0, maxInstances]. Defaults to 0.

weight Integer

Weight for the instance group, which is used to determine the fraction of total workers in the cluster from this instance group. For example, if primary workers have weight 2, and secondary workers have weight 1, the cluster will have approximately 2 primary workers for each secondary worker. The cluster may not reach the specified balance if constrained by min/max bounds or other autoscaling settings. For example, if maxInstances for secondary workers is 0, then only primary workers will be added. The cluster can also be out of balance when created. If weight is not set on any instance group, the cluster will default to equal weight for all groups: the cluster will attempt to maintain an equal number of workers in each group within the configured size bounds for each group. If weight is set for one group only, the cluster will default to zero weight on the unset group. For example if weight is set only on primary workers, the cluster will use primary workers only and no secondary workers.

maxInstances number

Maximum number of instances for this group. Note that by default, clusters will not use secondary workers. Required for secondary workers if the minimum secondary instances is set. Bounds: [minInstances, ). Defaults to 0.

minInstances number

Minimum number of instances for this group. Bounds: [0, maxInstances]. Defaults to 0.

weight number

Weight for the instance group, which is used to determine the fraction of total workers in the cluster from this instance group. For example, if primary workers have weight 2, and secondary workers have weight 1, the cluster will have approximately 2 primary workers for each secondary worker. The cluster may not reach the specified balance if constrained by min/max bounds or other autoscaling settings. For example, if maxInstances for secondary workers is 0, then only primary workers will be added. The cluster can also be out of balance when created. If weight is not set on any instance group, the cluster will default to equal weight for all groups: the cluster will attempt to maintain an equal number of workers in each group within the configured size bounds for each group. If weight is set for one group only, the cluster will default to zero weight on the unset group. For example if weight is set only on primary workers, the cluster will use primary workers only and no secondary workers.

max_instances int

Maximum number of instances for this group. Note that by default, clusters will not use secondary workers. Required for secondary workers if the minimum secondary instances is set. Bounds: [minInstances, ). Defaults to 0.

min_instances int

Minimum number of instances for this group. Bounds: [0, maxInstances]. Defaults to 0.

weight int

Weight for the instance group, which is used to determine the fraction of total workers in the cluster from this instance group. For example, if primary workers have weight 2, and secondary workers have weight 1, the cluster will have approximately 2 primary workers for each secondary worker. The cluster may not reach the specified balance if constrained by min/max bounds or other autoscaling settings. For example, if maxInstances for secondary workers is 0, then only primary workers will be added. The cluster can also be out of balance when created. If weight is not set on any instance group, the cluster will default to equal weight for all groups: the cluster will attempt to maintain an equal number of workers in each group within the configured size bounds for each group. If weight is set for one group only, the cluster will default to zero weight on the unset group. For example if weight is set only on primary workers, the cluster will use primary workers only and no secondary workers.

maxInstances Number

Maximum number of instances for this group. Note that by default, clusters will not use secondary workers. Required for secondary workers if the minimum secondary instances is set. Bounds: [minInstances, ). Defaults to 0.

minInstances Number

Minimum number of instances for this group. Bounds: [0, maxInstances]. Defaults to 0.

weight Number

Weight for the instance group, which is used to determine the fraction of total workers in the cluster from this instance group. For example, if primary workers have weight 2, and secondary workers have weight 1, the cluster will have approximately 2 primary workers for each secondary worker. The cluster may not reach the specified balance if constrained by min/max bounds or other autoscaling settings. For example, if maxInstances for secondary workers is 0, then only primary workers will be added. The cluster can also be out of balance when created. If weight is not set on any instance group, the cluster will default to equal weight for all groups: the cluster will attempt to maintain an equal number of workers in each group within the configured size bounds for each group. If weight is set for one group only, the cluster will default to zero weight on the unset group. For example if weight is set only on primary workers, the cluster will use primary workers only and no secondary workers.

AutoscalingPolicyWorkerConfig

MaxInstances int

Maximum number of instances for this group.

MinInstances int

Minimum number of instances for this group. Bounds: [2, maxInstances]. Defaults to 2.

Weight int

Weight for the instance group, which is used to determine the fraction of total workers in the cluster from this instance group. For example, if primary workers have weight 2, and secondary workers have weight 1, the cluster will have approximately 2 primary workers for each secondary worker. The cluster may not reach the specified balance if constrained by min/max bounds or other autoscaling settings. For example, if maxInstances for secondary workers is 0, then only primary workers will be added. The cluster can also be out of balance when created. If weight is not set on any instance group, the cluster will default to equal weight for all groups: the cluster will attempt to maintain an equal number of workers in each group within the configured size bounds for each group. If weight is set for one group only, the cluster will default to zero weight on the unset group. For example if weight is set only on primary workers, the cluster will use primary workers only and no secondary workers.

MaxInstances int

Maximum number of instances for this group.

MinInstances int

Minimum number of instances for this group. Bounds: [2, maxInstances]. Defaults to 2.

Weight int

Weight for the instance group, which is used to determine the fraction of total workers in the cluster from this instance group. For example, if primary workers have weight 2, and secondary workers have weight 1, the cluster will have approximately 2 primary workers for each secondary worker. The cluster may not reach the specified balance if constrained by min/max bounds or other autoscaling settings. For example, if maxInstances for secondary workers is 0, then only primary workers will be added. The cluster can also be out of balance when created. If weight is not set on any instance group, the cluster will default to equal weight for all groups: the cluster will attempt to maintain an equal number of workers in each group within the configured size bounds for each group. If weight is set for one group only, the cluster will default to zero weight on the unset group. For example if weight is set only on primary workers, the cluster will use primary workers only and no secondary workers.

maxInstances Integer

Maximum number of instances for this group.

minInstances Integer

Minimum number of instances for this group. Bounds: [2, maxInstances]. Defaults to 2.

weight Integer

Weight for the instance group, which is used to determine the fraction of total workers in the cluster from this instance group. For example, if primary workers have weight 2, and secondary workers have weight 1, the cluster will have approximately 2 primary workers for each secondary worker. The cluster may not reach the specified balance if constrained by min/max bounds or other autoscaling settings. For example, if maxInstances for secondary workers is 0, then only primary workers will be added. The cluster can also be out of balance when created. If weight is not set on any instance group, the cluster will default to equal weight for all groups: the cluster will attempt to maintain an equal number of workers in each group within the configured size bounds for each group. If weight is set for one group only, the cluster will default to zero weight on the unset group. For example if weight is set only on primary workers, the cluster will use primary workers only and no secondary workers.

maxInstances number

Maximum number of instances for this group.

minInstances number

Minimum number of instances for this group. Bounds: [2, maxInstances]. Defaults to 2.

weight number

Weight for the instance group, which is used to determine the fraction of total workers in the cluster from this instance group. For example, if primary workers have weight 2, and secondary workers have weight 1, the cluster will have approximately 2 primary workers for each secondary worker. The cluster may not reach the specified balance if constrained by min/max bounds or other autoscaling settings. For example, if maxInstances for secondary workers is 0, then only primary workers will be added. The cluster can also be out of balance when created. If weight is not set on any instance group, the cluster will default to equal weight for all groups: the cluster will attempt to maintain an equal number of workers in each group within the configured size bounds for each group. If weight is set for one group only, the cluster will default to zero weight on the unset group. For example if weight is set only on primary workers, the cluster will use primary workers only and no secondary workers.

max_instances int

Maximum number of instances for this group.

min_instances int

Minimum number of instances for this group. Bounds: [2, maxInstances]. Defaults to 2.

weight int

Weight for the instance group, which is used to determine the fraction of total workers in the cluster from this instance group. For example, if primary workers have weight 2, and secondary workers have weight 1, the cluster will have approximately 2 primary workers for each secondary worker. The cluster may not reach the specified balance if constrained by min/max bounds or other autoscaling settings. For example, if maxInstances for secondary workers is 0, then only primary workers will be added. The cluster can also be out of balance when created. If weight is not set on any instance group, the cluster will default to equal weight for all groups: the cluster will attempt to maintain an equal number of workers in each group within the configured size bounds for each group. If weight is set for one group only, the cluster will default to zero weight on the unset group. For example if weight is set only on primary workers, the cluster will use primary workers only and no secondary workers.

maxInstances Number

Maximum number of instances for this group.

minInstances Number

Minimum number of instances for this group. Bounds: [2, maxInstances]. Defaults to 2.

weight Number

Weight for the instance group, which is used to determine the fraction of total workers in the cluster from this instance group. For example, if primary workers have weight 2, and secondary workers have weight 1, the cluster will have approximately 2 primary workers for each secondary worker. The cluster may not reach the specified balance if constrained by min/max bounds or other autoscaling settings. For example, if maxInstances for secondary workers is 0, then only primary workers will be added. The cluster can also be out of balance when created. If weight is not set on any instance group, the cluster will default to equal weight for all groups: the cluster will attempt to maintain an equal number of workers in each group within the configured size bounds for each group. If weight is set for one group only, the cluster will default to zero weight on the unset group. For example if weight is set only on primary workers, the cluster will use primary workers only and no secondary workers.

Import

AutoscalingPolicy can be imported using any of these accepted formats

 $ pulumi import gcp:dataproc/autoscalingPolicy:AutoscalingPolicy default projects/{{project}}/locations/{{location}}/autoscalingPolicies/{{policy_id}}
 $ pulumi import gcp:dataproc/autoscalingPolicy:AutoscalingPolicy default {{project}}/{{location}}/{{policy_id}}
 $ pulumi import gcp:dataproc/autoscalingPolicy:AutoscalingPolicy default {{location}}/{{policy_id}}

Package Details

Repository
Google Cloud (GCP) Classic pulumi/pulumi-gcp
License
Apache-2.0
Notes

This Pulumi package is based on the google-beta Terraform Provider.