1. Packages
  2. Google Cloud (GCP) Classic
  3. API Docs
  4. dataproc
  5. AutoscalingPolicy
Google Cloud Classic v6.67.0 published on Wednesday, Sep 27, 2023 by Pulumi

gcp.dataproc.AutoscalingPolicy

Explore with Pulumi AI

gcp logo
Google Cloud Classic v6.67.0 published on Wednesday, Sep 27, 2023 by Pulumi

    Describes an autoscaling policy for Dataproc cluster autoscaler.

    Example Usage

    Dataproc Autoscaling Policy

    using System.Collections.Generic;
    using System.Linq;
    using Pulumi;
    using Gcp = Pulumi.Gcp;
    
    return await Deployment.RunAsync(() => 
    {
        var asp = new Gcp.Dataproc.AutoscalingPolicy("asp", new()
        {
            PolicyId = "dataproc-policy",
            Location = "us-central1",
            WorkerConfig = new Gcp.Dataproc.Inputs.AutoscalingPolicyWorkerConfigArgs
            {
                MaxInstances = 3,
            },
            BasicAlgorithm = new Gcp.Dataproc.Inputs.AutoscalingPolicyBasicAlgorithmArgs
            {
                YarnConfig = new Gcp.Dataproc.Inputs.AutoscalingPolicyBasicAlgorithmYarnConfigArgs
                {
                    GracefulDecommissionTimeout = "30s",
                    ScaleUpFactor = 0.5,
                    ScaleDownFactor = 0.5,
                },
            },
        });
    
        var basic = new Gcp.Dataproc.Cluster("basic", new()
        {
            Region = "us-central1",
            ClusterConfig = new Gcp.Dataproc.Inputs.ClusterClusterConfigArgs
            {
                AutoscalingConfig = new Gcp.Dataproc.Inputs.ClusterClusterConfigAutoscalingConfigArgs
                {
                    PolicyUri = asp.Name,
                },
            },
        });
    
    });
    
    package main
    
    import (
    	"github.com/pulumi/pulumi-gcp/sdk/v6/go/gcp/dataproc"
    	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
    )
    
    func main() {
    	pulumi.Run(func(ctx *pulumi.Context) error {
    		asp, err := dataproc.NewAutoscalingPolicy(ctx, "asp", &dataproc.AutoscalingPolicyArgs{
    			PolicyId: pulumi.String("dataproc-policy"),
    			Location: pulumi.String("us-central1"),
    			WorkerConfig: &dataproc.AutoscalingPolicyWorkerConfigArgs{
    				MaxInstances: pulumi.Int(3),
    			},
    			BasicAlgorithm: &dataproc.AutoscalingPolicyBasicAlgorithmArgs{
    				YarnConfig: &dataproc.AutoscalingPolicyBasicAlgorithmYarnConfigArgs{
    					GracefulDecommissionTimeout: pulumi.String("30s"),
    					ScaleUpFactor:               pulumi.Float64(0.5),
    					ScaleDownFactor:             pulumi.Float64(0.5),
    				},
    			},
    		})
    		if err != nil {
    			return err
    		}
    		_, err = dataproc.NewCluster(ctx, "basic", &dataproc.ClusterArgs{
    			Region: pulumi.String("us-central1"),
    			ClusterConfig: &dataproc.ClusterClusterConfigArgs{
    				AutoscalingConfig: &dataproc.ClusterClusterConfigAutoscalingConfigArgs{
    					PolicyUri: asp.Name,
    				},
    			},
    		})
    		if err != nil {
    			return err
    		}
    		return nil
    	})
    }
    
    package generated_program;
    
    import com.pulumi.Context;
    import com.pulumi.Pulumi;
    import com.pulumi.core.Output;
    import com.pulumi.gcp.dataproc.AutoscalingPolicy;
    import com.pulumi.gcp.dataproc.AutoscalingPolicyArgs;
    import com.pulumi.gcp.dataproc.inputs.AutoscalingPolicyWorkerConfigArgs;
    import com.pulumi.gcp.dataproc.inputs.AutoscalingPolicyBasicAlgorithmArgs;
    import com.pulumi.gcp.dataproc.inputs.AutoscalingPolicyBasicAlgorithmYarnConfigArgs;
    import com.pulumi.gcp.dataproc.Cluster;
    import com.pulumi.gcp.dataproc.ClusterArgs;
    import com.pulumi.gcp.dataproc.inputs.ClusterClusterConfigArgs;
    import com.pulumi.gcp.dataproc.inputs.ClusterClusterConfigAutoscalingConfigArgs;
    import java.util.List;
    import java.util.ArrayList;
    import java.util.Map;
    import java.io.File;
    import java.nio.file.Files;
    import java.nio.file.Paths;
    
    public class App {
        public static void main(String[] args) {
            Pulumi.run(App::stack);
        }
    
        public static void stack(Context ctx) {
            var asp = new AutoscalingPolicy("asp", AutoscalingPolicyArgs.builder()        
                .policyId("dataproc-policy")
                .location("us-central1")
                .workerConfig(AutoscalingPolicyWorkerConfigArgs.builder()
                    .maxInstances(3)
                    .build())
                .basicAlgorithm(AutoscalingPolicyBasicAlgorithmArgs.builder()
                    .yarnConfig(AutoscalingPolicyBasicAlgorithmYarnConfigArgs.builder()
                        .gracefulDecommissionTimeout("30s")
                        .scaleUpFactor(0.5)
                        .scaleDownFactor(0.5)
                        .build())
                    .build())
                .build());
    
            var basic = new Cluster("basic", ClusterArgs.builder()        
                .region("us-central1")
                .clusterConfig(ClusterClusterConfigArgs.builder()
                    .autoscalingConfig(ClusterClusterConfigAutoscalingConfigArgs.builder()
                        .policyUri(asp.name())
                        .build())
                    .build())
                .build());
    
        }
    }
    
    import pulumi
    import pulumi_gcp as gcp
    
    asp = gcp.dataproc.AutoscalingPolicy("asp",
        policy_id="dataproc-policy",
        location="us-central1",
        worker_config=gcp.dataproc.AutoscalingPolicyWorkerConfigArgs(
            max_instances=3,
        ),
        basic_algorithm=gcp.dataproc.AutoscalingPolicyBasicAlgorithmArgs(
            yarn_config=gcp.dataproc.AutoscalingPolicyBasicAlgorithmYarnConfigArgs(
                graceful_decommission_timeout="30s",
                scale_up_factor=0.5,
                scale_down_factor=0.5,
            ),
        ))
    basic = gcp.dataproc.Cluster("basic",
        region="us-central1",
        cluster_config=gcp.dataproc.ClusterClusterConfigArgs(
            autoscaling_config=gcp.dataproc.ClusterClusterConfigAutoscalingConfigArgs(
                policy_uri=asp.name,
            ),
        ))
    
    import * as pulumi from "@pulumi/pulumi";
    import * as gcp from "@pulumi/gcp";
    
    const asp = new gcp.dataproc.AutoscalingPolicy("asp", {
        policyId: "dataproc-policy",
        location: "us-central1",
        workerConfig: {
            maxInstances: 3,
        },
        basicAlgorithm: {
            yarnConfig: {
                gracefulDecommissionTimeout: "30s",
                scaleUpFactor: 0.5,
                scaleDownFactor: 0.5,
            },
        },
    });
    const basic = new gcp.dataproc.Cluster("basic", {
        region: "us-central1",
        clusterConfig: {
            autoscalingConfig: {
                policyUri: asp.name,
            },
        },
    });
    
    resources:
      basic:
        type: gcp:dataproc:Cluster
        properties:
          region: us-central1
          clusterConfig:
            autoscalingConfig:
              policyUri: ${asp.name}
      asp:
        type: gcp:dataproc:AutoscalingPolicy
        properties:
          policyId: dataproc-policy
          location: us-central1
          workerConfig:
            maxInstances: 3
          basicAlgorithm:
            yarnConfig:
              gracefulDecommissionTimeout: 30s
              scaleUpFactor: 0.5
              scaleDownFactor: 0.5
    

    Create AutoscalingPolicy Resource

    new AutoscalingPolicy(name: string, args: AutoscalingPolicyArgs, opts?: CustomResourceOptions);
    @overload
    def AutoscalingPolicy(resource_name: str,
                          opts: Optional[ResourceOptions] = None,
                          basic_algorithm: Optional[AutoscalingPolicyBasicAlgorithmArgs] = None,
                          location: Optional[str] = None,
                          policy_id: Optional[str] = None,
                          project: Optional[str] = None,
                          secondary_worker_config: Optional[AutoscalingPolicySecondaryWorkerConfigArgs] = None,
                          worker_config: Optional[AutoscalingPolicyWorkerConfigArgs] = None)
    @overload
    def AutoscalingPolicy(resource_name: str,
                          args: AutoscalingPolicyArgs,
                          opts: Optional[ResourceOptions] = None)
    func NewAutoscalingPolicy(ctx *Context, name string, args AutoscalingPolicyArgs, opts ...ResourceOption) (*AutoscalingPolicy, error)
    public AutoscalingPolicy(string name, AutoscalingPolicyArgs args, CustomResourceOptions? opts = null)
    public AutoscalingPolicy(String name, AutoscalingPolicyArgs args)
    public AutoscalingPolicy(String name, AutoscalingPolicyArgs args, CustomResourceOptions options)
    
    type: gcp:dataproc:AutoscalingPolicy
    properties: # The arguments to resource properties.
    options: # Bag of options to control resource's behavior.
    
    
    name string
    The unique name of the resource.
    args AutoscalingPolicyArgs
    The arguments to resource properties.
    opts CustomResourceOptions
    Bag of options to control resource's behavior.
    resource_name str
    The unique name of the resource.
    args AutoscalingPolicyArgs
    The arguments to resource properties.
    opts ResourceOptions
    Bag of options to control resource's behavior.
    ctx Context
    Context object for the current deployment.
    name string
    The unique name of the resource.
    args AutoscalingPolicyArgs
    The arguments to resource properties.
    opts ResourceOption
    Bag of options to control resource's behavior.
    name string
    The unique name of the resource.
    args AutoscalingPolicyArgs
    The arguments to resource properties.
    opts CustomResourceOptions
    Bag of options to control resource's behavior.
    name String
    The unique name of the resource.
    args AutoscalingPolicyArgs
    The arguments to resource properties.
    options CustomResourceOptions
    Bag of options to control resource's behavior.

    AutoscalingPolicy Resource Properties

    To learn more about resource properties and how to use them, see Inputs and Outputs in the Architecture and Concepts docs.

    Inputs

    The AutoscalingPolicy resource accepts the following input properties:

    PolicyId string

    The policy id. The id must contain only letters (a-z, A-Z), numbers (0-9), underscores (_), and hyphens (-). Cannot begin or end with underscore or hyphen. Must consist of between 3 and 50 characters.


    BasicAlgorithm AutoscalingPolicyBasicAlgorithm

    Basic algorithm for autoscaling. Structure is documented below.

    Location string

    The location where the autoscaling policy should reside. The default value is global.

    Project string

    The ID of the project in which the resource belongs. If it is not provided, the provider project is used.

    SecondaryWorkerConfig AutoscalingPolicySecondaryWorkerConfig

    Describes how the autoscaler will operate for secondary workers. Structure is documented below.

    WorkerConfig AutoscalingPolicyWorkerConfig

    Describes how the autoscaler will operate for primary workers. Structure is documented below.

    PolicyId string

    The policy id. The id must contain only letters (a-z, A-Z), numbers (0-9), underscores (_), and hyphens (-). Cannot begin or end with underscore or hyphen. Must consist of between 3 and 50 characters.


    BasicAlgorithm AutoscalingPolicyBasicAlgorithmArgs

    Basic algorithm for autoscaling. Structure is documented below.

    Location string

    The location where the autoscaling policy should reside. The default value is global.

    Project string

    The ID of the project in which the resource belongs. If it is not provided, the provider project is used.

    SecondaryWorkerConfig AutoscalingPolicySecondaryWorkerConfigArgs

    Describes how the autoscaler will operate for secondary workers. Structure is documented below.

    WorkerConfig AutoscalingPolicyWorkerConfigArgs

    Describes how the autoscaler will operate for primary workers. Structure is documented below.

    policyId String

    The policy id. The id must contain only letters (a-z, A-Z), numbers (0-9), underscores (_), and hyphens (-). Cannot begin or end with underscore or hyphen. Must consist of between 3 and 50 characters.


    basicAlgorithm AutoscalingPolicyBasicAlgorithm

    Basic algorithm for autoscaling. Structure is documented below.

    location String

    The location where the autoscaling policy should reside. The default value is global.

    project String

    The ID of the project in which the resource belongs. If it is not provided, the provider project is used.

    secondaryWorkerConfig AutoscalingPolicySecondaryWorkerConfig

    Describes how the autoscaler will operate for secondary workers. Structure is documented below.

    workerConfig AutoscalingPolicyWorkerConfig

    Describes how the autoscaler will operate for primary workers. Structure is documented below.

    policyId string

    The policy id. The id must contain only letters (a-z, A-Z), numbers (0-9), underscores (_), and hyphens (-). Cannot begin or end with underscore or hyphen. Must consist of between 3 and 50 characters.


    basicAlgorithm AutoscalingPolicyBasicAlgorithm

    Basic algorithm for autoscaling. Structure is documented below.

    location string

    The location where the autoscaling policy should reside. The default value is global.

    project string

    The ID of the project in which the resource belongs. If it is not provided, the provider project is used.

    secondaryWorkerConfig AutoscalingPolicySecondaryWorkerConfig

    Describes how the autoscaler will operate for secondary workers. Structure is documented below.

    workerConfig AutoscalingPolicyWorkerConfig

    Describes how the autoscaler will operate for primary workers. Structure is documented below.

    policy_id str

    The policy id. The id must contain only letters (a-z, A-Z), numbers (0-9), underscores (_), and hyphens (-). Cannot begin or end with underscore or hyphen. Must consist of between 3 and 50 characters.


    basic_algorithm AutoscalingPolicyBasicAlgorithmArgs

    Basic algorithm for autoscaling. Structure is documented below.

    location str

    The location where the autoscaling policy should reside. The default value is global.

    project str

    The ID of the project in which the resource belongs. If it is not provided, the provider project is used.

    secondary_worker_config AutoscalingPolicySecondaryWorkerConfigArgs

    Describes how the autoscaler will operate for secondary workers. Structure is documented below.

    worker_config AutoscalingPolicyWorkerConfigArgs

    Describes how the autoscaler will operate for primary workers. Structure is documented below.

    policyId String

    The policy id. The id must contain only letters (a-z, A-Z), numbers (0-9), underscores (_), and hyphens (-). Cannot begin or end with underscore or hyphen. Must consist of between 3 and 50 characters.


    basicAlgorithm Property Map

    Basic algorithm for autoscaling. Structure is documented below.

    location String

    The location where the autoscaling policy should reside. The default value is global.

    project String

    The ID of the project in which the resource belongs. If it is not provided, the provider project is used.

    secondaryWorkerConfig Property Map

    Describes how the autoscaler will operate for secondary workers. Structure is documented below.

    workerConfig Property Map

    Describes how the autoscaler will operate for primary workers. Structure is documented below.

    Outputs

    All input properties are implicitly available as output properties. Additionally, the AutoscalingPolicy resource produces the following output properties:

    Id string

    The provider-assigned unique ID for this managed resource.

    Name string

    The "resource name" of the autoscaling policy.

    Id string

    The provider-assigned unique ID for this managed resource.

    Name string

    The "resource name" of the autoscaling policy.

    id String

    The provider-assigned unique ID for this managed resource.

    name String

    The "resource name" of the autoscaling policy.

    id string

    The provider-assigned unique ID for this managed resource.

    name string

    The "resource name" of the autoscaling policy.

    id str

    The provider-assigned unique ID for this managed resource.

    name str

    The "resource name" of the autoscaling policy.

    id String

    The provider-assigned unique ID for this managed resource.

    name String

    The "resource name" of the autoscaling policy.

    Look up Existing AutoscalingPolicy Resource

    Get an existing AutoscalingPolicy resource’s state with the given name, ID, and optional extra properties used to qualify the lookup.

    public static get(name: string, id: Input<ID>, state?: AutoscalingPolicyState, opts?: CustomResourceOptions): AutoscalingPolicy
    @staticmethod
    def get(resource_name: str,
            id: str,
            opts: Optional[ResourceOptions] = None,
            basic_algorithm: Optional[AutoscalingPolicyBasicAlgorithmArgs] = None,
            location: Optional[str] = None,
            name: Optional[str] = None,
            policy_id: Optional[str] = None,
            project: Optional[str] = None,
            secondary_worker_config: Optional[AutoscalingPolicySecondaryWorkerConfigArgs] = None,
            worker_config: Optional[AutoscalingPolicyWorkerConfigArgs] = None) -> AutoscalingPolicy
    func GetAutoscalingPolicy(ctx *Context, name string, id IDInput, state *AutoscalingPolicyState, opts ...ResourceOption) (*AutoscalingPolicy, error)
    public static AutoscalingPolicy Get(string name, Input<string> id, AutoscalingPolicyState? state, CustomResourceOptions? opts = null)
    public static AutoscalingPolicy get(String name, Output<String> id, AutoscalingPolicyState state, CustomResourceOptions options)
    Resource lookup is not supported in YAML
    name
    The unique name of the resulting resource.
    id
    The unique provider ID of the resource to lookup.
    state
    Any extra arguments used during the lookup.
    opts
    A bag of options that control this resource's behavior.
    resource_name
    The unique name of the resulting resource.
    id
    The unique provider ID of the resource to lookup.
    name
    The unique name of the resulting resource.
    id
    The unique provider ID of the resource to lookup.
    state
    Any extra arguments used during the lookup.
    opts
    A bag of options that control this resource's behavior.
    name
    The unique name of the resulting resource.
    id
    The unique provider ID of the resource to lookup.
    state
    Any extra arguments used during the lookup.
    opts
    A bag of options that control this resource's behavior.
    name
    The unique name of the resulting resource.
    id
    The unique provider ID of the resource to lookup.
    state
    Any extra arguments used during the lookup.
    opts
    A bag of options that control this resource's behavior.
    The following state arguments are supported:
    BasicAlgorithm AutoscalingPolicyBasicAlgorithm

    Basic algorithm for autoscaling. Structure is documented below.

    Location string

    The location where the autoscaling policy should reside. The default value is global.

    Name string

    The "resource name" of the autoscaling policy.

    PolicyId string

    The policy id. The id must contain only letters (a-z, A-Z), numbers (0-9), underscores (_), and hyphens (-). Cannot begin or end with underscore or hyphen. Must consist of between 3 and 50 characters.


    Project string

    The ID of the project in which the resource belongs. If it is not provided, the provider project is used.

    SecondaryWorkerConfig AutoscalingPolicySecondaryWorkerConfig

    Describes how the autoscaler will operate for secondary workers. Structure is documented below.

    WorkerConfig AutoscalingPolicyWorkerConfig

    Describes how the autoscaler will operate for primary workers. Structure is documented below.

    BasicAlgorithm AutoscalingPolicyBasicAlgorithmArgs

    Basic algorithm for autoscaling. Structure is documented below.

    Location string

    The location where the autoscaling policy should reside. The default value is global.

    Name string

    The "resource name" of the autoscaling policy.

    PolicyId string

    The policy id. The id must contain only letters (a-z, A-Z), numbers (0-9), underscores (_), and hyphens (-). Cannot begin or end with underscore or hyphen. Must consist of between 3 and 50 characters.


    Project string

    The ID of the project in which the resource belongs. If it is not provided, the provider project is used.

    SecondaryWorkerConfig AutoscalingPolicySecondaryWorkerConfigArgs

    Describes how the autoscaler will operate for secondary workers. Structure is documented below.

    WorkerConfig AutoscalingPolicyWorkerConfigArgs

    Describes how the autoscaler will operate for primary workers. Structure is documented below.

    basicAlgorithm AutoscalingPolicyBasicAlgorithm

    Basic algorithm for autoscaling. Structure is documented below.

    location String

    The location where the autoscaling policy should reside. The default value is global.

    name String

    The "resource name" of the autoscaling policy.

    policyId String

    The policy id. The id must contain only letters (a-z, A-Z), numbers (0-9), underscores (_), and hyphens (-). Cannot begin or end with underscore or hyphen. Must consist of between 3 and 50 characters.


    project String

    The ID of the project in which the resource belongs. If it is not provided, the provider project is used.

    secondaryWorkerConfig AutoscalingPolicySecondaryWorkerConfig

    Describes how the autoscaler will operate for secondary workers. Structure is documented below.

    workerConfig AutoscalingPolicyWorkerConfig

    Describes how the autoscaler will operate for primary workers. Structure is documented below.

    basicAlgorithm AutoscalingPolicyBasicAlgorithm

    Basic algorithm for autoscaling. Structure is documented below.

    location string

    The location where the autoscaling policy should reside. The default value is global.

    name string

    The "resource name" of the autoscaling policy.

    policyId string

    The policy id. The id must contain only letters (a-z, A-Z), numbers (0-9), underscores (_), and hyphens (-). Cannot begin or end with underscore or hyphen. Must consist of between 3 and 50 characters.


    project string

    The ID of the project in which the resource belongs. If it is not provided, the provider project is used.

    secondaryWorkerConfig AutoscalingPolicySecondaryWorkerConfig

    Describes how the autoscaler will operate for secondary workers. Structure is documented below.

    workerConfig AutoscalingPolicyWorkerConfig

    Describes how the autoscaler will operate for primary workers. Structure is documented below.

    basic_algorithm AutoscalingPolicyBasicAlgorithmArgs

    Basic algorithm for autoscaling. Structure is documented below.

    location str

    The location where the autoscaling policy should reside. The default value is global.

    name str

    The "resource name" of the autoscaling policy.

    policy_id str

    The policy id. The id must contain only letters (a-z, A-Z), numbers (0-9), underscores (_), and hyphens (-). Cannot begin or end with underscore or hyphen. Must consist of between 3 and 50 characters.


    project str

    The ID of the project in which the resource belongs. If it is not provided, the provider project is used.

    secondary_worker_config AutoscalingPolicySecondaryWorkerConfigArgs

    Describes how the autoscaler will operate for secondary workers. Structure is documented below.

    worker_config AutoscalingPolicyWorkerConfigArgs

    Describes how the autoscaler will operate for primary workers. Structure is documented below.

    basicAlgorithm Property Map

    Basic algorithm for autoscaling. Structure is documented below.

    location String

    The location where the autoscaling policy should reside. The default value is global.

    name String

    The "resource name" of the autoscaling policy.

    policyId String

    The policy id. The id must contain only letters (a-z, A-Z), numbers (0-9), underscores (_), and hyphens (-). Cannot begin or end with underscore or hyphen. Must consist of between 3 and 50 characters.


    project String

    The ID of the project in which the resource belongs. If it is not provided, the provider project is used.

    secondaryWorkerConfig Property Map

    Describes how the autoscaler will operate for secondary workers. Structure is documented below.

    workerConfig Property Map

    Describes how the autoscaler will operate for primary workers. Structure is documented below.

    Supporting Types

    AutoscalingPolicyBasicAlgorithm, AutoscalingPolicyBasicAlgorithmArgs

    YarnConfig AutoscalingPolicyBasicAlgorithmYarnConfig

    YARN autoscaling configuration. Structure is documented below.

    CooldownPeriod string

    Duration between scaling events. A scaling period starts after the update operation from the previous event has completed. Bounds: [2m, 1d]. Default: 2m.

    YarnConfig AutoscalingPolicyBasicAlgorithmYarnConfig

    YARN autoscaling configuration. Structure is documented below.

    CooldownPeriod string

    Duration between scaling events. A scaling period starts after the update operation from the previous event has completed. Bounds: [2m, 1d]. Default: 2m.

    yarnConfig AutoscalingPolicyBasicAlgorithmYarnConfig

    YARN autoscaling configuration. Structure is documented below.

    cooldownPeriod String

    Duration between scaling events. A scaling period starts after the update operation from the previous event has completed. Bounds: [2m, 1d]. Default: 2m.

    yarnConfig AutoscalingPolicyBasicAlgorithmYarnConfig

    YARN autoscaling configuration. Structure is documented below.

    cooldownPeriod string

    Duration between scaling events. A scaling period starts after the update operation from the previous event has completed. Bounds: [2m, 1d]. Default: 2m.

    yarn_config AutoscalingPolicyBasicAlgorithmYarnConfig

    YARN autoscaling configuration. Structure is documented below.

    cooldown_period str

    Duration between scaling events. A scaling period starts after the update operation from the previous event has completed. Bounds: [2m, 1d]. Default: 2m.

    yarnConfig Property Map

    YARN autoscaling configuration. Structure is documented below.

    cooldownPeriod String

    Duration between scaling events. A scaling period starts after the update operation from the previous event has completed. Bounds: [2m, 1d]. Default: 2m.

    AutoscalingPolicyBasicAlgorithmYarnConfig, AutoscalingPolicyBasicAlgorithmYarnConfigArgs

    GracefulDecommissionTimeout string

    Timeout for YARN graceful decommissioning of Node Managers. Specifies the duration to wait for jobs to complete before forcefully removing workers (and potentially interrupting jobs). Only applicable to downscaling operations. Bounds: [0s, 1d].

    ScaleDownFactor double

    Fraction of average pending memory in the last cooldown period for which to remove workers. A scale-down factor of 1 will result in scaling down so that there is no available memory remaining after the update (more aggressive scaling). A scale-down factor of 0 disables removing workers, which can be beneficial for autoscaling a single job. Bounds: [0.0, 1.0].

    ScaleUpFactor double

    Fraction of average pending memory in the last cooldown period for which to add workers. A scale-up factor of 1.0 will result in scaling up so that there is no pending memory remaining after the update (more aggressive scaling). A scale-up factor closer to 0 will result in a smaller magnitude of scaling up (less aggressive scaling). Bounds: [0.0, 1.0].

    ScaleDownMinWorkerFraction double

    Minimum scale-down threshold as a fraction of total cluster size before scaling occurs. For example, in a 20-worker cluster, a threshold of 0.1 means the autoscaler must recommend at least a 2 worker scale-down for the cluster to scale. A threshold of 0 means the autoscaler will scale down on any recommended change. Bounds: [0.0, 1.0]. Default: 0.0.

    ScaleUpMinWorkerFraction double

    Minimum scale-up threshold as a fraction of total cluster size before scaling occurs. For example, in a 20-worker cluster, a threshold of 0.1 means the autoscaler must recommend at least a 2-worker scale-up for the cluster to scale. A threshold of 0 means the autoscaler will scale up on any recommended change. Bounds: [0.0, 1.0]. Default: 0.0.

    GracefulDecommissionTimeout string

    Timeout for YARN graceful decommissioning of Node Managers. Specifies the duration to wait for jobs to complete before forcefully removing workers (and potentially interrupting jobs). Only applicable to downscaling operations. Bounds: [0s, 1d].

    ScaleDownFactor float64

    Fraction of average pending memory in the last cooldown period for which to remove workers. A scale-down factor of 1 will result in scaling down so that there is no available memory remaining after the update (more aggressive scaling). A scale-down factor of 0 disables removing workers, which can be beneficial for autoscaling a single job. Bounds: [0.0, 1.0].

    ScaleUpFactor float64

    Fraction of average pending memory in the last cooldown period for which to add workers. A scale-up factor of 1.0 will result in scaling up so that there is no pending memory remaining after the update (more aggressive scaling). A scale-up factor closer to 0 will result in a smaller magnitude of scaling up (less aggressive scaling). Bounds: [0.0, 1.0].

    ScaleDownMinWorkerFraction float64

    Minimum scale-down threshold as a fraction of total cluster size before scaling occurs. For example, in a 20-worker cluster, a threshold of 0.1 means the autoscaler must recommend at least a 2 worker scale-down for the cluster to scale. A threshold of 0 means the autoscaler will scale down on any recommended change. Bounds: [0.0, 1.0]. Default: 0.0.

    ScaleUpMinWorkerFraction float64

    Minimum scale-up threshold as a fraction of total cluster size before scaling occurs. For example, in a 20-worker cluster, a threshold of 0.1 means the autoscaler must recommend at least a 2-worker scale-up for the cluster to scale. A threshold of 0 means the autoscaler will scale up on any recommended change. Bounds: [0.0, 1.0]. Default: 0.0.

    gracefulDecommissionTimeout String

    Timeout for YARN graceful decommissioning of Node Managers. Specifies the duration to wait for jobs to complete before forcefully removing workers (and potentially interrupting jobs). Only applicable to downscaling operations. Bounds: [0s, 1d].

    scaleDownFactor Double

    Fraction of average pending memory in the last cooldown period for which to remove workers. A scale-down factor of 1 will result in scaling down so that there is no available memory remaining after the update (more aggressive scaling). A scale-down factor of 0 disables removing workers, which can be beneficial for autoscaling a single job. Bounds: [0.0, 1.0].

    scaleUpFactor Double

    Fraction of average pending memory in the last cooldown period for which to add workers. A scale-up factor of 1.0 will result in scaling up so that there is no pending memory remaining after the update (more aggressive scaling). A scale-up factor closer to 0 will result in a smaller magnitude of scaling up (less aggressive scaling). Bounds: [0.0, 1.0].

    scaleDownMinWorkerFraction Double

    Minimum scale-down threshold as a fraction of total cluster size before scaling occurs. For example, in a 20-worker cluster, a threshold of 0.1 means the autoscaler must recommend at least a 2 worker scale-down for the cluster to scale. A threshold of 0 means the autoscaler will scale down on any recommended change. Bounds: [0.0, 1.0]. Default: 0.0.

    scaleUpMinWorkerFraction Double

    Minimum scale-up threshold as a fraction of total cluster size before scaling occurs. For example, in a 20-worker cluster, a threshold of 0.1 means the autoscaler must recommend at least a 2-worker scale-up for the cluster to scale. A threshold of 0 means the autoscaler will scale up on any recommended change. Bounds: [0.0, 1.0]. Default: 0.0.

    gracefulDecommissionTimeout string

    Timeout for YARN graceful decommissioning of Node Managers. Specifies the duration to wait for jobs to complete before forcefully removing workers (and potentially interrupting jobs). Only applicable to downscaling operations. Bounds: [0s, 1d].

    scaleDownFactor number

    Fraction of average pending memory in the last cooldown period for which to remove workers. A scale-down factor of 1 will result in scaling down so that there is no available memory remaining after the update (more aggressive scaling). A scale-down factor of 0 disables removing workers, which can be beneficial for autoscaling a single job. Bounds: [0.0, 1.0].

    scaleUpFactor number

    Fraction of average pending memory in the last cooldown period for which to add workers. A scale-up factor of 1.0 will result in scaling up so that there is no pending memory remaining after the update (more aggressive scaling). A scale-up factor closer to 0 will result in a smaller magnitude of scaling up (less aggressive scaling). Bounds: [0.0, 1.0].

    scaleDownMinWorkerFraction number

    Minimum scale-down threshold as a fraction of total cluster size before scaling occurs. For example, in a 20-worker cluster, a threshold of 0.1 means the autoscaler must recommend at least a 2 worker scale-down for the cluster to scale. A threshold of 0 means the autoscaler will scale down on any recommended change. Bounds: [0.0, 1.0]. Default: 0.0.

    scaleUpMinWorkerFraction number

    Minimum scale-up threshold as a fraction of total cluster size before scaling occurs. For example, in a 20-worker cluster, a threshold of 0.1 means the autoscaler must recommend at least a 2-worker scale-up for the cluster to scale. A threshold of 0 means the autoscaler will scale up on any recommended change. Bounds: [0.0, 1.0]. Default: 0.0.

    graceful_decommission_timeout str

    Timeout for YARN graceful decommissioning of Node Managers. Specifies the duration to wait for jobs to complete before forcefully removing workers (and potentially interrupting jobs). Only applicable to downscaling operations. Bounds: [0s, 1d].

    scale_down_factor float

    Fraction of average pending memory in the last cooldown period for which to remove workers. A scale-down factor of 1 will result in scaling down so that there is no available memory remaining after the update (more aggressive scaling). A scale-down factor of 0 disables removing workers, which can be beneficial for autoscaling a single job. Bounds: [0.0, 1.0].

    scale_up_factor float

    Fraction of average pending memory in the last cooldown period for which to add workers. A scale-up factor of 1.0 will result in scaling up so that there is no pending memory remaining after the update (more aggressive scaling). A scale-up factor closer to 0 will result in a smaller magnitude of scaling up (less aggressive scaling). Bounds: [0.0, 1.0].

    scale_down_min_worker_fraction float

    Minimum scale-down threshold as a fraction of total cluster size before scaling occurs. For example, in a 20-worker cluster, a threshold of 0.1 means the autoscaler must recommend at least a 2 worker scale-down for the cluster to scale. A threshold of 0 means the autoscaler will scale down on any recommended change. Bounds: [0.0, 1.0]. Default: 0.0.

    scale_up_min_worker_fraction float

    Minimum scale-up threshold as a fraction of total cluster size before scaling occurs. For example, in a 20-worker cluster, a threshold of 0.1 means the autoscaler must recommend at least a 2-worker scale-up for the cluster to scale. A threshold of 0 means the autoscaler will scale up on any recommended change. Bounds: [0.0, 1.0]. Default: 0.0.

    gracefulDecommissionTimeout String

    Timeout for YARN graceful decommissioning of Node Managers. Specifies the duration to wait for jobs to complete before forcefully removing workers (and potentially interrupting jobs). Only applicable to downscaling operations. Bounds: [0s, 1d].

    scaleDownFactor Number

    Fraction of average pending memory in the last cooldown period for which to remove workers. A scale-down factor of 1 will result in scaling down so that there is no available memory remaining after the update (more aggressive scaling). A scale-down factor of 0 disables removing workers, which can be beneficial for autoscaling a single job. Bounds: [0.0, 1.0].

    scaleUpFactor Number

    Fraction of average pending memory in the last cooldown period for which to add workers. A scale-up factor of 1.0 will result in scaling up so that there is no pending memory remaining after the update (more aggressive scaling). A scale-up factor closer to 0 will result in a smaller magnitude of scaling up (less aggressive scaling). Bounds: [0.0, 1.0].

    scaleDownMinWorkerFraction Number

    Minimum scale-down threshold as a fraction of total cluster size before scaling occurs. For example, in a 20-worker cluster, a threshold of 0.1 means the autoscaler must recommend at least a 2 worker scale-down for the cluster to scale. A threshold of 0 means the autoscaler will scale down on any recommended change. Bounds: [0.0, 1.0]. Default: 0.0.

    scaleUpMinWorkerFraction Number

    Minimum scale-up threshold as a fraction of total cluster size before scaling occurs. For example, in a 20-worker cluster, a threshold of 0.1 means the autoscaler must recommend at least a 2-worker scale-up for the cluster to scale. A threshold of 0 means the autoscaler will scale up on any recommended change. Bounds: [0.0, 1.0]. Default: 0.0.

    AutoscalingPolicySecondaryWorkerConfig, AutoscalingPolicySecondaryWorkerConfigArgs

    MaxInstances int

    Maximum number of instances for this group. Note that by default, clusters will not use secondary workers. Required for secondary workers if the minimum secondary instances is set. Bounds: [minInstances, ). Defaults to 0.

    MinInstances int

    Minimum number of instances for this group. Bounds: [0, maxInstances]. Defaults to 0.

    Weight int

    Weight for the instance group, which is used to determine the fraction of total workers in the cluster from this instance group. For example, if primary workers have weight 2, and secondary workers have weight 1, the cluster will have approximately 2 primary workers for each secondary worker. The cluster may not reach the specified balance if constrained by min/max bounds or other autoscaling settings. For example, if maxInstances for secondary workers is 0, then only primary workers will be added. The cluster can also be out of balance when created. If weight is not set on any instance group, the cluster will default to equal weight for all groups: the cluster will attempt to maintain an equal number of workers in each group within the configured size bounds for each group. If weight is set for one group only, the cluster will default to zero weight on the unset group. For example if weight is set only on primary workers, the cluster will use primary workers only and no secondary workers.

    MaxInstances int

    Maximum number of instances for this group. Note that by default, clusters will not use secondary workers. Required for secondary workers if the minimum secondary instances is set. Bounds: [minInstances, ). Defaults to 0.

    MinInstances int

    Minimum number of instances for this group. Bounds: [0, maxInstances]. Defaults to 0.

    Weight int

    Weight for the instance group, which is used to determine the fraction of total workers in the cluster from this instance group. For example, if primary workers have weight 2, and secondary workers have weight 1, the cluster will have approximately 2 primary workers for each secondary worker. The cluster may not reach the specified balance if constrained by min/max bounds or other autoscaling settings. For example, if maxInstances for secondary workers is 0, then only primary workers will be added. The cluster can also be out of balance when created. If weight is not set on any instance group, the cluster will default to equal weight for all groups: the cluster will attempt to maintain an equal number of workers in each group within the configured size bounds for each group. If weight is set for one group only, the cluster will default to zero weight on the unset group. For example if weight is set only on primary workers, the cluster will use primary workers only and no secondary workers.

    maxInstances Integer

    Maximum number of instances for this group. Note that by default, clusters will not use secondary workers. Required for secondary workers if the minimum secondary instances is set. Bounds: [minInstances, ). Defaults to 0.

    minInstances Integer

    Minimum number of instances for this group. Bounds: [0, maxInstances]. Defaults to 0.

    weight Integer

    Weight for the instance group, which is used to determine the fraction of total workers in the cluster from this instance group. For example, if primary workers have weight 2, and secondary workers have weight 1, the cluster will have approximately 2 primary workers for each secondary worker. The cluster may not reach the specified balance if constrained by min/max bounds or other autoscaling settings. For example, if maxInstances for secondary workers is 0, then only primary workers will be added. The cluster can also be out of balance when created. If weight is not set on any instance group, the cluster will default to equal weight for all groups: the cluster will attempt to maintain an equal number of workers in each group within the configured size bounds for each group. If weight is set for one group only, the cluster will default to zero weight on the unset group. For example if weight is set only on primary workers, the cluster will use primary workers only and no secondary workers.

    maxInstances number

    Maximum number of instances for this group. Note that by default, clusters will not use secondary workers. Required for secondary workers if the minimum secondary instances is set. Bounds: [minInstances, ). Defaults to 0.

    minInstances number

    Minimum number of instances for this group. Bounds: [0, maxInstances]. Defaults to 0.

    weight number

    Weight for the instance group, which is used to determine the fraction of total workers in the cluster from this instance group. For example, if primary workers have weight 2, and secondary workers have weight 1, the cluster will have approximately 2 primary workers for each secondary worker. The cluster may not reach the specified balance if constrained by min/max bounds or other autoscaling settings. For example, if maxInstances for secondary workers is 0, then only primary workers will be added. The cluster can also be out of balance when created. If weight is not set on any instance group, the cluster will default to equal weight for all groups: the cluster will attempt to maintain an equal number of workers in each group within the configured size bounds for each group. If weight is set for one group only, the cluster will default to zero weight on the unset group. For example if weight is set only on primary workers, the cluster will use primary workers only and no secondary workers.

    max_instances int

    Maximum number of instances for this group. Note that by default, clusters will not use secondary workers. Required for secondary workers if the minimum secondary instances is set. Bounds: [minInstances, ). Defaults to 0.

    min_instances int

    Minimum number of instances for this group. Bounds: [0, maxInstances]. Defaults to 0.

    weight int

    Weight for the instance group, which is used to determine the fraction of total workers in the cluster from this instance group. For example, if primary workers have weight 2, and secondary workers have weight 1, the cluster will have approximately 2 primary workers for each secondary worker. The cluster may not reach the specified balance if constrained by min/max bounds or other autoscaling settings. For example, if maxInstances for secondary workers is 0, then only primary workers will be added. The cluster can also be out of balance when created. If weight is not set on any instance group, the cluster will default to equal weight for all groups: the cluster will attempt to maintain an equal number of workers in each group within the configured size bounds for each group. If weight is set for one group only, the cluster will default to zero weight on the unset group. For example if weight is set only on primary workers, the cluster will use primary workers only and no secondary workers.

    maxInstances Number

    Maximum number of instances for this group. Note that by default, clusters will not use secondary workers. Required for secondary workers if the minimum secondary instances is set. Bounds: [minInstances, ). Defaults to 0.

    minInstances Number

    Minimum number of instances for this group. Bounds: [0, maxInstances]. Defaults to 0.

    weight Number

    Weight for the instance group, which is used to determine the fraction of total workers in the cluster from this instance group. For example, if primary workers have weight 2, and secondary workers have weight 1, the cluster will have approximately 2 primary workers for each secondary worker. The cluster may not reach the specified balance if constrained by min/max bounds or other autoscaling settings. For example, if maxInstances for secondary workers is 0, then only primary workers will be added. The cluster can also be out of balance when created. If weight is not set on any instance group, the cluster will default to equal weight for all groups: the cluster will attempt to maintain an equal number of workers in each group within the configured size bounds for each group. If weight is set for one group only, the cluster will default to zero weight on the unset group. For example if weight is set only on primary workers, the cluster will use primary workers only and no secondary workers.

    AutoscalingPolicyWorkerConfig, AutoscalingPolicyWorkerConfigArgs

    MaxInstances int

    Maximum number of instances for this group.

    MinInstances int

    Minimum number of instances for this group. Bounds: [2, maxInstances]. Defaults to 2.

    Weight int

    Weight for the instance group, which is used to determine the fraction of total workers in the cluster from this instance group. For example, if primary workers have weight 2, and secondary workers have weight 1, the cluster will have approximately 2 primary workers for each secondary worker. The cluster may not reach the specified balance if constrained by min/max bounds or other autoscaling settings. For example, if maxInstances for secondary workers is 0, then only primary workers will be added. The cluster can also be out of balance when created. If weight is not set on any instance group, the cluster will default to equal weight for all groups: the cluster will attempt to maintain an equal number of workers in each group within the configured size bounds for each group. If weight is set for one group only, the cluster will default to zero weight on the unset group. For example if weight is set only on primary workers, the cluster will use primary workers only and no secondary workers.

    MaxInstances int

    Maximum number of instances for this group.

    MinInstances int

    Minimum number of instances for this group. Bounds: [2, maxInstances]. Defaults to 2.

    Weight int

    Weight for the instance group, which is used to determine the fraction of total workers in the cluster from this instance group. For example, if primary workers have weight 2, and secondary workers have weight 1, the cluster will have approximately 2 primary workers for each secondary worker. The cluster may not reach the specified balance if constrained by min/max bounds or other autoscaling settings. For example, if maxInstances for secondary workers is 0, then only primary workers will be added. The cluster can also be out of balance when created. If weight is not set on any instance group, the cluster will default to equal weight for all groups: the cluster will attempt to maintain an equal number of workers in each group within the configured size bounds for each group. If weight is set for one group only, the cluster will default to zero weight on the unset group. For example if weight is set only on primary workers, the cluster will use primary workers only and no secondary workers.

    maxInstances Integer

    Maximum number of instances for this group.

    minInstances Integer

    Minimum number of instances for this group. Bounds: [2, maxInstances]. Defaults to 2.

    weight Integer

    Weight for the instance group, which is used to determine the fraction of total workers in the cluster from this instance group. For example, if primary workers have weight 2, and secondary workers have weight 1, the cluster will have approximately 2 primary workers for each secondary worker. The cluster may not reach the specified balance if constrained by min/max bounds or other autoscaling settings. For example, if maxInstances for secondary workers is 0, then only primary workers will be added. The cluster can also be out of balance when created. If weight is not set on any instance group, the cluster will default to equal weight for all groups: the cluster will attempt to maintain an equal number of workers in each group within the configured size bounds for each group. If weight is set for one group only, the cluster will default to zero weight on the unset group. For example if weight is set only on primary workers, the cluster will use primary workers only and no secondary workers.

    maxInstances number

    Maximum number of instances for this group.

    minInstances number

    Minimum number of instances for this group. Bounds: [2, maxInstances]. Defaults to 2.

    weight number

    Weight for the instance group, which is used to determine the fraction of total workers in the cluster from this instance group. For example, if primary workers have weight 2, and secondary workers have weight 1, the cluster will have approximately 2 primary workers for each secondary worker. The cluster may not reach the specified balance if constrained by min/max bounds or other autoscaling settings. For example, if maxInstances for secondary workers is 0, then only primary workers will be added. The cluster can also be out of balance when created. If weight is not set on any instance group, the cluster will default to equal weight for all groups: the cluster will attempt to maintain an equal number of workers in each group within the configured size bounds for each group. If weight is set for one group only, the cluster will default to zero weight on the unset group. For example if weight is set only on primary workers, the cluster will use primary workers only and no secondary workers.

    max_instances int

    Maximum number of instances for this group.

    min_instances int

    Minimum number of instances for this group. Bounds: [2, maxInstances]. Defaults to 2.

    weight int

    Weight for the instance group, which is used to determine the fraction of total workers in the cluster from this instance group. For example, if primary workers have weight 2, and secondary workers have weight 1, the cluster will have approximately 2 primary workers for each secondary worker. The cluster may not reach the specified balance if constrained by min/max bounds or other autoscaling settings. For example, if maxInstances for secondary workers is 0, then only primary workers will be added. The cluster can also be out of balance when created. If weight is not set on any instance group, the cluster will default to equal weight for all groups: the cluster will attempt to maintain an equal number of workers in each group within the configured size bounds for each group. If weight is set for one group only, the cluster will default to zero weight on the unset group. For example if weight is set only on primary workers, the cluster will use primary workers only and no secondary workers.

    maxInstances Number

    Maximum number of instances for this group.

    minInstances Number

    Minimum number of instances for this group. Bounds: [2, maxInstances]. Defaults to 2.

    weight Number

    Weight for the instance group, which is used to determine the fraction of total workers in the cluster from this instance group. For example, if primary workers have weight 2, and secondary workers have weight 1, the cluster will have approximately 2 primary workers for each secondary worker. The cluster may not reach the specified balance if constrained by min/max bounds or other autoscaling settings. For example, if maxInstances for secondary workers is 0, then only primary workers will be added. The cluster can also be out of balance when created. If weight is not set on any instance group, the cluster will default to equal weight for all groups: the cluster will attempt to maintain an equal number of workers in each group within the configured size bounds for each group. If weight is set for one group only, the cluster will default to zero weight on the unset group. For example if weight is set only on primary workers, the cluster will use primary workers only and no secondary workers.

    Import

    AutoscalingPolicy can be imported using any of these accepted formats

     $ pulumi import gcp:dataproc/autoscalingPolicy:AutoscalingPolicy default projects/{{project}}/locations/{{location}}/autoscalingPolicies/{{policy_id}}
    
     $ pulumi import gcp:dataproc/autoscalingPolicy:AutoscalingPolicy default {{project}}/{{location}}/{{policy_id}}
    
     $ pulumi import gcp:dataproc/autoscalingPolicy:AutoscalingPolicy default {{location}}/{{policy_id}}
    

    Package Details

    Repository
    Google Cloud (GCP) Classic pulumi/pulumi-gcp
    License
    Apache-2.0
    Notes

    This Pulumi package is based on the google-beta Terraform Provider.

    gcp logo
    Google Cloud Classic v6.67.0 published on Wednesday, Sep 27, 2023 by Pulumi