Cluster
Provides an Elastic MapReduce Cluster, a web service that makes it easy to process large amounts of data efficiently. See Amazon Elastic MapReduce Documentation for more information.
To configure Instance Groups for task nodes, see the aws.emr.InstanceGroup resource.
Example Usage
using System.Collections.Generic;
using Pulumi;
using Aws = Pulumi.Aws;
return await Deployment.RunAsync(() =>
{
var cluster = new Aws.Emr.Cluster("cluster", new()
{
ReleaseLabel = "emr-4.6.0",
Applications = new[]
{
"Spark",
},
AdditionalInfo = @"{
""instanceAwsClientConfiguration"": {
""proxyPort"": 8099,
""proxyHost"": ""myproxy.example.com""
}
}
",
TerminationProtection = false,
KeepJobFlowAliveWhenNoSteps = true,
Ec2Attributes = new Aws.Emr.Inputs.ClusterEc2AttributesArgs
{
SubnetId = aws_subnet.Main.Id,
EmrManagedMasterSecurityGroup = aws_security_group.Sg.Id,
EmrManagedSlaveSecurityGroup = aws_security_group.Sg.Id,
InstanceProfile = aws_iam_instance_profile.Emr_profile.Arn,
},
MasterInstanceGroup = new Aws.Emr.Inputs.ClusterMasterInstanceGroupArgs
{
InstanceType = "m4.large",
},
CoreInstanceGroup = new Aws.Emr.Inputs.ClusterCoreInstanceGroupArgs
{
InstanceType = "c4.large",
InstanceCount = 1,
EbsConfigs = new[]
{
new Aws.Emr.Inputs.ClusterCoreInstanceGroupEbsConfigArgs
{
Size = 40,
Type = "gp2",
VolumesPerInstance = 1,
},
},
BidPrice = "0.30",
AutoscalingPolicy = @"{
""Constraints"": {
""MinCapacity"": 1,
""MaxCapacity"": 2
},
""Rules"": [
{
""Name"": ""ScaleOutMemoryPercentage"",
""Description"": ""Scale out if YARNMemoryAvailablePercentage is less than 15"",
""Action"": {
""SimpleScalingPolicyConfiguration"": {
""AdjustmentType"": ""CHANGE_IN_CAPACITY"",
""ScalingAdjustment"": 1,
""CoolDown"": 300
}
},
""Trigger"": {
""CloudWatchAlarmDefinition"": {
""ComparisonOperator"": ""LESS_THAN"",
""EvaluationPeriods"": 1,
""MetricName"": ""YARNMemoryAvailablePercentage"",
""Namespace"": ""AWS/ElasticMapReduce"",
""Period"": 300,
""Statistic"": ""AVERAGE"",
""Threshold"": 15.0,
""Unit"": ""PERCENT""
}
}
}
]
}
",
},
EbsRootVolumeSize = 100,
Tags =
{
{ "role", "rolename" },
{ "env", "env" },
},
BootstrapActions = new[]
{
new Aws.Emr.Inputs.ClusterBootstrapActionArgs
{
Path = "s3://elasticmapreduce/bootstrap-actions/run-if",
Name = "runif",
Args = new[]
{
"instance.isMaster=true",
"echo running on master node",
},
},
},
ConfigurationsJson = @" [
{
""Classification"": ""hadoop-env"",
""Configurations"": [
{
""Classification"": ""export"",
""Properties"": {
""JAVA_HOME"": ""/usr/lib/jvm/java-1.8.0""
}
}
],
""Properties"": {}
},
{
""Classification"": ""spark-env"",
""Configurations"": [
{
""Classification"": ""export"",
""Properties"": {
""JAVA_HOME"": ""/usr/lib/jvm/java-1.8.0""
}
}
],
""Properties"": {}
}
]
",
ServiceRole = aws_iam_role.Iam_emr_service_role.Arn,
});
});
package main
import (
"fmt"
"github.com/pulumi/pulumi-aws/sdk/v5/go/aws/emr"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
_, err := emr.NewCluster(ctx, "cluster", &emr.ClusterArgs{
ReleaseLabel: pulumi.String("emr-4.6.0"),
Applications: pulumi.StringArray{
pulumi.String("Spark"),
},
AdditionalInfo: pulumi.String(fmt.Sprintf(`{
"instanceAwsClientConfiguration": {
"proxyPort": 8099,
"proxyHost": "myproxy.example.com"
}
}
`)),
TerminationProtection: pulumi.Bool(false),
KeepJobFlowAliveWhenNoSteps: pulumi.Bool(true),
Ec2Attributes: &emr.ClusterEc2AttributesArgs{
SubnetId: pulumi.Any(aws_subnet.Main.Id),
EmrManagedMasterSecurityGroup: pulumi.Any(aws_security_group.Sg.Id),
EmrManagedSlaveSecurityGroup: pulumi.Any(aws_security_group.Sg.Id),
InstanceProfile: pulumi.Any(aws_iam_instance_profile.Emr_profile.Arn),
},
MasterInstanceGroup: &emr.ClusterMasterInstanceGroupArgs{
InstanceType: pulumi.String("m4.large"),
},
CoreInstanceGroup: &emr.ClusterCoreInstanceGroupArgs{
InstanceType: pulumi.String("c4.large"),
InstanceCount: pulumi.Int(1),
EbsConfigs: emr.ClusterCoreInstanceGroupEbsConfigArray{
&emr.ClusterCoreInstanceGroupEbsConfigArgs{
Size: pulumi.Int(40),
Type: pulumi.String("gp2"),
VolumesPerInstance: pulumi.Int(1),
},
},
BidPrice: pulumi.String("0.30"),
AutoscalingPolicy: pulumi.String(fmt.Sprintf(`{
"Constraints": {
"MinCapacity": 1,
"MaxCapacity": 2
},
"Rules": [
{
"Name": "ScaleOutMemoryPercentage",
"Description": "Scale out if YARNMemoryAvailablePercentage is less than 15",
"Action": {
"SimpleScalingPolicyConfiguration": {
"AdjustmentType": "CHANGE_IN_CAPACITY",
"ScalingAdjustment": 1,
"CoolDown": 300
}
},
"Trigger": {
"CloudWatchAlarmDefinition": {
"ComparisonOperator": "LESS_THAN",
"EvaluationPeriods": 1,
"MetricName": "YARNMemoryAvailablePercentage",
"Namespace": "AWS/ElasticMapReduce",
"Period": 300,
"Statistic": "AVERAGE",
"Threshold": 15.0,
"Unit": "PERCENT"
}
}
}
]
}
`)),
},
EbsRootVolumeSize: pulumi.Int(100),
Tags: pulumi.StringMap{
"role": pulumi.String("rolename"),
"env": pulumi.String("env"),
},
BootstrapActions: emr.ClusterBootstrapActionArray{
&emr.ClusterBootstrapActionArgs{
Path: pulumi.String("s3://elasticmapreduce/bootstrap-actions/run-if"),
Name: pulumi.String("runif"),
Args: pulumi.StringArray{
pulumi.String("instance.isMaster=true"),
pulumi.String("echo running on master node"),
},
},
},
ConfigurationsJson: pulumi.String(fmt.Sprintf(` [
{
"Classification": "hadoop-env",
"Configurations": [
{
"Classification": "export",
"Properties": {
"JAVA_HOME": "/usr/lib/jvm/java-1.8.0"
}
}
],
"Properties": {}
},
{
"Classification": "spark-env",
"Configurations": [
{
"Classification": "export",
"Properties": {
"JAVA_HOME": "/usr/lib/jvm/java-1.8.0"
}
}
],
"Properties": {}
}
]
`)),
ServiceRole: pulumi.Any(aws_iam_role.Iam_emr_service_role.Arn),
})
if err != nil {
return err
}
return nil
})
}
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.emr.Cluster;
import com.pulumi.aws.emr.ClusterArgs;
import com.pulumi.aws.emr.inputs.ClusterEc2AttributesArgs;
import com.pulumi.aws.emr.inputs.ClusterMasterInstanceGroupArgs;
import com.pulumi.aws.emr.inputs.ClusterCoreInstanceGroupArgs;
import com.pulumi.aws.emr.inputs.ClusterBootstrapActionArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var cluster = new Cluster("cluster", ClusterArgs.builder()
.releaseLabel("emr-4.6.0")
.applications("Spark")
.additionalInfo("""
{
"instanceAwsClientConfiguration": {
"proxyPort": 8099,
"proxyHost": "myproxy.example.com"
}
}
""")
.terminationProtection(false)
.keepJobFlowAliveWhenNoSteps(true)
.ec2Attributes(ClusterEc2AttributesArgs.builder()
.subnetId(aws_subnet.main().id())
.emrManagedMasterSecurityGroup(aws_security_group.sg().id())
.emrManagedSlaveSecurityGroup(aws_security_group.sg().id())
.instanceProfile(aws_iam_instance_profile.emr_profile().arn())
.build())
.masterInstanceGroup(ClusterMasterInstanceGroupArgs.builder()
.instanceType("m4.large")
.build())
.coreInstanceGroup(ClusterCoreInstanceGroupArgs.builder()
.instanceType("c4.large")
.instanceCount(1)
.ebsConfigs(ClusterCoreInstanceGroupEbsConfigArgs.builder()
.size("40")
.type("gp2")
.volumesPerInstance(1)
.build())
.bidPrice("0.30")
.autoscalingPolicy("""
{
"Constraints": {
"MinCapacity": 1,
"MaxCapacity": 2
},
"Rules": [
{
"Name": "ScaleOutMemoryPercentage",
"Description": "Scale out if YARNMemoryAvailablePercentage is less than 15",
"Action": {
"SimpleScalingPolicyConfiguration": {
"AdjustmentType": "CHANGE_IN_CAPACITY",
"ScalingAdjustment": 1,
"CoolDown": 300
}
},
"Trigger": {
"CloudWatchAlarmDefinition": {
"ComparisonOperator": "LESS_THAN",
"EvaluationPeriods": 1,
"MetricName": "YARNMemoryAvailablePercentage",
"Namespace": "AWS/ElasticMapReduce",
"Period": 300,
"Statistic": "AVERAGE",
"Threshold": 15.0,
"Unit": "PERCENT"
}
}
}
]
}
""")
.build())
.ebsRootVolumeSize(100)
.tags(Map.ofEntries(
Map.entry("role", "rolename"),
Map.entry("env", "env")
))
.bootstrapActions(ClusterBootstrapActionArgs.builder()
.path("s3://elasticmapreduce/bootstrap-actions/run-if")
.name("runif")
.args(
"instance.isMaster=true",
"echo running on master node")
.build())
.configurationsJson("""
[
{
"Classification": "hadoop-env",
"Configurations": [
{
"Classification": "export",
"Properties": {
"JAVA_HOME": "/usr/lib/jvm/java-1.8.0"
}
}
],
"Properties": {}
},
{
"Classification": "spark-env",
"Configurations": [
{
"Classification": "export",
"Properties": {
"JAVA_HOME": "/usr/lib/jvm/java-1.8.0"
}
}
],
"Properties": {}
}
]
""")
.serviceRole(aws_iam_role.iam_emr_service_role().arn())
.build());
}
}
import pulumi
import pulumi_aws as aws
cluster = aws.emr.Cluster("cluster",
release_label="emr-4.6.0",
applications=["Spark"],
additional_info="""{
"instanceAwsClientConfiguration": {
"proxyPort": 8099,
"proxyHost": "myproxy.example.com"
}
}
""",
termination_protection=False,
keep_job_flow_alive_when_no_steps=True,
ec2_attributes=aws.emr.ClusterEc2AttributesArgs(
subnet_id=aws_subnet["main"]["id"],
emr_managed_master_security_group=aws_security_group["sg"]["id"],
emr_managed_slave_security_group=aws_security_group["sg"]["id"],
instance_profile=aws_iam_instance_profile["emr_profile"]["arn"],
),
master_instance_group=aws.emr.ClusterMasterInstanceGroupArgs(
instance_type="m4.large",
),
core_instance_group=aws.emr.ClusterCoreInstanceGroupArgs(
instance_type="c4.large",
instance_count=1,
ebs_configs=[aws.emr.ClusterCoreInstanceGroupEbsConfigArgs(
size=40,
type="gp2",
volumes_per_instance=1,
)],
bid_price="0.30",
autoscaling_policy="""{
"Constraints": {
"MinCapacity": 1,
"MaxCapacity": 2
},
"Rules": [
{
"Name": "ScaleOutMemoryPercentage",
"Description": "Scale out if YARNMemoryAvailablePercentage is less than 15",
"Action": {
"SimpleScalingPolicyConfiguration": {
"AdjustmentType": "CHANGE_IN_CAPACITY",
"ScalingAdjustment": 1,
"CoolDown": 300
}
},
"Trigger": {
"CloudWatchAlarmDefinition": {
"ComparisonOperator": "LESS_THAN",
"EvaluationPeriods": 1,
"MetricName": "YARNMemoryAvailablePercentage",
"Namespace": "AWS/ElasticMapReduce",
"Period": 300,
"Statistic": "AVERAGE",
"Threshold": 15.0,
"Unit": "PERCENT"
}
}
}
]
}
""",
),
ebs_root_volume_size=100,
tags={
"role": "rolename",
"env": "env",
},
bootstrap_actions=[aws.emr.ClusterBootstrapActionArgs(
path="s3://elasticmapreduce/bootstrap-actions/run-if",
name="runif",
args=[
"instance.isMaster=true",
"echo running on master node",
],
)],
configurations_json=""" [
{
"Classification": "hadoop-env",
"Configurations": [
{
"Classification": "export",
"Properties": {
"JAVA_HOME": "/usr/lib/jvm/java-1.8.0"
}
}
],
"Properties": {}
},
{
"Classification": "spark-env",
"Configurations": [
{
"Classification": "export",
"Properties": {
"JAVA_HOME": "/usr/lib/jvm/java-1.8.0"
}
}
],
"Properties": {}
}
]
""",
service_role=aws_iam_role["iam_emr_service_role"]["arn"])
import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
const cluster = new aws.emr.Cluster("cluster", {
releaseLabel: "emr-4.6.0",
applications: ["Spark"],
additionalInfo: `{
"instanceAwsClientConfiguration": {
"proxyPort": 8099,
"proxyHost": "myproxy.example.com"
}
}
`,
terminationProtection: false,
keepJobFlowAliveWhenNoSteps: true,
ec2Attributes: {
subnetId: aws_subnet.main.id,
emrManagedMasterSecurityGroup: aws_security_group.sg.id,
emrManagedSlaveSecurityGroup: aws_security_group.sg.id,
instanceProfile: aws_iam_instance_profile.emr_profile.arn,
},
masterInstanceGroup: {
instanceType: "m4.large",
},
coreInstanceGroup: {
instanceType: "c4.large",
instanceCount: 1,
ebsConfigs: [{
size: 40,
type: "gp2",
volumesPerInstance: 1,
}],
bidPrice: "0.30",
autoscalingPolicy: `{
"Constraints": {
"MinCapacity": 1,
"MaxCapacity": 2
},
"Rules": [
{
"Name": "ScaleOutMemoryPercentage",
"Description": "Scale out if YARNMemoryAvailablePercentage is less than 15",
"Action": {
"SimpleScalingPolicyConfiguration": {
"AdjustmentType": "CHANGE_IN_CAPACITY",
"ScalingAdjustment": 1,
"CoolDown": 300
}
},
"Trigger": {
"CloudWatchAlarmDefinition": {
"ComparisonOperator": "LESS_THAN",
"EvaluationPeriods": 1,
"MetricName": "YARNMemoryAvailablePercentage",
"Namespace": "AWS/ElasticMapReduce",
"Period": 300,
"Statistic": "AVERAGE",
"Threshold": 15.0,
"Unit": "PERCENT"
}
}
}
]
}
`,
},
ebsRootVolumeSize: 100,
tags: {
role: "rolename",
env: "env",
},
bootstrapActions: [{
path: "s3://elasticmapreduce/bootstrap-actions/run-if",
name: "runif",
args: [
"instance.isMaster=true",
"echo running on master node",
],
}],
configurationsJson: ` [
{
"Classification": "hadoop-env",
"Configurations": [
{
"Classification": "export",
"Properties": {
"JAVA_HOME": "/usr/lib/jvm/java-1.8.0"
}
}
],
"Properties": {}
},
{
"Classification": "spark-env",
"Configurations": [
{
"Classification": "export",
"Properties": {
"JAVA_HOME": "/usr/lib/jvm/java-1.8.0"
}
}
],
"Properties": {}
}
]
`,
serviceRole: aws_iam_role.iam_emr_service_role.arn,
});
resources:
cluster:
type: aws:emr:Cluster
properties:
releaseLabel: emr-4.6.0
applications:
- Spark
additionalInfo: |
{
"instanceAwsClientConfiguration": {
"proxyPort": 8099,
"proxyHost": "myproxy.example.com"
}
}
terminationProtection: false
keepJobFlowAliveWhenNoSteps: true
ec2Attributes:
subnetId: ${aws_subnet.main.id}
emrManagedMasterSecurityGroup: ${aws_security_group.sg.id}
emrManagedSlaveSecurityGroup: ${aws_security_group.sg.id}
instanceProfile: ${aws_iam_instance_profile.emr_profile.arn}
masterInstanceGroup:
instanceType: m4.large
coreInstanceGroup:
instanceType: c4.large
instanceCount: 1
ebsConfigs:
- size: 40
type: gp2
volumesPerInstance: 1
bidPrice: 0.30
autoscalingPolicy: |
{
"Constraints": {
"MinCapacity": 1,
"MaxCapacity": 2
},
"Rules": [
{
"Name": "ScaleOutMemoryPercentage",
"Description": "Scale out if YARNMemoryAvailablePercentage is less than 15",
"Action": {
"SimpleScalingPolicyConfiguration": {
"AdjustmentType": "CHANGE_IN_CAPACITY",
"ScalingAdjustment": 1,
"CoolDown": 300
}
},
"Trigger": {
"CloudWatchAlarmDefinition": {
"ComparisonOperator": "LESS_THAN",
"EvaluationPeriods": 1,
"MetricName": "YARNMemoryAvailablePercentage",
"Namespace": "AWS/ElasticMapReduce",
"Period": 300,
"Statistic": "AVERAGE",
"Threshold": 15.0,
"Unit": "PERCENT"
}
}
}
]
}
ebsRootVolumeSize: 100
tags:
role: rolename
env: env
bootstrapActions:
- path: s3://elasticmapreduce/bootstrap-actions/run-if
name: runif
args:
- instance.isMaster=true
- echo running on master node
configurationsJson: |2
[
{
"Classification": "hadoop-env",
"Configurations": [
{
"Classification": "export",
"Properties": {
"JAVA_HOME": "/usr/lib/jvm/java-1.8.0"
}
}
],
"Properties": {}
},
{
"Classification": "spark-env",
"Configurations": [
{
"Classification": "export",
"Properties": {
"JAVA_HOME": "/usr/lib/jvm/java-1.8.0"
}
}
],
"Properties": {}
}
]
serviceRole: ${aws_iam_role.iam_emr_service_role.arn}
Instance Fleet
using System.Collections.Generic;
using Pulumi;
using Aws = Pulumi.Aws;
return await Deployment.RunAsync(() =>
{
var example = new Aws.Emr.Cluster("example", new()
{
MasterInstanceFleet = new Aws.Emr.Inputs.ClusterMasterInstanceFleetArgs
{
InstanceTypeConfigs = new[]
{
new Aws.Emr.Inputs.ClusterMasterInstanceFleetInstanceTypeConfigArgs
{
InstanceType = "m4.xlarge",
},
},
TargetOnDemandCapacity = 1,
},
CoreInstanceFleet = new Aws.Emr.Inputs.ClusterCoreInstanceFleetArgs
{
InstanceTypeConfigs = new[]
{
new Aws.Emr.Inputs.ClusterCoreInstanceFleetInstanceTypeConfigArgs
{
BidPriceAsPercentageOfOnDemandPrice = 80,
EbsConfigs = new[]
{
new Aws.Emr.Inputs.ClusterCoreInstanceFleetInstanceTypeConfigEbsConfigArgs
{
Size = 100,
Type = "gp2",
VolumesPerInstance = 1,
},
},
InstanceType = "m3.xlarge",
WeightedCapacity = 1,
},
new Aws.Emr.Inputs.ClusterCoreInstanceFleetInstanceTypeConfigArgs
{
BidPriceAsPercentageOfOnDemandPrice = 100,
EbsConfigs = new[]
{
new Aws.Emr.Inputs.ClusterCoreInstanceFleetInstanceTypeConfigEbsConfigArgs
{
Size = 100,
Type = "gp2",
VolumesPerInstance = 1,
},
},
InstanceType = "m4.xlarge",
WeightedCapacity = 1,
},
new Aws.Emr.Inputs.ClusterCoreInstanceFleetInstanceTypeConfigArgs
{
BidPriceAsPercentageOfOnDemandPrice = 100,
EbsConfigs = new[]
{
new Aws.Emr.Inputs.ClusterCoreInstanceFleetInstanceTypeConfigEbsConfigArgs
{
Size = 100,
Type = "gp2",
VolumesPerInstance = 1,
},
},
InstanceType = "m4.2xlarge",
WeightedCapacity = 2,
},
},
LaunchSpecifications = new Aws.Emr.Inputs.ClusterCoreInstanceFleetLaunchSpecificationsArgs
{
SpotSpecifications = new[]
{
new Aws.Emr.Inputs.ClusterCoreInstanceFleetLaunchSpecificationsSpotSpecificationArgs
{
AllocationStrategy = "capacity-optimized",
BlockDurationMinutes = 0,
TimeoutAction = "SWITCH_TO_ON_DEMAND",
TimeoutDurationMinutes = 10,
},
},
},
Name = "core fleet",
TargetOnDemandCapacity = 2,
TargetSpotCapacity = 2,
},
});
var task = new Aws.Emr.InstanceFleet("task", new()
{
ClusterId = example.Id,
InstanceTypeConfigs = new[]
{
new Aws.Emr.Inputs.InstanceFleetInstanceTypeConfigArgs
{
BidPriceAsPercentageOfOnDemandPrice = 100,
EbsConfigs = new[]
{
new Aws.Emr.Inputs.InstanceFleetInstanceTypeConfigEbsConfigArgs
{
Size = 100,
Type = "gp2",
VolumesPerInstance = 1,
},
},
InstanceType = "m4.xlarge",
WeightedCapacity = 1,
},
new Aws.Emr.Inputs.InstanceFleetInstanceTypeConfigArgs
{
BidPriceAsPercentageOfOnDemandPrice = 100,
EbsConfigs = new[]
{
new Aws.Emr.Inputs.InstanceFleetInstanceTypeConfigEbsConfigArgs
{
Size = 100,
Type = "gp2",
VolumesPerInstance = 1,
},
},
InstanceType = "m4.2xlarge",
WeightedCapacity = 2,
},
},
LaunchSpecifications = new Aws.Emr.Inputs.InstanceFleetLaunchSpecificationsArgs
{
SpotSpecifications = new[]
{
new Aws.Emr.Inputs.InstanceFleetLaunchSpecificationsSpotSpecificationArgs
{
AllocationStrategy = "capacity-optimized",
BlockDurationMinutes = 0,
TimeoutAction = "TERMINATE_CLUSTER",
TimeoutDurationMinutes = 10,
},
},
},
TargetOnDemandCapacity = 1,
TargetSpotCapacity = 1,
});
});
package main
import (
"github.com/pulumi/pulumi-aws/sdk/v5/go/aws/emr"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
example, err := emr.NewCluster(ctx, "example", &emr.ClusterArgs{
MasterInstanceFleet: &emr.ClusterMasterInstanceFleetArgs{
InstanceTypeConfigs: emr.ClusterMasterInstanceFleetInstanceTypeConfigArray{
&emr.ClusterMasterInstanceFleetInstanceTypeConfigArgs{
InstanceType: pulumi.String("m4.xlarge"),
},
},
TargetOnDemandCapacity: pulumi.Int(1),
},
CoreInstanceFleet: &emr.ClusterCoreInstanceFleetArgs{
InstanceTypeConfigs: emr.ClusterCoreInstanceFleetInstanceTypeConfigArray{
&emr.ClusterCoreInstanceFleetInstanceTypeConfigArgs{
BidPriceAsPercentageOfOnDemandPrice: pulumi.Float64(80),
EbsConfigs: emr.ClusterCoreInstanceFleetInstanceTypeConfigEbsConfigArray{
&emr.ClusterCoreInstanceFleetInstanceTypeConfigEbsConfigArgs{
Size: pulumi.Int(100),
Type: pulumi.String("gp2"),
VolumesPerInstance: pulumi.Int(1),
},
},
InstanceType: pulumi.String("m3.xlarge"),
WeightedCapacity: pulumi.Int(1),
},
&emr.ClusterCoreInstanceFleetInstanceTypeConfigArgs{
BidPriceAsPercentageOfOnDemandPrice: pulumi.Float64(100),
EbsConfigs: emr.ClusterCoreInstanceFleetInstanceTypeConfigEbsConfigArray{
&emr.ClusterCoreInstanceFleetInstanceTypeConfigEbsConfigArgs{
Size: pulumi.Int(100),
Type: pulumi.String("gp2"),
VolumesPerInstance: pulumi.Int(1),
},
},
InstanceType: pulumi.String("m4.xlarge"),
WeightedCapacity: pulumi.Int(1),
},
&emr.ClusterCoreInstanceFleetInstanceTypeConfigArgs{
BidPriceAsPercentageOfOnDemandPrice: pulumi.Float64(100),
EbsConfigs: emr.ClusterCoreInstanceFleetInstanceTypeConfigEbsConfigArray{
&emr.ClusterCoreInstanceFleetInstanceTypeConfigEbsConfigArgs{
Size: pulumi.Int(100),
Type: pulumi.String("gp2"),
VolumesPerInstance: pulumi.Int(1),
},
},
InstanceType: pulumi.String("m4.2xlarge"),
WeightedCapacity: pulumi.Int(2),
},
},
LaunchSpecifications: &emr.ClusterCoreInstanceFleetLaunchSpecificationsArgs{
SpotSpecifications: emr.ClusterCoreInstanceFleetLaunchSpecificationsSpotSpecificationArray{
&emr.ClusterCoreInstanceFleetLaunchSpecificationsSpotSpecificationArgs{
AllocationStrategy: pulumi.String("capacity-optimized"),
BlockDurationMinutes: pulumi.Int(0),
TimeoutAction: pulumi.String("SWITCH_TO_ON_DEMAND"),
TimeoutDurationMinutes: pulumi.Int(10),
},
},
},
Name: pulumi.String("core fleet"),
TargetOnDemandCapacity: pulumi.Int(2),
TargetSpotCapacity: pulumi.Int(2),
},
})
if err != nil {
return err
}
_, err = emr.NewInstanceFleet(ctx, "task", &emr.InstanceFleetArgs{
ClusterId: example.ID(),
InstanceTypeConfigs: emr.InstanceFleetInstanceTypeConfigArray{
&emr.InstanceFleetInstanceTypeConfigArgs{
BidPriceAsPercentageOfOnDemandPrice: pulumi.Float64(100),
EbsConfigs: emr.InstanceFleetInstanceTypeConfigEbsConfigArray{
&emr.InstanceFleetInstanceTypeConfigEbsConfigArgs{
Size: pulumi.Int(100),
Type: pulumi.String("gp2"),
VolumesPerInstance: pulumi.Int(1),
},
},
InstanceType: pulumi.String("m4.xlarge"),
WeightedCapacity: pulumi.Int(1),
},
&emr.InstanceFleetInstanceTypeConfigArgs{
BidPriceAsPercentageOfOnDemandPrice: pulumi.Float64(100),
EbsConfigs: emr.InstanceFleetInstanceTypeConfigEbsConfigArray{
&emr.InstanceFleetInstanceTypeConfigEbsConfigArgs{
Size: pulumi.Int(100),
Type: pulumi.String("gp2"),
VolumesPerInstance: pulumi.Int(1),
},
},
InstanceType: pulumi.String("m4.2xlarge"),
WeightedCapacity: pulumi.Int(2),
},
},
LaunchSpecifications: &emr.InstanceFleetLaunchSpecificationsArgs{
SpotSpecifications: emr.InstanceFleetLaunchSpecificationsSpotSpecificationArray{
&emr.InstanceFleetLaunchSpecificationsSpotSpecificationArgs{
AllocationStrategy: pulumi.String("capacity-optimized"),
BlockDurationMinutes: pulumi.Int(0),
TimeoutAction: pulumi.String("TERMINATE_CLUSTER"),
TimeoutDurationMinutes: pulumi.Int(10),
},
},
},
TargetOnDemandCapacity: pulumi.Int(1),
TargetSpotCapacity: pulumi.Int(1),
})
if err != nil {
return err
}
return nil
})
}
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.emr.Cluster;
import com.pulumi.aws.emr.ClusterArgs;
import com.pulumi.aws.emr.inputs.ClusterMasterInstanceFleetArgs;
import com.pulumi.aws.emr.inputs.ClusterCoreInstanceFleetArgs;
import com.pulumi.aws.emr.inputs.ClusterCoreInstanceFleetLaunchSpecificationsArgs;
import com.pulumi.aws.emr.InstanceFleet;
import com.pulumi.aws.emr.InstanceFleetArgs;
import com.pulumi.aws.emr.inputs.InstanceFleetInstanceTypeConfigArgs;
import com.pulumi.aws.emr.inputs.InstanceFleetLaunchSpecificationsArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var example = new Cluster("example", ClusterArgs.builder()
.masterInstanceFleet(ClusterMasterInstanceFleetArgs.builder()
.instanceTypeConfigs(ClusterMasterInstanceFleetInstanceTypeConfigArgs.builder()
.instanceType("m4.xlarge")
.build())
.targetOnDemandCapacity(1)
.build())
.coreInstanceFleet(ClusterCoreInstanceFleetArgs.builder()
.instanceTypeConfigs(
ClusterCoreInstanceFleetInstanceTypeConfigArgs.builder()
.bidPriceAsPercentageOfOnDemandPrice(80)
.ebsConfigs(ClusterCoreInstanceFleetInstanceTypeConfigEbsConfigArgs.builder()
.size(100)
.type("gp2")
.volumesPerInstance(1)
.build())
.instanceType("m3.xlarge")
.weightedCapacity(1)
.build(),
ClusterCoreInstanceFleetInstanceTypeConfigArgs.builder()
.bidPriceAsPercentageOfOnDemandPrice(100)
.ebsConfigs(ClusterCoreInstanceFleetInstanceTypeConfigEbsConfigArgs.builder()
.size(100)
.type("gp2")
.volumesPerInstance(1)
.build())
.instanceType("m4.xlarge")
.weightedCapacity(1)
.build(),
ClusterCoreInstanceFleetInstanceTypeConfigArgs.builder()
.bidPriceAsPercentageOfOnDemandPrice(100)
.ebsConfigs(ClusterCoreInstanceFleetInstanceTypeConfigEbsConfigArgs.builder()
.size(100)
.type("gp2")
.volumesPerInstance(1)
.build())
.instanceType("m4.2xlarge")
.weightedCapacity(2)
.build())
.launchSpecifications(ClusterCoreInstanceFleetLaunchSpecificationsArgs.builder()
.spotSpecifications(ClusterCoreInstanceFleetLaunchSpecificationsSpotSpecificationArgs.builder()
.allocationStrategy("capacity-optimized")
.blockDurationMinutes(0)
.timeoutAction("SWITCH_TO_ON_DEMAND")
.timeoutDurationMinutes(10)
.build())
.build())
.name("core fleet")
.targetOnDemandCapacity(2)
.targetSpotCapacity(2)
.build())
.build());
var task = new InstanceFleet("task", InstanceFleetArgs.builder()
.clusterId(example.id())
.instanceTypeConfigs(
InstanceFleetInstanceTypeConfigArgs.builder()
.bidPriceAsPercentageOfOnDemandPrice(100)
.ebsConfigs(InstanceFleetInstanceTypeConfigEbsConfigArgs.builder()
.size(100)
.type("gp2")
.volumesPerInstance(1)
.build())
.instanceType("m4.xlarge")
.weightedCapacity(1)
.build(),
InstanceFleetInstanceTypeConfigArgs.builder()
.bidPriceAsPercentageOfOnDemandPrice(100)
.ebsConfigs(InstanceFleetInstanceTypeConfigEbsConfigArgs.builder()
.size(100)
.type("gp2")
.volumesPerInstance(1)
.build())
.instanceType("m4.2xlarge")
.weightedCapacity(2)
.build())
.launchSpecifications(InstanceFleetLaunchSpecificationsArgs.builder()
.spotSpecifications(InstanceFleetLaunchSpecificationsSpotSpecificationArgs.builder()
.allocationStrategy("capacity-optimized")
.blockDurationMinutes(0)
.timeoutAction("TERMINATE_CLUSTER")
.timeoutDurationMinutes(10)
.build())
.build())
.targetOnDemandCapacity(1)
.targetSpotCapacity(1)
.build());
}
}
import pulumi
import pulumi_aws as aws
example = aws.emr.Cluster("example",
master_instance_fleet=aws.emr.ClusterMasterInstanceFleetArgs(
instance_type_configs=[aws.emr.ClusterMasterInstanceFleetInstanceTypeConfigArgs(
instance_type="m4.xlarge",
)],
target_on_demand_capacity=1,
),
core_instance_fleet=aws.emr.ClusterCoreInstanceFleetArgs(
instance_type_configs=[
aws.emr.ClusterCoreInstanceFleetInstanceTypeConfigArgs(
bid_price_as_percentage_of_on_demand_price=80,
ebs_configs=[aws.emr.ClusterCoreInstanceFleetInstanceTypeConfigEbsConfigArgs(
size=100,
type="gp2",
volumes_per_instance=1,
)],
instance_type="m3.xlarge",
weighted_capacity=1,
),
aws.emr.ClusterCoreInstanceFleetInstanceTypeConfigArgs(
bid_price_as_percentage_of_on_demand_price=100,
ebs_configs=[aws.emr.ClusterCoreInstanceFleetInstanceTypeConfigEbsConfigArgs(
size=100,
type="gp2",
volumes_per_instance=1,
)],
instance_type="m4.xlarge",
weighted_capacity=1,
),
aws.emr.ClusterCoreInstanceFleetInstanceTypeConfigArgs(
bid_price_as_percentage_of_on_demand_price=100,
ebs_configs=[aws.emr.ClusterCoreInstanceFleetInstanceTypeConfigEbsConfigArgs(
size=100,
type="gp2",
volumes_per_instance=1,
)],
instance_type="m4.2xlarge",
weighted_capacity=2,
),
],
launch_specifications=aws.emr.ClusterCoreInstanceFleetLaunchSpecificationsArgs(
spot_specifications=[aws.emr.ClusterCoreInstanceFleetLaunchSpecificationsSpotSpecificationArgs(
allocation_strategy="capacity-optimized",
block_duration_minutes=0,
timeout_action="SWITCH_TO_ON_DEMAND",
timeout_duration_minutes=10,
)],
),
name="core fleet",
target_on_demand_capacity=2,
target_spot_capacity=2,
))
task = aws.emr.InstanceFleet("task",
cluster_id=example.id,
instance_type_configs=[
aws.emr.InstanceFleetInstanceTypeConfigArgs(
bid_price_as_percentage_of_on_demand_price=100,
ebs_configs=[aws.emr.InstanceFleetInstanceTypeConfigEbsConfigArgs(
size=100,
type="gp2",
volumes_per_instance=1,
)],
instance_type="m4.xlarge",
weighted_capacity=1,
),
aws.emr.InstanceFleetInstanceTypeConfigArgs(
bid_price_as_percentage_of_on_demand_price=100,
ebs_configs=[aws.emr.InstanceFleetInstanceTypeConfigEbsConfigArgs(
size=100,
type="gp2",
volumes_per_instance=1,
)],
instance_type="m4.2xlarge",
weighted_capacity=2,
),
],
launch_specifications=aws.emr.InstanceFleetLaunchSpecificationsArgs(
spot_specifications=[aws.emr.InstanceFleetLaunchSpecificationsSpotSpecificationArgs(
allocation_strategy="capacity-optimized",
block_duration_minutes=0,
timeout_action="TERMINATE_CLUSTER",
timeout_duration_minutes=10,
)],
),
target_on_demand_capacity=1,
target_spot_capacity=1)
import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
const example = new aws.emr.Cluster("example", {
masterInstanceFleet: {
instanceTypeConfigs: [{
instanceType: "m4.xlarge",
}],
targetOnDemandCapacity: 1,
},
coreInstanceFleet: {
instanceTypeConfigs: [
{
bidPriceAsPercentageOfOnDemandPrice: 80,
ebsConfigs: [{
size: 100,
type: "gp2",
volumesPerInstance: 1,
}],
instanceType: "m3.xlarge",
weightedCapacity: 1,
},
{
bidPriceAsPercentageOfOnDemandPrice: 100,
ebsConfigs: [{
size: 100,
type: "gp2",
volumesPerInstance: 1,
}],
instanceType: "m4.xlarge",
weightedCapacity: 1,
},
{
bidPriceAsPercentageOfOnDemandPrice: 100,
ebsConfigs: [{
size: 100,
type: "gp2",
volumesPerInstance: 1,
}],
instanceType: "m4.2xlarge",
weightedCapacity: 2,
},
],
launchSpecifications: {
spotSpecifications: [{
allocationStrategy: "capacity-optimized",
blockDurationMinutes: 0,
timeoutAction: "SWITCH_TO_ON_DEMAND",
timeoutDurationMinutes: 10,
}],
},
name: "core fleet",
targetOnDemandCapacity: 2,
targetSpotCapacity: 2,
},
});
const task = new aws.emr.InstanceFleet("task", {
clusterId: example.id,
instanceTypeConfigs: [
{
bidPriceAsPercentageOfOnDemandPrice: 100,
ebsConfigs: [{
size: 100,
type: "gp2",
volumesPerInstance: 1,
}],
instanceType: "m4.xlarge",
weightedCapacity: 1,
},
{
bidPriceAsPercentageOfOnDemandPrice: 100,
ebsConfigs: [{
size: 100,
type: "gp2",
volumesPerInstance: 1,
}],
instanceType: "m4.2xlarge",
weightedCapacity: 2,
},
],
launchSpecifications: {
spotSpecifications: [{
allocationStrategy: "capacity-optimized",
blockDurationMinutes: 0,
timeoutAction: "TERMINATE_CLUSTER",
timeoutDurationMinutes: 10,
}],
},
targetOnDemandCapacity: 1,
targetSpotCapacity: 1,
});
resources:
example:
type: aws:emr:Cluster
properties:
masterInstanceFleet:
instanceTypeConfigs:
- instanceType: m4.xlarge
targetOnDemandCapacity: 1
coreInstanceFleet:
instanceTypeConfigs:
- bidPriceAsPercentageOfOnDemandPrice: 80
ebsConfigs:
- size: 100
type: gp2
volumesPerInstance: 1
instanceType: m3.xlarge
weightedCapacity: 1
- bidPriceAsPercentageOfOnDemandPrice: 100
ebsConfigs:
- size: 100
type: gp2
volumesPerInstance: 1
instanceType: m4.xlarge
weightedCapacity: 1
- bidPriceAsPercentageOfOnDemandPrice: 100
ebsConfigs:
- size: 100
type: gp2
volumesPerInstance: 1
instanceType: m4.2xlarge
weightedCapacity: 2
launchSpecifications:
spotSpecifications:
- allocationStrategy: capacity-optimized
blockDurationMinutes: 0
timeoutAction: SWITCH_TO_ON_DEMAND
timeoutDurationMinutes: 10
name: core fleet
targetOnDemandCapacity: 2
targetSpotCapacity: 2
task:
type: aws:emr:InstanceFleet
properties:
clusterId: ${example.id}
instanceTypeConfigs:
- bidPriceAsPercentageOfOnDemandPrice: 100
ebsConfigs:
- size: 100
type: gp2
volumesPerInstance: 1
instanceType: m4.xlarge
weightedCapacity: 1
- bidPriceAsPercentageOfOnDemandPrice: 100
ebsConfigs:
- size: 100
type: gp2
volumesPerInstance: 1
instanceType: m4.2xlarge
weightedCapacity: 2
launchSpecifications:
spotSpecifications:
- allocationStrategy: capacity-optimized
blockDurationMinutes: 0
timeoutAction: TERMINATE_CLUSTER
timeoutDurationMinutes: 10
targetOnDemandCapacity: 1
targetSpotCapacity: 1
Enable Debug Logging
using System.Collections.Generic;
using Pulumi;
using Aws = Pulumi.Aws;
return await Deployment.RunAsync(() =>
{
// ... other configuration ...
var example = new Aws.Emr.Cluster("example", new()
{
Steps = new[]
{
new Aws.Emr.Inputs.ClusterStepArgs
{
ActionOnFailure = "TERMINATE_CLUSTER",
Name = "Setup Hadoop Debugging",
HadoopJarStep = new Aws.Emr.Inputs.ClusterStepHadoopJarStepArgs
{
Jar = "command-runner.jar",
Args = new[]
{
"state-pusher-script",
},
},
},
},
});
});
package main
import (
"github.com/pulumi/pulumi-aws/sdk/v5/go/aws/emr"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
_, err := emr.NewCluster(ctx, "example", &emr.ClusterArgs{
Steps: emr.ClusterStepArray{
&emr.ClusterStepArgs{
ActionOnFailure: pulumi.String("TERMINATE_CLUSTER"),
Name: pulumi.String("Setup Hadoop Debugging"),
HadoopJarStep: &emr.ClusterStepHadoopJarStepArgs{
Jar: pulumi.String("command-runner.jar"),
Args: pulumi.StringArray{
pulumi.String("state-pusher-script"),
},
},
},
},
})
if err != nil {
return err
}
return nil
})
}
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.emr.Cluster;
import com.pulumi.aws.emr.ClusterArgs;
import com.pulumi.aws.emr.inputs.ClusterStepArgs;
import com.pulumi.aws.emr.inputs.ClusterStepHadoopJarStepArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var example = new Cluster("example", ClusterArgs.builder()
.steps(ClusterStepArgs.builder()
.actionOnFailure("TERMINATE_CLUSTER")
.name("Setup Hadoop Debugging")
.hadoopJarStep(ClusterStepHadoopJarStepArgs.builder()
.jar("command-runner.jar")
.args("state-pusher-script")
.build())
.build())
.build());
}
}
import pulumi
import pulumi_aws as aws
# ... other configuration ...
example = aws.emr.Cluster("example", steps=[aws.emr.ClusterStepArgs(
action_on_failure="TERMINATE_CLUSTER",
name="Setup Hadoop Debugging",
hadoop_jar_step=aws.emr.ClusterStepHadoopJarStepArgs(
jar="command-runner.jar",
args=["state-pusher-script"],
),
)])
import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
// ... other configuration ...
const example = new aws.emr.Cluster("example", {steps: [{
actionOnFailure: "TERMINATE_CLUSTER",
name: "Setup Hadoop Debugging",
hadoopJarStep: {
jar: "command-runner.jar",
args: ["state-pusher-script"],
},
}]});
resources:
example:
type: aws:emr:Cluster
properties:
steps:
- actionOnFailure: TERMINATE_CLUSTER
name: Setup Hadoop Debugging
hadoopJarStep:
jar: command-runner.jar
args:
- state-pusher-script
Multiple Node Master Instance Group
using System.Collections.Generic;
using Pulumi;
using Aws = Pulumi.Aws;
return await Deployment.RunAsync(() =>
{
// This configuration is for illustrative purposes and highlights
// only relevant configurations for working with this functionality.
// Map public IP on launch must be enabled for public (Internet accessible) subnets
// ... other configuration ...
var exampleSubnet = new Aws.Ec2.Subnet("exampleSubnet", new()
{
MapPublicIpOnLaunch = true,
});
// ... other configuration ...
var exampleCluster = new Aws.Emr.Cluster("exampleCluster", new()
{
ReleaseLabel = "emr-5.24.1",
TerminationProtection = true,
Ec2Attributes = new Aws.Emr.Inputs.ClusterEc2AttributesArgs
{
SubnetId = exampleSubnet.Id,
},
MasterInstanceGroup = new Aws.Emr.Inputs.ClusterMasterInstanceGroupArgs
{
InstanceCount = 3,
},
CoreInstanceGroup = ,
});
});
package main
import (
"github.com/pulumi/pulumi-aws/sdk/v5/go/aws/ec2"
"github.com/pulumi/pulumi-aws/sdk/v5/go/aws/emr"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
exampleSubnet, err := ec2.NewSubnet(ctx, "exampleSubnet", &ec2.SubnetArgs{
MapPublicIpOnLaunch: pulumi.Bool(true),
})
if err != nil {
return err
}
_, err = emr.NewCluster(ctx, "exampleCluster", &emr.ClusterArgs{
ReleaseLabel: pulumi.String("emr-5.24.1"),
TerminationProtection: pulumi.Bool(true),
Ec2Attributes: &emr.ClusterEc2AttributesArgs{
SubnetId: exampleSubnet.ID(),
},
MasterInstanceGroup: &emr.ClusterMasterInstanceGroupArgs{
InstanceCount: pulumi.Int(3),
},
CoreInstanceGroup: nil,
})
if err != nil {
return err
}
return nil
})
}
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.ec2.Subnet;
import com.pulumi.aws.ec2.SubnetArgs;
import com.pulumi.aws.emr.Cluster;
import com.pulumi.aws.emr.ClusterArgs;
import com.pulumi.aws.emr.inputs.ClusterEc2AttributesArgs;
import com.pulumi.aws.emr.inputs.ClusterMasterInstanceGroupArgs;
import com.pulumi.aws.emr.inputs.ClusterCoreInstanceGroupArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var exampleSubnet = new Subnet("exampleSubnet", SubnetArgs.builder()
.mapPublicIpOnLaunch(true)
.build());
var exampleCluster = new Cluster("exampleCluster", ClusterArgs.builder()
.releaseLabel("emr-5.24.1")
.terminationProtection(true)
.ec2Attributes(ClusterEc2AttributesArgs.builder()
.subnetId(exampleSubnet.id())
.build())
.masterInstanceGroup(ClusterMasterInstanceGroupArgs.builder()
.instanceCount(3)
.build())
.coreInstanceGroup()
.build());
}
}
import pulumi
import pulumi_aws as aws
# This configuration is for illustrative purposes and highlights
# only relevant configurations for working with this functionality.
# Map public IP on launch must be enabled for public (Internet accessible) subnets
# ... other configuration ...
example_subnet = aws.ec2.Subnet("exampleSubnet", map_public_ip_on_launch=True)
# ... other configuration ...
example_cluster = aws.emr.Cluster("exampleCluster",
release_label="emr-5.24.1",
termination_protection=True,
ec2_attributes=aws.emr.ClusterEc2AttributesArgs(
subnet_id=example_subnet.id,
),
master_instance_group=aws.emr.ClusterMasterInstanceGroupArgs(
instance_count=3,
),
core_instance_group=aws.emr.ClusterCoreInstanceGroupArgs())
import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
// This configuration is for illustrative purposes and highlights
// only relevant configurations for working with this functionality.
// Map public IP on launch must be enabled for public (Internet accessible) subnets
// ... other configuration ...
const exampleSubnet = new aws.ec2.Subnet("exampleSubnet", {mapPublicIpOnLaunch: true});
// ... other configuration ...
const exampleCluster = new aws.emr.Cluster("exampleCluster", {
releaseLabel: "emr-5.24.1",
terminationProtection: true,
ec2Attributes: {
subnetId: exampleSubnet.id,
},
masterInstanceGroup: {
instanceCount: 3,
},
coreInstanceGroup: {},
});
resources:
# This configuration is for illustrative purposes and highlights
# only relevant configurations for working with this functionality.
# Map public IP on launch must be enabled for public (Internet accessible) subnets
exampleSubnet:
type: aws:ec2:Subnet
properties:
mapPublicIpOnLaunch: true
exampleCluster:
type: aws:emr:Cluster
properties:
# EMR version must be 5.23.0 or later
releaseLabel: emr-5.24.1
# Termination protection is automatically enabled for multiple masters
# # To destroy the cluster, this must be configured to false and applied first
terminationProtection: true
ec2Attributes:
subnetId: ${exampleSubnet.id}
masterInstanceGroup:
instanceCount: 3
coreInstanceGroup: {}
Bootable Cluster
using System.Collections.Generic;
using Pulumi;
using Aws = Pulumi.Aws;
return await Deployment.RunAsync(() =>
{
var mainVpc = new Aws.Ec2.Vpc("mainVpc", new()
{
CidrBlock = "168.31.0.0/16",
EnableDnsHostnames = true,
Tags =
{
{ "name", "emr_test" },
},
});
var mainSubnet = new Aws.Ec2.Subnet("mainSubnet", new()
{
VpcId = mainVpc.Id,
CidrBlock = "168.31.0.0/20",
Tags =
{
{ "name", "emr_test" },
},
});
var allowAccess = new Aws.Ec2.SecurityGroup("allowAccess", new()
{
Description = "Allow inbound traffic",
VpcId = mainVpc.Id,
Ingress = new[]
{
new Aws.Ec2.Inputs.SecurityGroupIngressArgs
{
FromPort = 0,
ToPort = 0,
Protocol = "-1",
CidrBlocks = new[]
{
mainVpc.CidrBlock,
},
},
},
Egress = new[]
{
new Aws.Ec2.Inputs.SecurityGroupEgressArgs
{
FromPort = 0,
ToPort = 0,
Protocol = "-1",
CidrBlocks = new[]
{
"0.0.0.0/0",
},
},
},
Tags =
{
{ "name", "emr_test" },
},
}, new CustomResourceOptions
{
DependsOn = new[]
{
mainSubnet,
},
});
// IAM role for EMR Service
var iamEmrServiceRole = new Aws.Iam.Role("iamEmrServiceRole", new()
{
AssumeRolePolicy = @"{
""Version"": ""2008-10-17"",
""Statement"": [
{
""Sid"": """",
""Effect"": ""Allow"",
""Principal"": {
""Service"": ""elasticmapreduce.amazonaws.com""
},
""Action"": ""sts:AssumeRole""
}
]
}
",
});
// IAM Role for EC2 Instance Profile
var iamEmrProfileRole = new Aws.Iam.Role("iamEmrProfileRole", new()
{
AssumeRolePolicy = @"{
""Version"": ""2008-10-17"",
""Statement"": [
{
""Sid"": """",
""Effect"": ""Allow"",
""Principal"": {
""Service"": ""ec2.amazonaws.com""
},
""Action"": ""sts:AssumeRole""
}
]
}
",
});
var emrProfile = new Aws.Iam.InstanceProfile("emrProfile", new()
{
Role = iamEmrProfileRole.Name,
});
var cluster = new Aws.Emr.Cluster("cluster", new()
{
ReleaseLabel = "emr-4.6.0",
Applications = new[]
{
"Spark",
},
Ec2Attributes = new Aws.Emr.Inputs.ClusterEc2AttributesArgs
{
SubnetId = mainSubnet.Id,
EmrManagedMasterSecurityGroup = allowAccess.Id,
EmrManagedSlaveSecurityGroup = allowAccess.Id,
InstanceProfile = emrProfile.Arn,
},
MasterInstanceGroup = new Aws.Emr.Inputs.ClusterMasterInstanceGroupArgs
{
InstanceType = "m5.xlarge",
},
CoreInstanceGroup = new Aws.Emr.Inputs.ClusterCoreInstanceGroupArgs
{
InstanceCount = 1,
InstanceType = "m5.xlarge",
},
Tags =
{
{ "role", "rolename" },
{ "dns_zone", "env_zone" },
{ "env", "env" },
{ "name", "name-env" },
},
BootstrapActions = new[]
{
new Aws.Emr.Inputs.ClusterBootstrapActionArgs
{
Path = "s3://elasticmapreduce/bootstrap-actions/run-if",
Name = "runif",
Args = new[]
{
"instance.isMaster=true",
"echo running on master node",
},
},
},
ConfigurationsJson = @" [
{
""Classification"": ""hadoop-env"",
""Configurations"": [
{
""Classification"": ""export"",
""Properties"": {
""JAVA_HOME"": ""/usr/lib/jvm/java-1.8.0""
}
}
],
""Properties"": {}
},
{
""Classification"": ""spark-env"",
""Configurations"": [
{
""Classification"": ""export"",
""Properties"": {
""JAVA_HOME"": ""/usr/lib/jvm/java-1.8.0""
}
}
],
""Properties"": {}
}
]
",
ServiceRole = iamEmrServiceRole.Arn,
});
var gw = new Aws.Ec2.InternetGateway("gw", new()
{
VpcId = mainVpc.Id,
});
var routeTable = new Aws.Ec2.RouteTable("routeTable", new()
{
VpcId = mainVpc.Id,
Routes = new[]
{
new Aws.Ec2.Inputs.RouteTableRouteArgs
{
CidrBlock = "0.0.0.0/0",
GatewayId = gw.Id,
},
},
});
var mainRouteTableAssociation = new Aws.Ec2.MainRouteTableAssociation("mainRouteTableAssociation", new()
{
VpcId = mainVpc.Id,
RouteTableId = routeTable.Id,
});
//##
var iamEmrServicePolicy = new Aws.Iam.RolePolicy("iamEmrServicePolicy", new()
{
Role = iamEmrServiceRole.Id,
Policy = @"{
""Version"": ""2012-10-17"",
""Statement"": [{
""Effect"": ""Allow"",
""Resource"": ""*"",
""Action"": [
""ec2:AuthorizeSecurityGroupEgress"",
""ec2:AuthorizeSecurityGroupIngress"",
""ec2:CancelSpotInstanceRequests"",
""ec2:CreateNetworkInterface"",
""ec2:CreateSecurityGroup"",
""ec2:CreateTags"",
""ec2:DeleteNetworkInterface"",
""ec2:DeleteSecurityGroup"",
""ec2:DeleteTags"",
""ec2:DescribeAvailabilityZones"",
""ec2:DescribeAccountAttributes"",
""ec2:DescribeDhcpOptions"",
""ec2:DescribeInstanceStatus"",
""ec2:DescribeInstances"",
""ec2:DescribeKeyPairs"",
""ec2:DescribeNetworkAcls"",
""ec2:DescribeNetworkInterfaces"",
""ec2:DescribePrefixLists"",
""ec2:DescribeRouteTables"",
""ec2:DescribeSecurityGroups"",
""ec2:DescribeSpotInstanceRequests"",
""ec2:DescribeSpotPriceHistory"",
""ec2:DescribeSubnets"",
""ec2:DescribeVpcAttribute"",
""ec2:DescribeVpcEndpoints"",
""ec2:DescribeVpcEndpointServices"",
""ec2:DescribeVpcs"",
""ec2:DetachNetworkInterface"",
""ec2:ModifyImageAttribute"",
""ec2:ModifyInstanceAttribute"",
""ec2:RequestSpotInstances"",
""ec2:RevokeSecurityGroupEgress"",
""ec2:RunInstances"",
""ec2:TerminateInstances"",
""ec2:DeleteVolume"",
""ec2:DescribeVolumeStatus"",
""ec2:DescribeVolumes"",
""ec2:DetachVolume"",
""iam:GetRole"",
""iam:GetRolePolicy"",
""iam:ListInstanceProfiles"",
""iam:ListRolePolicies"",
""iam:PassRole"",
""s3:CreateBucket"",
""s3:Get*"",
""s3:List*"",
""sdb:BatchPutAttributes"",
""sdb:Select"",
""sqs:CreateQueue"",
""sqs:Delete*"",
""sqs:GetQueue*"",
""sqs:PurgeQueue"",
""sqs:ReceiveMessage""
]
}]
}
",
});
var iamEmrProfilePolicy = new Aws.Iam.RolePolicy("iamEmrProfilePolicy", new()
{
Role = iamEmrProfileRole.Id,
Policy = @"{
""Version"": ""2012-10-17"",
""Statement"": [{
""Effect"": ""Allow"",
""Resource"": ""*"",
""Action"": [
""cloudwatch:*"",
""dynamodb:*"",
""ec2:Describe*"",
""elasticmapreduce:Describe*"",
""elasticmapreduce:ListBootstrapActions"",
""elasticmapreduce:ListClusters"",
""elasticmapreduce:ListInstanceGroups"",
""elasticmapreduce:ListInstances"",
""elasticmapreduce:ListSteps"",
""kinesis:CreateStream"",
""kinesis:DeleteStream"",
""kinesis:DescribeStream"",
""kinesis:GetRecords"",
""kinesis:GetShardIterator"",
""kinesis:MergeShards"",
""kinesis:PutRecord"",
""kinesis:SplitShard"",
""rds:Describe*"",
""s3:*"",
""sdb:*"",
""sns:*"",
""sqs:*""
]
}]
}
",
});
});
package main
import (
"fmt"
"github.com/pulumi/pulumi-aws/sdk/v5/go/aws/ec2"
"github.com/pulumi/pulumi-aws/sdk/v5/go/aws/emr"
"github.com/pulumi/pulumi-aws/sdk/v5/go/aws/iam"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
mainVpc, err := ec2.NewVpc(ctx, "mainVpc", &ec2.VpcArgs{
CidrBlock: pulumi.String("168.31.0.0/16"),
EnableDnsHostnames: pulumi.Bool(true),
Tags: pulumi.StringMap{
"name": pulumi.String("emr_test"),
},
})
if err != nil {
return err
}
mainSubnet, err := ec2.NewSubnet(ctx, "mainSubnet", &ec2.SubnetArgs{
VpcId: mainVpc.ID(),
CidrBlock: pulumi.String("168.31.0.0/20"),
Tags: pulumi.StringMap{
"name": pulumi.String("emr_test"),
},
})
if err != nil {
return err
}
allowAccess, err := ec2.NewSecurityGroup(ctx, "allowAccess", &ec2.SecurityGroupArgs{
Description: pulumi.String("Allow inbound traffic"),
VpcId: mainVpc.ID(),
Ingress: ec2.SecurityGroupIngressArray{
&ec2.SecurityGroupIngressArgs{
FromPort: pulumi.Int(0),
ToPort: pulumi.Int(0),
Protocol: pulumi.String("-1"),
CidrBlocks: pulumi.StringArray{
mainVpc.CidrBlock,
},
},
},
Egress: ec2.SecurityGroupEgressArray{
&ec2.SecurityGroupEgressArgs{
FromPort: pulumi.Int(0),
ToPort: pulumi.Int(0),
Protocol: pulumi.String("-1"),
CidrBlocks: pulumi.StringArray{
pulumi.String("0.0.0.0/0"),
},
},
},
Tags: pulumi.StringMap{
"name": pulumi.String("emr_test"),
},
}, pulumi.DependsOn([]pulumi.Resource{
mainSubnet,
}))
if err != nil {
return err
}
iamEmrServiceRole, err := iam.NewRole(ctx, "iamEmrServiceRole", &iam.RoleArgs{
AssumeRolePolicy: pulumi.Any(fmt.Sprintf(`{
"Version": "2008-10-17",
"Statement": [
{
"Sid": "",
"Effect": "Allow",
"Principal": {
"Service": "elasticmapreduce.amazonaws.com"
},
"Action": "sts:AssumeRole"
}
]
}
`)),
})
if err != nil {
return err
}
iamEmrProfileRole, err := iam.NewRole(ctx, "iamEmrProfileRole", &iam.RoleArgs{
AssumeRolePolicy: pulumi.Any(fmt.Sprintf(`{
"Version": "2008-10-17",
"Statement": [
{
"Sid": "",
"Effect": "Allow",
"Principal": {
"Service": "ec2.amazonaws.com"
},
"Action": "sts:AssumeRole"
}
]
}
`)),
})
if err != nil {
return err
}
emrProfile, err := iam.NewInstanceProfile(ctx, "emrProfile", &iam.InstanceProfileArgs{
Role: iamEmrProfileRole.Name,
})
if err != nil {
return err
}
_, err = emr.NewCluster(ctx, "cluster", &emr.ClusterArgs{
ReleaseLabel: pulumi.String("emr-4.6.0"),
Applications: pulumi.StringArray{
pulumi.String("Spark"),
},
Ec2Attributes: &emr.ClusterEc2AttributesArgs{
SubnetId: mainSubnet.ID(),
EmrManagedMasterSecurityGroup: allowAccess.ID(),
EmrManagedSlaveSecurityGroup: allowAccess.ID(),
InstanceProfile: emrProfile.Arn,
},
MasterInstanceGroup: &emr.ClusterMasterInstanceGroupArgs{
InstanceType: pulumi.String("m5.xlarge"),
},
CoreInstanceGroup: &emr.ClusterCoreInstanceGroupArgs{
InstanceCount: pulumi.Int(1),
InstanceType: pulumi.String("m5.xlarge"),
},
Tags: pulumi.StringMap{
"role": pulumi.String("rolename"),
"dns_zone": pulumi.String("env_zone"),
"env": pulumi.String("env"),
"name": pulumi.String("name-env"),
},
BootstrapActions: emr.ClusterBootstrapActionArray{
&emr.ClusterBootstrapActionArgs{
Path: pulumi.String("s3://elasticmapreduce/bootstrap-actions/run-if"),
Name: pulumi.String("runif"),
Args: pulumi.StringArray{
pulumi.String("instance.isMaster=true"),
pulumi.String("echo running on master node"),
},
},
},
ConfigurationsJson: pulumi.String(fmt.Sprintf(` [
{
"Classification": "hadoop-env",
"Configurations": [
{
"Classification": "export",
"Properties": {
"JAVA_HOME": "/usr/lib/jvm/java-1.8.0"
}
}
],
"Properties": {}
},
{
"Classification": "spark-env",
"Configurations": [
{
"Classification": "export",
"Properties": {
"JAVA_HOME": "/usr/lib/jvm/java-1.8.0"
}
}
],
"Properties": {}
}
]
`)),
ServiceRole: iamEmrServiceRole.Arn,
})
if err != nil {
return err
}
gw, err := ec2.NewInternetGateway(ctx, "gw", &ec2.InternetGatewayArgs{
VpcId: mainVpc.ID(),
})
if err != nil {
return err
}
routeTable, err := ec2.NewRouteTable(ctx, "routeTable", &ec2.RouteTableArgs{
VpcId: mainVpc.ID(),
Routes: ec2.RouteTableRouteArray{
&ec2.RouteTableRouteArgs{
CidrBlock: pulumi.String("0.0.0.0/0"),
GatewayId: gw.ID(),
},
},
})
if err != nil {
return err
}
_, err = ec2.NewMainRouteTableAssociation(ctx, "mainRouteTableAssociation", &ec2.MainRouteTableAssociationArgs{
VpcId: mainVpc.ID(),
RouteTableId: routeTable.ID(),
})
if err != nil {
return err
}
_, err = iam.NewRolePolicy(ctx, "iamEmrServicePolicy", &iam.RolePolicyArgs{
Role: iamEmrServiceRole.ID(),
Policy: pulumi.Any(fmt.Sprintf(`{
"Version": "2012-10-17",
"Statement": [{
"Effect": "Allow",
"Resource": "*",
"Action": [
"ec2:AuthorizeSecurityGroupEgress",
"ec2:AuthorizeSecurityGroupIngress",
"ec2:CancelSpotInstanceRequests",
"ec2:CreateNetworkInterface",
"ec2:CreateSecurityGroup",
"ec2:CreateTags",
"ec2:DeleteNetworkInterface",
"ec2:DeleteSecurityGroup",
"ec2:DeleteTags",
"ec2:DescribeAvailabilityZones",
"ec2:DescribeAccountAttributes",
"ec2:DescribeDhcpOptions",
"ec2:DescribeInstanceStatus",
"ec2:DescribeInstances",
"ec2:DescribeKeyPairs",
"ec2:DescribeNetworkAcls",
"ec2:DescribeNetworkInterfaces",
"ec2:DescribePrefixLists",
"ec2:DescribeRouteTables",
"ec2:DescribeSecurityGroups",
"ec2:DescribeSpotInstanceRequests",
"ec2:DescribeSpotPriceHistory",
"ec2:DescribeSubnets",
"ec2:DescribeVpcAttribute",
"ec2:DescribeVpcEndpoints",
"ec2:DescribeVpcEndpointServices",
"ec2:DescribeVpcs",
"ec2:DetachNetworkInterface",
"ec2:ModifyImageAttribute",
"ec2:ModifyInstanceAttribute",
"ec2:RequestSpotInstances",
"ec2:RevokeSecurityGroupEgress",
"ec2:RunInstances",
"ec2:TerminateInstances",
"ec2:DeleteVolume",
"ec2:DescribeVolumeStatus",
"ec2:DescribeVolumes",
"ec2:DetachVolume",
"iam:GetRole",
"iam:GetRolePolicy",
"iam:ListInstanceProfiles",
"iam:ListRolePolicies",
"iam:PassRole",
"s3:CreateBucket",
"s3:Get*",
"s3:List*",
"sdb:BatchPutAttributes",
"sdb:Select",
"sqs:CreateQueue",
"sqs:Delete*",
"sqs:GetQueue*",
"sqs:PurgeQueue",
"sqs:ReceiveMessage"
]
}]
}
`)),
})
if err != nil {
return err
}
_, err = iam.NewRolePolicy(ctx, "iamEmrProfilePolicy", &iam.RolePolicyArgs{
Role: iamEmrProfileRole.ID(),
Policy: pulumi.Any(fmt.Sprintf(`{
"Version": "2012-10-17",
"Statement": [{
"Effect": "Allow",
"Resource": "*",
"Action": [
"cloudwatch:*",
"dynamodb:*",
"ec2:Describe*",
"elasticmapreduce:Describe*",
"elasticmapreduce:ListBootstrapActions",
"elasticmapreduce:ListClusters",
"elasticmapreduce:ListInstanceGroups",
"elasticmapreduce:ListInstances",
"elasticmapreduce:ListSteps",
"kinesis:CreateStream",
"kinesis:DeleteStream",
"kinesis:DescribeStream",
"kinesis:GetRecords",
"kinesis:GetShardIterator",
"kinesis:MergeShards",
"kinesis:PutRecord",
"kinesis:SplitShard",
"rds:Describe*",
"s3:*",
"sdb:*",
"sns:*",
"sqs:*"
]
}]
}
`)),
})
if err != nil {
return err
}
return nil
})
}
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.ec2.Vpc;
import com.pulumi.aws.ec2.VpcArgs;
import com.pulumi.aws.ec2.Subnet;
import com.pulumi.aws.ec2.SubnetArgs;
import com.pulumi.aws.ec2.SecurityGroup;
import com.pulumi.aws.ec2.SecurityGroupArgs;
import com.pulumi.aws.ec2.inputs.SecurityGroupIngressArgs;
import com.pulumi.aws.ec2.inputs.SecurityGroupEgressArgs;
import com.pulumi.aws.iam.Role;
import com.pulumi.aws.iam.RoleArgs;
import com.pulumi.aws.iam.InstanceProfile;
import com.pulumi.aws.iam.InstanceProfileArgs;
import com.pulumi.aws.emr.Cluster;
import com.pulumi.aws.emr.ClusterArgs;
import com.pulumi.aws.emr.inputs.ClusterEc2AttributesArgs;
import com.pulumi.aws.emr.inputs.ClusterMasterInstanceGroupArgs;
import com.pulumi.aws.emr.inputs.ClusterCoreInstanceGroupArgs;
import com.pulumi.aws.emr.inputs.ClusterBootstrapActionArgs;
import com.pulumi.aws.ec2.InternetGateway;
import com.pulumi.aws.ec2.InternetGatewayArgs;
import com.pulumi.aws.ec2.RouteTable;
import com.pulumi.aws.ec2.RouteTableArgs;
import com.pulumi.aws.ec2.inputs.RouteTableRouteArgs;
import com.pulumi.aws.ec2.MainRouteTableAssociation;
import com.pulumi.aws.ec2.MainRouteTableAssociationArgs;
import com.pulumi.aws.iam.RolePolicy;
import com.pulumi.aws.iam.RolePolicyArgs;
import com.pulumi.resources.CustomResourceOptions;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var mainVpc = new Vpc("mainVpc", VpcArgs.builder()
.cidrBlock("168.31.0.0/16")
.enableDnsHostnames(true)
.tags(Map.of("name", "emr_test"))
.build());
var mainSubnet = new Subnet("mainSubnet", SubnetArgs.builder()
.vpcId(mainVpc.id())
.cidrBlock("168.31.0.0/20")
.tags(Map.of("name", "emr_test"))
.build());
var allowAccess = new SecurityGroup("allowAccess", SecurityGroupArgs.builder()
.description("Allow inbound traffic")
.vpcId(mainVpc.id())
.ingress(SecurityGroupIngressArgs.builder()
.fromPort(0)
.toPort(0)
.protocol("-1")
.cidrBlocks(mainVpc.cidrBlock())
.build())
.egress(SecurityGroupEgressArgs.builder()
.fromPort(0)
.toPort(0)
.protocol("-1")
.cidrBlocks("0.0.0.0/0")
.build())
.tags(Map.of("name", "emr_test"))
.build(), CustomResourceOptions.builder()
.dependsOn(mainSubnet)
.build());
var iamEmrServiceRole = new Role("iamEmrServiceRole", RoleArgs.builder()
.assumeRolePolicy("""
{
"Version": "2008-10-17",
"Statement": [
{
"Sid": "",
"Effect": "Allow",
"Principal": {
"Service": "elasticmapreduce.amazonaws.com"
},
"Action": "sts:AssumeRole"
}
]
}
""")
.build());
var iamEmrProfileRole = new Role("iamEmrProfileRole", RoleArgs.builder()
.assumeRolePolicy("""
{
"Version": "2008-10-17",
"Statement": [
{
"Sid": "",
"Effect": "Allow",
"Principal": {
"Service": "ec2.amazonaws.com"
},
"Action": "sts:AssumeRole"
}
]
}
""")
.build());
var emrProfile = new InstanceProfile("emrProfile", InstanceProfileArgs.builder()
.role(iamEmrProfileRole.name())
.build());
var cluster = new Cluster("cluster", ClusterArgs.builder()
.releaseLabel("emr-4.6.0")
.applications("Spark")
.ec2Attributes(ClusterEc2AttributesArgs.builder()
.subnetId(mainSubnet.id())
.emrManagedMasterSecurityGroup(allowAccess.id())
.emrManagedSlaveSecurityGroup(allowAccess.id())
.instanceProfile(emrProfile.arn())
.build())
.masterInstanceGroup(ClusterMasterInstanceGroupArgs.builder()
.instanceType("m5.xlarge")
.build())
.coreInstanceGroup(ClusterCoreInstanceGroupArgs.builder()
.instanceCount(1)
.instanceType("m5.xlarge")
.build())
.tags(Map.ofEntries(
Map.entry("role", "rolename"),
Map.entry("dns_zone", "env_zone"),
Map.entry("env", "env"),
Map.entry("name", "name-env")
))
.bootstrapActions(ClusterBootstrapActionArgs.builder()
.path("s3://elasticmapreduce/bootstrap-actions/run-if")
.name("runif")
.args(
"instance.isMaster=true",
"echo running on master node")
.build())
.configurationsJson("""
[
{
"Classification": "hadoop-env",
"Configurations": [
{
"Classification": "export",
"Properties": {
"JAVA_HOME": "/usr/lib/jvm/java-1.8.0"
}
}
],
"Properties": {}
},
{
"Classification": "spark-env",
"Configurations": [
{
"Classification": "export",
"Properties": {
"JAVA_HOME": "/usr/lib/jvm/java-1.8.0"
}
}
],
"Properties": {}
}
]
""")
.serviceRole(iamEmrServiceRole.arn())
.build());
var gw = new InternetGateway("gw", InternetGatewayArgs.builder()
.vpcId(mainVpc.id())
.build());
var routeTable = new RouteTable("routeTable", RouteTableArgs.builder()
.vpcId(mainVpc.id())
.routes(RouteTableRouteArgs.builder()
.cidrBlock("0.0.0.0/0")
.gatewayId(gw.id())
.build())
.build());
var mainRouteTableAssociation = new MainRouteTableAssociation("mainRouteTableAssociation", MainRouteTableAssociationArgs.builder()
.vpcId(mainVpc.id())
.routeTableId(routeTable.id())
.build());
var iamEmrServicePolicy = new RolePolicy("iamEmrServicePolicy", RolePolicyArgs.builder()
.role(iamEmrServiceRole.id())
.policy("""
{
"Version": "2012-10-17",
"Statement": [{
"Effect": "Allow",
"Resource": "*",
"Action": [
"ec2:AuthorizeSecurityGroupEgress",
"ec2:AuthorizeSecurityGroupIngress",
"ec2:CancelSpotInstanceRequests",
"ec2:CreateNetworkInterface",
"ec2:CreateSecurityGroup",
"ec2:CreateTags",
"ec2:DeleteNetworkInterface",
"ec2:DeleteSecurityGroup",
"ec2:DeleteTags",
"ec2:DescribeAvailabilityZones",
"ec2:DescribeAccountAttributes",
"ec2:DescribeDhcpOptions",
"ec2:DescribeInstanceStatus",
"ec2:DescribeInstances",
"ec2:DescribeKeyPairs",
"ec2:DescribeNetworkAcls",
"ec2:DescribeNetworkInterfaces",
"ec2:DescribePrefixLists",
"ec2:DescribeRouteTables",
"ec2:DescribeSecurityGroups",
"ec2:DescribeSpotInstanceRequests",
"ec2:DescribeSpotPriceHistory",
"ec2:DescribeSubnets",
"ec2:DescribeVpcAttribute",
"ec2:DescribeVpcEndpoints",
"ec2:DescribeVpcEndpointServices",
"ec2:DescribeVpcs",
"ec2:DetachNetworkInterface",
"ec2:ModifyImageAttribute",
"ec2:ModifyInstanceAttribute",
"ec2:RequestSpotInstances",
"ec2:RevokeSecurityGroupEgress",
"ec2:RunInstances",
"ec2:TerminateInstances",
"ec2:DeleteVolume",
"ec2:DescribeVolumeStatus",
"ec2:DescribeVolumes",
"ec2:DetachVolume",
"iam:GetRole",
"iam:GetRolePolicy",
"iam:ListInstanceProfiles",
"iam:ListRolePolicies",
"iam:PassRole",
"s3:CreateBucket",
"s3:Get*",
"s3:List*",
"sdb:BatchPutAttributes",
"sdb:Select",
"sqs:CreateQueue",
"sqs:Delete*",
"sqs:GetQueue*",
"sqs:PurgeQueue",
"sqs:ReceiveMessage"
]
}]
}
""")
.build());
var iamEmrProfilePolicy = new RolePolicy("iamEmrProfilePolicy", RolePolicyArgs.builder()
.role(iamEmrProfileRole.id())
.policy("""
{
"Version": "2012-10-17",
"Statement": [{
"Effect": "Allow",
"Resource": "*",
"Action": [
"cloudwatch:*",
"dynamodb:*",
"ec2:Describe*",
"elasticmapreduce:Describe*",
"elasticmapreduce:ListBootstrapActions",
"elasticmapreduce:ListClusters",
"elasticmapreduce:ListInstanceGroups",
"elasticmapreduce:ListInstances",
"elasticmapreduce:ListSteps",
"kinesis:CreateStream",
"kinesis:DeleteStream",
"kinesis:DescribeStream",
"kinesis:GetRecords",
"kinesis:GetShardIterator",
"kinesis:MergeShards",
"kinesis:PutRecord",
"kinesis:SplitShard",
"rds:Describe*",
"s3:*",
"sdb:*",
"sns:*",
"sqs:*"
]
}]
}
""")
.build());
}
}
import pulumi
import pulumi_aws as aws
main_vpc = aws.ec2.Vpc("mainVpc",
cidr_block="168.31.0.0/16",
enable_dns_hostnames=True,
tags={
"name": "emr_test",
})
main_subnet = aws.ec2.Subnet("mainSubnet",
vpc_id=main_vpc.id,
cidr_block="168.31.0.0/20",
tags={
"name": "emr_test",
})
allow_access = aws.ec2.SecurityGroup("allowAccess",
description="Allow inbound traffic",
vpc_id=main_vpc.id,
ingress=[aws.ec2.SecurityGroupIngressArgs(
from_port=0,
to_port=0,
protocol="-1",
cidr_blocks=[main_vpc.cidr_block],
)],
egress=[aws.ec2.SecurityGroupEgressArgs(
from_port=0,
to_port=0,
protocol="-1",
cidr_blocks=["0.0.0.0/0"],
)],
tags={
"name": "emr_test",
},
opts=pulumi.ResourceOptions(depends_on=[main_subnet]))
# IAM role for EMR Service
iam_emr_service_role = aws.iam.Role("iamEmrServiceRole", assume_role_policy="""{
"Version": "2008-10-17",
"Statement": [
{
"Sid": "",
"Effect": "Allow",
"Principal": {
"Service": "elasticmapreduce.amazonaws.com"
},
"Action": "sts:AssumeRole"
}
]
}
""")
# IAM Role for EC2 Instance Profile
iam_emr_profile_role = aws.iam.Role("iamEmrProfileRole", assume_role_policy="""{
"Version": "2008-10-17",
"Statement": [
{
"Sid": "",
"Effect": "Allow",
"Principal": {
"Service": "ec2.amazonaws.com"
},
"Action": "sts:AssumeRole"
}
]
}
""")
emr_profile = aws.iam.InstanceProfile("emrProfile", role=iam_emr_profile_role.name)
cluster = aws.emr.Cluster("cluster",
release_label="emr-4.6.0",
applications=["Spark"],
ec2_attributes=aws.emr.ClusterEc2AttributesArgs(
subnet_id=main_subnet.id,
emr_managed_master_security_group=allow_access.id,
emr_managed_slave_security_group=allow_access.id,
instance_profile=emr_profile.arn,
),
master_instance_group=aws.emr.ClusterMasterInstanceGroupArgs(
instance_type="m5.xlarge",
),
core_instance_group=aws.emr.ClusterCoreInstanceGroupArgs(
instance_count=1,
instance_type="m5.xlarge",
),
tags={
"role": "rolename",
"dns_zone": "env_zone",
"env": "env",
"name": "name-env",
},
bootstrap_actions=[aws.emr.ClusterBootstrapActionArgs(
path="s3://elasticmapreduce/bootstrap-actions/run-if",
name="runif",
args=[
"instance.isMaster=true",
"echo running on master node",
],
)],
configurations_json=""" [
{
"Classification": "hadoop-env",
"Configurations": [
{
"Classification": "export",
"Properties": {
"JAVA_HOME": "/usr/lib/jvm/java-1.8.0"
}
}
],
"Properties": {}
},
{
"Classification": "spark-env",
"Configurations": [
{
"Classification": "export",
"Properties": {
"JAVA_HOME": "/usr/lib/jvm/java-1.8.0"
}
}
],
"Properties": {}
}
]
""",
service_role=iam_emr_service_role.arn)
gw = aws.ec2.InternetGateway("gw", vpc_id=main_vpc.id)
route_table = aws.ec2.RouteTable("routeTable",
vpc_id=main_vpc.id,
routes=[aws.ec2.RouteTableRouteArgs(
cidr_block="0.0.0.0/0",
gateway_id=gw.id,
)])
main_route_table_association = aws.ec2.MainRouteTableAssociation("mainRouteTableAssociation",
vpc_id=main_vpc.id,
route_table_id=route_table.id)
###
iam_emr_service_policy = aws.iam.RolePolicy("iamEmrServicePolicy",
role=iam_emr_service_role.id,
policy="""{
"Version": "2012-10-17",
"Statement": [{
"Effect": "Allow",
"Resource": "*",
"Action": [
"ec2:AuthorizeSecurityGroupEgress",
"ec2:AuthorizeSecurityGroupIngress",
"ec2:CancelSpotInstanceRequests",
"ec2:CreateNetworkInterface",
"ec2:CreateSecurityGroup",
"ec2:CreateTags",
"ec2:DeleteNetworkInterface",
"ec2:DeleteSecurityGroup",
"ec2:DeleteTags",
"ec2:DescribeAvailabilityZones",
"ec2:DescribeAccountAttributes",
"ec2:DescribeDhcpOptions",
"ec2:DescribeInstanceStatus",
"ec2:DescribeInstances",
"ec2:DescribeKeyPairs",
"ec2:DescribeNetworkAcls",
"ec2:DescribeNetworkInterfaces",
"ec2:DescribePrefixLists",
"ec2:DescribeRouteTables",
"ec2:DescribeSecurityGroups",
"ec2:DescribeSpotInstanceRequests",
"ec2:DescribeSpotPriceHistory",
"ec2:DescribeSubnets",
"ec2:DescribeVpcAttribute",
"ec2:DescribeVpcEndpoints",
"ec2:DescribeVpcEndpointServices",
"ec2:DescribeVpcs",
"ec2:DetachNetworkInterface",
"ec2:ModifyImageAttribute",
"ec2:ModifyInstanceAttribute",
"ec2:RequestSpotInstances",
"ec2:RevokeSecurityGroupEgress",
"ec2:RunInstances",
"ec2:TerminateInstances",
"ec2:DeleteVolume",
"ec2:DescribeVolumeStatus",
"ec2:DescribeVolumes",
"ec2:DetachVolume",
"iam:GetRole",
"iam:GetRolePolicy",
"iam:ListInstanceProfiles",
"iam:ListRolePolicies",
"iam:PassRole",
"s3:CreateBucket",
"s3:Get*",
"s3:List*",
"sdb:BatchPutAttributes",
"sdb:Select",
"sqs:CreateQueue",
"sqs:Delete*",
"sqs:GetQueue*",
"sqs:PurgeQueue",
"sqs:ReceiveMessage"
]
}]
}
""")
iam_emr_profile_policy = aws.iam.RolePolicy("iamEmrProfilePolicy",
role=iam_emr_profile_role.id,
policy="""{
"Version": "2012-10-17",
"Statement": [{
"Effect": "Allow",
"Resource": "*",
"Action": [
"cloudwatch:*",
"dynamodb:*",
"ec2:Describe*",
"elasticmapreduce:Describe*",
"elasticmapreduce:ListBootstrapActions",
"elasticmapreduce:ListClusters",
"elasticmapreduce:ListInstanceGroups",
"elasticmapreduce:ListInstances",
"elasticmapreduce:ListSteps",
"kinesis:CreateStream",
"kinesis:DeleteStream",
"kinesis:DescribeStream",
"kinesis:GetRecords",
"kinesis:GetShardIterator",
"kinesis:MergeShards",
"kinesis:PutRecord",
"kinesis:SplitShard",
"rds:Describe*",
"s3:*",
"sdb:*",
"sns:*",
"sqs:*"
]
}]
}
""")
import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
const mainVpc = new aws.ec2.Vpc("mainVpc", {
cidrBlock: "168.31.0.0/16",
enableDnsHostnames: true,
tags: {
name: "emr_test",
},
});
const mainSubnet = new aws.ec2.Subnet("mainSubnet", {
vpcId: mainVpc.id,
cidrBlock: "168.31.0.0/20",
tags: {
name: "emr_test",
},
});
const allowAccess = new aws.ec2.SecurityGroup("allowAccess", {
description: "Allow inbound traffic",
vpcId: mainVpc.id,
ingress: [{
fromPort: 0,
toPort: 0,
protocol: "-1",
cidrBlocks: [mainVpc.cidrBlock],
}],
egress: [{
fromPort: 0,
toPort: 0,
protocol: "-1",
cidrBlocks: ["0.0.0.0/0"],
}],
tags: {
name: "emr_test",
},
}, {
dependsOn: [mainSubnet],
});
// IAM role for EMR Service
const iamEmrServiceRole = new aws.iam.Role("iamEmrServiceRole", {assumeRolePolicy: `{
"Version": "2008-10-17",
"Statement": [
{
"Sid": "",
"Effect": "Allow",
"Principal": {
"Service": "elasticmapreduce.amazonaws.com"
},
"Action": "sts:AssumeRole"
}
]
}
`});
// IAM Role for EC2 Instance Profile
const iamEmrProfileRole = new aws.iam.Role("iamEmrProfileRole", {assumeRolePolicy: `{
"Version": "2008-10-17",
"Statement": [
{
"Sid": "",
"Effect": "Allow",
"Principal": {
"Service": "ec2.amazonaws.com"
},
"Action": "sts:AssumeRole"
}
]
}
`});
const emrProfile = new aws.iam.InstanceProfile("emrProfile", {role: iamEmrProfileRole.name});
const cluster = new aws.emr.Cluster("cluster", {
releaseLabel: "emr-4.6.0",
applications: ["Spark"],
ec2Attributes: {
subnetId: mainSubnet.id,
emrManagedMasterSecurityGroup: allowAccess.id,
emrManagedSlaveSecurityGroup: allowAccess.id,
instanceProfile: emrProfile.arn,
},
masterInstanceGroup: {
instanceType: "m5.xlarge",
},
coreInstanceGroup: {
instanceCount: 1,
instanceType: "m5.xlarge",
},
tags: {
role: "rolename",
dns_zone: "env_zone",
env: "env",
name: "name-env",
},
bootstrapActions: [{
path: "s3://elasticmapreduce/bootstrap-actions/run-if",
name: "runif",
args: [
"instance.isMaster=true",
"echo running on master node",
],
}],
configurationsJson: ` [
{
"Classification": "hadoop-env",
"Configurations": [
{
"Classification": "export",
"Properties": {
"JAVA_HOME": "/usr/lib/jvm/java-1.8.0"
}
}
],
"Properties": {}
},
{
"Classification": "spark-env",
"Configurations": [
{
"Classification": "export",
"Properties": {
"JAVA_HOME": "/usr/lib/jvm/java-1.8.0"
}
}
],
"Properties": {}
}
]
`,
serviceRole: iamEmrServiceRole.arn,
});
const gw = new aws.ec2.InternetGateway("gw", {vpcId: mainVpc.id});
const routeTable = new aws.ec2.RouteTable("routeTable", {
vpcId: mainVpc.id,
routes: [{
cidrBlock: "0.0.0.0/0",
gatewayId: gw.id,
}],
});
const mainRouteTableAssociation = new aws.ec2.MainRouteTableAssociation("mainRouteTableAssociation", {
vpcId: mainVpc.id,
routeTableId: routeTable.id,
});
//##
const iamEmrServicePolicy = new aws.iam.RolePolicy("iamEmrServicePolicy", {
role: iamEmrServiceRole.id,
policy: `{
"Version": "2012-10-17",
"Statement": [{
"Effect": "Allow",
"Resource": "*",
"Action": [
"ec2:AuthorizeSecurityGroupEgress",
"ec2:AuthorizeSecurityGroupIngress",
"ec2:CancelSpotInstanceRequests",
"ec2:CreateNetworkInterface",
"ec2:CreateSecurityGroup",
"ec2:CreateTags",
"ec2:DeleteNetworkInterface",
"ec2:DeleteSecurityGroup",
"ec2:DeleteTags",
"ec2:DescribeAvailabilityZones",
"ec2:DescribeAccountAttributes",
"ec2:DescribeDhcpOptions",
"ec2:DescribeInstanceStatus",
"ec2:DescribeInstances",
"ec2:DescribeKeyPairs",
"ec2:DescribeNetworkAcls",
"ec2:DescribeNetworkInterfaces",
"ec2:DescribePrefixLists",
"ec2:DescribeRouteTables",
"ec2:DescribeSecurityGroups",
"ec2:DescribeSpotInstanceRequests",
"ec2:DescribeSpotPriceHistory",
"ec2:DescribeSubnets",
"ec2:DescribeVpcAttribute",
"ec2:DescribeVpcEndpoints",
"ec2:DescribeVpcEndpointServices",
"ec2:DescribeVpcs",
"ec2:DetachNetworkInterface",
"ec2:ModifyImageAttribute",
"ec2:ModifyInstanceAttribute",
"ec2:RequestSpotInstances",
"ec2:RevokeSecurityGroupEgress",
"ec2:RunInstances",
"ec2:TerminateInstances",
"ec2:DeleteVolume",
"ec2:DescribeVolumeStatus",
"ec2:DescribeVolumes",
"ec2:DetachVolume",
"iam:GetRole",
"iam:GetRolePolicy",
"iam:ListInstanceProfiles",
"iam:ListRolePolicies",
"iam:PassRole",
"s3:CreateBucket",
"s3:Get*",
"s3:List*",
"sdb:BatchPutAttributes",
"sdb:Select",
"sqs:CreateQueue",
"sqs:Delete*",
"sqs:GetQueue*",
"sqs:PurgeQueue",
"sqs:ReceiveMessage"
]
}]
}
`,
});
const iamEmrProfilePolicy = new aws.iam.RolePolicy("iamEmrProfilePolicy", {
role: iamEmrProfileRole.id,
policy: `{
"Version": "2012-10-17",
"Statement": [{
"Effect": "Allow",
"Resource": "*",
"Action": [
"cloudwatch:*",
"dynamodb:*",
"ec2:Describe*",
"elasticmapreduce:Describe*",
"elasticmapreduce:ListBootstrapActions",
"elasticmapreduce:ListClusters",
"elasticmapreduce:ListInstanceGroups",
"elasticmapreduce:ListInstances",
"elasticmapreduce:ListSteps",
"kinesis:CreateStream",
"kinesis:DeleteStream",
"kinesis:DescribeStream",
"kinesis:GetRecords",
"kinesis:GetShardIterator",
"kinesis:MergeShards",
"kinesis:PutRecord",
"kinesis:SplitShard",
"rds:Describe*",
"s3:*",
"sdb:*",
"sns:*",
"sqs:*"
]
}]
}
`,
});
resources:
cluster:
type: aws:emr:Cluster
properties:
releaseLabel: emr-4.6.0
applications:
- Spark
ec2Attributes:
subnetId: ${mainSubnet.id}
emrManagedMasterSecurityGroup: ${allowAccess.id}
emrManagedSlaveSecurityGroup: ${allowAccess.id}
instanceProfile: ${emrProfile.arn}
masterInstanceGroup:
instanceType: m5.xlarge
coreInstanceGroup:
instanceCount: 1
instanceType: m5.xlarge
tags:
role: rolename
dns_zone: env_zone
env: env
name: name-env
bootstrapActions:
- path: s3://elasticmapreduce/bootstrap-actions/run-if
name: runif
args:
- instance.isMaster=true
- echo running on master node
configurationsJson: |2
[
{
"Classification": "hadoop-env",
"Configurations": [
{
"Classification": "export",
"Properties": {
"JAVA_HOME": "/usr/lib/jvm/java-1.8.0"
}
}
],
"Properties": {}
},
{
"Classification": "spark-env",
"Configurations": [
{
"Classification": "export",
"Properties": {
"JAVA_HOME": "/usr/lib/jvm/java-1.8.0"
}
}
],
"Properties": {}
}
]
serviceRole: ${iamEmrServiceRole.arn}
allowAccess:
type: aws:ec2:SecurityGroup
properties:
description: Allow inbound traffic
vpcId: ${mainVpc.id}
ingress:
- fromPort: 0
toPort: 0
protocol: -1
cidrBlocks:
- ${mainVpc.cidrBlock}
egress:
- fromPort: 0
toPort: 0
protocol: -1
cidrBlocks:
- 0.0.0.0/0
tags:
name: emr_test
options:
dependson:
- ${mainSubnet}
mainVpc:
type: aws:ec2:Vpc
properties:
cidrBlock: 168.31.0.0/16
enableDnsHostnames: true
tags:
name: emr_test
mainSubnet:
type: aws:ec2:Subnet
properties:
vpcId: ${mainVpc.id}
cidrBlock: 168.31.0.0/20
tags:
name: emr_test
gw:
type: aws:ec2:InternetGateway
properties:
vpcId: ${mainVpc.id}
routeTable:
type: aws:ec2:RouteTable
properties:
vpcId: ${mainVpc.id}
routes:
- cidrBlock: 0.0.0.0/0
gatewayId: ${gw.id}
mainRouteTableAssociation: ###
type: aws:ec2:MainRouteTableAssociation
properties:
vpcId: ${mainVpc.id}
routeTableId: ${routeTable.id}
# IAM role for EMR Service
iamEmrServiceRole:
type: aws:iam:Role
properties:
assumeRolePolicy: |
{
"Version": "2008-10-17",
"Statement": [
{
"Sid": "",
"Effect": "Allow",
"Principal": {
"Service": "elasticmapreduce.amazonaws.com"
},
"Action": "sts:AssumeRole"
}
]
}
iamEmrServicePolicy:
type: aws:iam:RolePolicy
properties:
role: ${iamEmrServiceRole.id}
policy: |
{
"Version": "2012-10-17",
"Statement": [{
"Effect": "Allow",
"Resource": "*",
"Action": [
"ec2:AuthorizeSecurityGroupEgress",
"ec2:AuthorizeSecurityGroupIngress",
"ec2:CancelSpotInstanceRequests",
"ec2:CreateNetworkInterface",
"ec2:CreateSecurityGroup",
"ec2:CreateTags",
"ec2:DeleteNetworkInterface",
"ec2:DeleteSecurityGroup",
"ec2:DeleteTags",
"ec2:DescribeAvailabilityZones",
"ec2:DescribeAccountAttributes",
"ec2:DescribeDhcpOptions",
"ec2:DescribeInstanceStatus",
"ec2:DescribeInstances",
"ec2:DescribeKeyPairs",
"ec2:DescribeNetworkAcls",
"ec2:DescribeNetworkInterfaces",
"ec2:DescribePrefixLists",
"ec2:DescribeRouteTables",
"ec2:DescribeSecurityGroups",
"ec2:DescribeSpotInstanceRequests",
"ec2:DescribeSpotPriceHistory",
"ec2:DescribeSubnets",
"ec2:DescribeVpcAttribute",
"ec2:DescribeVpcEndpoints",
"ec2:DescribeVpcEndpointServices",
"ec2:DescribeVpcs",
"ec2:DetachNetworkInterface",
"ec2:ModifyImageAttribute",
"ec2:ModifyInstanceAttribute",
"ec2:RequestSpotInstances",
"ec2:RevokeSecurityGroupEgress",
"ec2:RunInstances",
"ec2:TerminateInstances",
"ec2:DeleteVolume",
"ec2:DescribeVolumeStatus",
"ec2:DescribeVolumes",
"ec2:DetachVolume",
"iam:GetRole",
"iam:GetRolePolicy",
"iam:ListInstanceProfiles",
"iam:ListRolePolicies",
"iam:PassRole",
"s3:CreateBucket",
"s3:Get*",
"s3:List*",
"sdb:BatchPutAttributes",
"sdb:Select",
"sqs:CreateQueue",
"sqs:Delete*",
"sqs:GetQueue*",
"sqs:PurgeQueue",
"sqs:ReceiveMessage"
]
}]
}
# IAM Role for EC2 Instance Profile
iamEmrProfileRole:
type: aws:iam:Role
properties:
assumeRolePolicy: |
{
"Version": "2008-10-17",
"Statement": [
{
"Sid": "",
"Effect": "Allow",
"Principal": {
"Service": "ec2.amazonaws.com"
},
"Action": "sts:AssumeRole"
}
]
}
emrProfile:
type: aws:iam:InstanceProfile
properties:
role: ${iamEmrProfileRole.name}
iamEmrProfilePolicy:
type: aws:iam:RolePolicy
properties:
role: ${iamEmrProfileRole.id}
policy: |
{
"Version": "2012-10-17",
"Statement": [{
"Effect": "Allow",
"Resource": "*",
"Action": [
"cloudwatch:*",
"dynamodb:*",
"ec2:Describe*",
"elasticmapreduce:Describe*",
"elasticmapreduce:ListBootstrapActions",
"elasticmapreduce:ListClusters",
"elasticmapreduce:ListInstanceGroups",
"elasticmapreduce:ListInstances",
"elasticmapreduce:ListSteps",
"kinesis:CreateStream",
"kinesis:DeleteStream",
"kinesis:DescribeStream",
"kinesis:GetRecords",
"kinesis:GetShardIterator",
"kinesis:MergeShards",
"kinesis:PutRecord",
"kinesis:SplitShard",
"rds:Describe*",
"s3:*",
"sdb:*",
"sns:*",
"sqs:*"
]
}]
}
Create a Cluster Resource
new Cluster(name: string, args: ClusterArgs, opts?: CustomResourceOptions);@overload
def Cluster(resource_name: str,
opts: Optional[ResourceOptions] = None,
additional_info: Optional[str] = None,
applications: Optional[Sequence[str]] = None,
auto_termination_policy: Optional[ClusterAutoTerminationPolicyArgs] = None,
autoscaling_role: Optional[str] = None,
bootstrap_actions: Optional[Sequence[ClusterBootstrapActionArgs]] = None,
configurations: Optional[str] = None,
configurations_json: Optional[str] = None,
core_instance_fleet: Optional[ClusterCoreInstanceFleetArgs] = None,
core_instance_group: Optional[ClusterCoreInstanceGroupArgs] = None,
custom_ami_id: Optional[str] = None,
ebs_root_volume_size: Optional[int] = None,
ec2_attributes: Optional[ClusterEc2AttributesArgs] = None,
keep_job_flow_alive_when_no_steps: Optional[bool] = None,
kerberos_attributes: Optional[ClusterKerberosAttributesArgs] = None,
list_steps_states: Optional[Sequence[str]] = None,
log_encryption_kms_key_id: Optional[str] = None,
log_uri: Optional[str] = None,
master_instance_fleet: Optional[ClusterMasterInstanceFleetArgs] = None,
master_instance_group: Optional[ClusterMasterInstanceGroupArgs] = None,
name: Optional[str] = None,
release_label: Optional[str] = None,
scale_down_behavior: Optional[str] = None,
security_configuration: Optional[str] = None,
service_role: Optional[str] = None,
step_concurrency_level: Optional[int] = None,
steps: Optional[Sequence[ClusterStepArgs]] = None,
tags: Optional[Mapping[str, str]] = None,
termination_protection: Optional[bool] = None,
visible_to_all_users: Optional[bool] = None)
@overload
def Cluster(resource_name: str,
args: ClusterArgs,
opts: Optional[ResourceOptions] = None)func NewCluster(ctx *Context, name string, args ClusterArgs, opts ...ResourceOption) (*Cluster, error)public Cluster(string name, ClusterArgs args, CustomResourceOptions? opts = null)
public Cluster(String name, ClusterArgs args)
public Cluster(String name, ClusterArgs args, CustomResourceOptions options)
type: aws:emr:Cluster
properties: # The arguments to resource properties.
options: # Bag of options to control resource's behavior.
- name string
- The unique name of the resource.
- args ClusterArgs
- The arguments to resource properties.
- opts CustomResourceOptions
- Bag of options to control resource's behavior.
- resource_name str
- The unique name of the resource.
- args ClusterArgs
- The arguments to resource properties.
- opts ResourceOptions
- Bag of options to control resource's behavior.
- ctx Context
- Context object for the current deployment.
- name string
- The unique name of the resource.
- args ClusterArgs
- The arguments to resource properties.
- opts ResourceOption
- Bag of options to control resource's behavior.
- name string
- The unique name of the resource.
- args ClusterArgs
- The arguments to resource properties.
- opts CustomResourceOptions
- Bag of options to control resource's behavior.
- name String
- The unique name of the resource.
- args ClusterArgs
- The arguments to resource properties.
- options CustomResourceOptions
- Bag of options to control resource's behavior.
Cluster Resource Properties
To learn more about resource properties and how to use them, see Inputs and Outputs in the Architecture and Concepts docs.
Inputs
The Cluster resource accepts the following input properties:
- Release
Label string Release label for the Amazon EMR release.
- Service
Role string IAM role that will be assumed by the Amazon EMR service to access AWS resources.
- Additional
Info string - Applications List<string>
A case-insensitive list of applications for Amazon EMR to install and configure when launching the cluster. For a list of applications available for each Amazon EMR release version, see the Amazon EMR Release Guide.
- Auto
Termination ClusterPolicy Auto Termination Policy Args An auto-termination policy for an Amazon EMR cluster. An auto-termination policy defines the amount of idle time in seconds after which a cluster automatically terminates. See Auto Termination Policy Below.
- Autoscaling
Role string IAM role for automatic scaling policies. The IAM role provides permissions that the automatic scaling feature requires to launch and terminate EC2 instances in an instance group.
- Bootstrap
Actions List<ClusterBootstrap Action Args> Ordered list of bootstrap actions that will be run before Hadoop is started on the cluster nodes. See below.
- Configurations string
Configuration classification that applies when provisioning cluster instances, which can include configurations for applications and software that run on the cluster. List of
configurationblocks.- Configurations
Json string JSON string for supplying list of configurations for the EMR cluster.
- Core
Instance ClusterFleet Core Instance Fleet Args Configuration block to use an Instance Fleet for the core node type. Cannot be specified if any
core_instance_groupconfiguration blocks are set. Detailed below.- Core
Instance ClusterGroup Core Instance Group Args Configuration block to use an Instance Group for the core node type.
- Custom
Ami stringId Custom Amazon Linux AMI for the cluster (instead of an EMR-owned AMI). Available in Amazon EMR version 5.7.0 and later.
- Ebs
Root intVolume Size Size in GiB of the EBS root device volume of the Linux AMI that is used for each EC2 instance. Available in Amazon EMR version 4.x and later.
- Ec2Attributes
Cluster
Ec2Attributes Args Attributes for the EC2 instances running the job flow. See below.
- Keep
Job boolFlow Alive When No Steps Switch on/off run cluster with no steps or when all steps are complete (default is on)
- Kerberos
Attributes ClusterKerberos Attributes Args Kerberos configuration for the cluster. See below.
- List
Steps List<string>States List of step states used to filter returned steps
- Log
Encryption stringKms Key Id AWS KMS customer master key (CMK) key ID or arn used for encrypting log files. This attribute is only available with EMR version 5.30.0 and later, excluding EMR 6.0.0.
- Log
Uri string S3 bucket to write the log files of the job flow. If a value is not provided, logs are not created.
- Master
Instance ClusterFleet Master Instance Fleet Args Configuration block to use an Instance Fleet for the master node type. Cannot be specified if any
master_instance_groupconfiguration blocks are set. Detailed below.- Master
Instance ClusterGroup Master Instance Group Args Configuration block to use an Instance Group for the master node type.
- Name string
Name of the step.
- Scale
Down stringBehavior Way that individual Amazon EC2 instances terminate when an automatic scale-in activity occurs or an
instance groupis resized.- Security
Configuration string Security configuration name to attach to the EMR cluster. Only valid for EMR clusters with
release_label4.8.0 or greater.- Step
Concurrency intLevel Number of steps that can be executed concurrently. You can specify a maximum of 256 steps. Only valid for EMR clusters with
release_label5.28.0 or greater (default is 1).- Steps
List<Cluster
Step Args> List of steps to run when creating the cluster. See below. It is highly recommended to utilize the lifecycle resource options block with
ignoreChangesif other steps are being managed outside of this provider.- Dictionary<string, string>
list of tags to apply to the EMR Cluster. If configured with a provider
default_tagsconfiguration block present, tags with matching keys will overwrite those defined at the provider-level.- Termination
Protection bool Switch on/off termination protection (default is
false, except when using multiple master nodes). Before attempting to destroy the resource when termination protection is enabled, this configuration must be applied with its value set tofalse.- Visible
To boolAll Users Whether the job flow is visible to all IAM users of the AWS account associated with the job flow. Default value is
true.
- Release
Label string Release label for the Amazon EMR release.
- Service
Role string IAM role that will be assumed by the Amazon EMR service to access AWS resources.
- Additional
Info string - Applications []string
A case-insensitive list of applications for Amazon EMR to install and configure when launching the cluster. For a list of applications available for each Amazon EMR release version, see the Amazon EMR Release Guide.
- Auto
Termination ClusterPolicy Auto Termination Policy Args An auto-termination policy for an Amazon EMR cluster. An auto-termination policy defines the amount of idle time in seconds after which a cluster automatically terminates. See Auto Termination Policy Below.
- Autoscaling
Role string IAM role for automatic scaling policies. The IAM role provides permissions that the automatic scaling feature requires to launch and terminate EC2 instances in an instance group.
- Bootstrap
Actions []ClusterBootstrap Action Args Ordered list of bootstrap actions that will be run before Hadoop is started on the cluster nodes. See below.
- Configurations string
Configuration classification that applies when provisioning cluster instances, which can include configurations for applications and software that run on the cluster. List of
configurationblocks.- Configurations
Json string JSON string for supplying list of configurations for the EMR cluster.
- Core
Instance ClusterFleet Core Instance Fleet Args Configuration block to use an Instance Fleet for the core node type. Cannot be specified if any
core_instance_groupconfiguration blocks are set. Detailed below.- Core
Instance ClusterGroup Core Instance Group Args Configuration block to use an Instance Group for the core node type.
- Custom
Ami stringId Custom Amazon Linux AMI for the cluster (instead of an EMR-owned AMI). Available in Amazon EMR version 5.7.0 and later.
- Ebs
Root intVolume Size Size in GiB of the EBS root device volume of the Linux AMI that is used for each EC2 instance. Available in Amazon EMR version 4.x and later.
- Ec2Attributes
Cluster
Ec2Attributes Args Attributes for the EC2 instances running the job flow. See below.
- Keep
Job boolFlow Alive When No Steps Switch on/off run cluster with no steps or when all steps are complete (default is on)
- Kerberos
Attributes ClusterKerberos Attributes Args Kerberos configuration for the cluster. See below.
- List
Steps []stringStates List of step states used to filter returned steps
- Log
Encryption stringKms Key Id AWS KMS customer master key (CMK) key ID or arn used for encrypting log files. This attribute is only available with EMR version 5.30.0 and later, excluding EMR 6.0.0.
- Log
Uri string S3 bucket to write the log files of the job flow. If a value is not provided, logs are not created.
- Master
Instance ClusterFleet Master Instance Fleet Args Configuration block to use an Instance Fleet for the master node type. Cannot be specified if any
master_instance_groupconfiguration blocks are set. Detailed below.- Master
Instance ClusterGroup Master Instance Group Args Configuration block to use an Instance Group for the master node type.
- Name string
Name of the step.
- Scale
Down stringBehavior Way that individual Amazon EC2 instances terminate when an automatic scale-in activity occurs or an
instance groupis resized.- Security
Configuration string Security configuration name to attach to the EMR cluster. Only valid for EMR clusters with
release_label4.8.0 or greater.- Step
Concurrency intLevel Number of steps that can be executed concurrently. You can specify a maximum of 256 steps. Only valid for EMR clusters with
release_label5.28.0 or greater (default is 1).- Steps
[]Cluster
Step Args List of steps to run when creating the cluster. See below. It is highly recommended to utilize the lifecycle resource options block with
ignoreChangesif other steps are being managed outside of this provider.- map[string]string
list of tags to apply to the EMR Cluster. If configured with a provider
default_tagsconfiguration block present, tags with matching keys will overwrite those defined at the provider-level.- Termination
Protection bool Switch on/off termination protection (default is
false, except when using multiple master nodes). Before attempting to destroy the resource when termination protection is enabled, this configuration must be applied with its value set tofalse.- Visible
To boolAll Users Whether the job flow is visible to all IAM users of the AWS account associated with the job flow. Default value is
true.
- release
Label String Release label for the Amazon EMR release.
- service
Role String IAM role that will be assumed by the Amazon EMR service to access AWS resources.
- additional
Info String - applications List<String>
A case-insensitive list of applications for Amazon EMR to install and configure when launching the cluster. For a list of applications available for each Amazon EMR release version, see the Amazon EMR Release Guide.
- auto
Termination ClusterPolicy Auto Termination Policy Args An auto-termination policy for an Amazon EMR cluster. An auto-termination policy defines the amount of idle time in seconds after which a cluster automatically terminates. See Auto Termination Policy Below.
- autoscaling
Role String IAM role for automatic scaling policies. The IAM role provides permissions that the automatic scaling feature requires to launch and terminate EC2 instances in an instance group.
- bootstrap
Actions List<ClusterBootstrap Action Args> Ordered list of bootstrap actions that will be run before Hadoop is started on the cluster nodes. See below.
- configurations String
Configuration classification that applies when provisioning cluster instances, which can include configurations for applications and software that run on the cluster. List of
configurationblocks.- configurations
Json String JSON string for supplying list of configurations for the EMR cluster.
- core
Instance ClusterFleet Core Instance Fleet Args Configuration block to use an Instance Fleet for the core node type. Cannot be specified if any
core_instance_groupconfiguration blocks are set. Detailed below.- core
Instance ClusterGroup Core Instance Group Args Configuration block to use an Instance Group for the core node type.
- custom
Ami StringId Custom Amazon Linux AMI for the cluster (instead of an EMR-owned AMI). Available in Amazon EMR version 5.7.0 and later.
- ebs
Root IntegerVolume Size Size in GiB of the EBS root device volume of the Linux AMI that is used for each EC2 instance. Available in Amazon EMR version 4.x and later.
- ec2Attributes
Cluster
Ec2Attributes Args Attributes for the EC2 instances running the job flow. See below.
- keep
Job BooleanFlow Alive When No Steps Switch on/off run cluster with no steps or when all steps are complete (default is on)
- kerberos
Attributes ClusterKerberos Attributes Args Kerberos configuration for the cluster. See below.
- list
Steps List<String>States List of step states used to filter returned steps
- log
Encryption StringKms Key Id AWS KMS customer master key (CMK) key ID or arn used for encrypting log files. This attribute is only available with EMR version 5.30.0 and later, excluding EMR 6.0.0.
- log
Uri String S3 bucket to write the log files of the job flow. If a value is not provided, logs are not created.
- master
Instance ClusterFleet Master Instance Fleet Args Configuration block to use an Instance Fleet for the master node type. Cannot be specified if any
master_instance_groupconfiguration blocks are set. Detailed below.- master
Instance ClusterGroup Master Instance Group Args Configuration block to use an Instance Group for the master node type.
- name String
Name of the step.
- scale
Down StringBehavior Way that individual Amazon EC2 instances terminate when an automatic scale-in activity occurs or an
instance groupis resized.- security
Configuration String Security configuration name to attach to the EMR cluster. Only valid for EMR clusters with
release_label4.8.0 or greater.- step
Concurrency IntegerLevel Number of steps that can be executed concurrently. You can specify a maximum of 256 steps. Only valid for EMR clusters with
release_label5.28.0 or greater (default is 1).- steps
List<Cluster
Step Args> List of steps to run when creating the cluster. See below. It is highly recommended to utilize the lifecycle resource options block with
ignoreChangesif other steps are being managed outside of this provider.- Map<String,String>
list of tags to apply to the EMR Cluster. If configured with a provider
default_tagsconfiguration block present, tags with matching keys will overwrite those defined at the provider-level.- termination
Protection Boolean Switch on/off termination protection (default is
false, except when using multiple master nodes). Before attempting to destroy the resource when termination protection is enabled, this configuration must be applied with its value set tofalse.- visible
To BooleanAll Users Whether the job flow is visible to all IAM users of the AWS account associated with the job flow. Default value is
true.
- release
Label string Release label for the Amazon EMR release.
- service
Role string IAM role that will be assumed by the Amazon EMR service to access AWS resources.
- additional
Info string - applications string[]
A case-insensitive list of applications for Amazon EMR to install and configure when launching the cluster. For a list of applications available for each Amazon EMR release version, see the Amazon EMR Release Guide.
- auto
Termination ClusterPolicy Auto Termination Policy Args An auto-termination policy for an Amazon EMR cluster. An auto-termination policy defines the amount of idle time in seconds after which a cluster automatically terminates. See Auto Termination Policy Below.
- autoscaling
Role string IAM role for automatic scaling policies. The IAM role provides permissions that the automatic scaling feature requires to launch and terminate EC2 instances in an instance group.
- bootstrap
Actions ClusterBootstrap Action Args[] Ordered list of bootstrap actions that will be run before Hadoop is started on the cluster nodes. See below.
- configurations string
Configuration classification that applies when provisioning cluster instances, which can include configurations for applications and software that run on the cluster. List of
configurationblocks.- configurations
Json string JSON string for supplying list of configurations for the EMR cluster.
- core
Instance ClusterFleet Core Instance Fleet Args Configuration block to use an Instance Fleet for the core node type. Cannot be specified if any
core_instance_groupconfiguration blocks are set. Detailed below.- core
Instance ClusterGroup Core Instance Group Args Configuration block to use an Instance Group for the core node type.
- custom
Ami stringId Custom Amazon Linux AMI for the cluster (instead of an EMR-owned AMI). Available in Amazon EMR version 5.7.0 and later.
- ebs
Root numberVolume Size Size in GiB of the EBS root device volume of the Linux AMI that is used for each EC2 instance. Available in Amazon EMR version 4.x and later.
- ec2Attributes
Cluster
Ec2Attributes Args Attributes for the EC2 instances running the job flow. See below.
- keep
Job booleanFlow Alive When No Steps Switch on/off run cluster with no steps or when all steps are complete (default is on)
- kerberos
Attributes ClusterKerberos Attributes Args Kerberos configuration for the cluster. See below.
- list
Steps string[]States List of step states used to filter returned steps
- log
Encryption stringKms Key Id AWS KMS customer master key (CMK) key ID or arn used for encrypting log files. This attribute is only available with EMR version 5.30.0 and later, excluding EMR 6.0.0.
- log
Uri string S3 bucket to write the log files of the job flow. If a value is not provided, logs are not created.
- master
Instance ClusterFleet Master Instance Fleet Args Configuration block to use an Instance Fleet for the master node type. Cannot be specified if any
master_instance_groupconfiguration blocks are set. Detailed below.- master
Instance ClusterGroup Master Instance Group Args Configuration block to use an Instance Group for the master node type.
- name string
Name of the step.
- scale
Down stringBehavior Way that individual Amazon EC2 instances terminate when an automatic scale-in activity occurs or an
instance groupis resized.- security
Configuration string Security configuration name to attach to the EMR cluster. Only valid for EMR clusters with
release_label4.8.0 or greater.- step
Concurrency numberLevel Number of steps that can be executed concurrently. You can specify a maximum of 256 steps. Only valid for EMR clusters with
release_label5.28.0 or greater (default is 1).- steps
Cluster
Step Args[] List of steps to run when creating the cluster. See below. It is highly recommended to utilize the lifecycle resource options block with
ignoreChangesif other steps are being managed outside of this provider.- {[key: string]: string}
list of tags to apply to the EMR Cluster. If configured with a provider
default_tagsconfiguration block present, tags with matching keys will overwrite those defined at the provider-level.- termination
Protection boolean Switch on/off termination protection (default is
false, except when using multiple master nodes). Before attempting to destroy the resource when termination protection is enabled, this configuration must be applied with its value set tofalse.- visible
To booleanAll Users Whether the job flow is visible to all IAM users of the AWS account associated with the job flow. Default value is
true.
- release_
label str Release label for the Amazon EMR release.
- service_
role str IAM role that will be assumed by the Amazon EMR service to access AWS resources.
- additional_
info str - applications Sequence[str]
A case-insensitive list of applications for Amazon EMR to install and configure when launching the cluster. For a list of applications available for each Amazon EMR release version, see the Amazon EMR Release Guide.
- auto_
termination_ Clusterpolicy Auto Termination Policy Args An auto-termination policy for an Amazon EMR cluster. An auto-termination policy defines the amount of idle time in seconds after which a cluster automatically terminates. See Auto Termination Policy Below.
- autoscaling_
role str IAM role for automatic scaling policies. The IAM role provides permissions that the automatic scaling feature requires to launch and terminate EC2 instances in an instance group.
- bootstrap_
actions Sequence[ClusterBootstrap Action Args] Ordered list of bootstrap actions that will be run before Hadoop is started on the cluster nodes. See below.
- configurations str
Configuration classification that applies when provisioning cluster instances, which can include configurations for applications and software that run on the cluster. List of
configurationblocks.- configurations_
json str JSON string for supplying list of configurations for the EMR cluster.
- core_
instance_ Clusterfleet Core Instance Fleet Args Configuration block to use an Instance Fleet for the core node type. Cannot be specified if any
core_instance_groupconfiguration blocks are set. Detailed below.- core_
instance_ Clustergroup Core Instance Group Args Configuration block to use an Instance Group for the core node type.
- custom_
ami_ strid Custom Amazon Linux AMI for the cluster (instead of an EMR-owned AMI). Available in Amazon EMR version 5.7.0 and later.
- ebs_
root_ intvolume_ size Size in GiB of the EBS root device volume of the Linux AMI that is used for each EC2 instance. Available in Amazon EMR version 4.x and later.
- ec2_
attributes ClusterEc2Attributes Args Attributes for the EC2 instances running the job flow. See below.
- keep_
job_ boolflow_ alive_ when_ no_ steps Switch on/off run cluster with no steps or when all steps are complete (default is on)
- kerberos_
attributes ClusterKerberos Attributes Args Kerberos configuration for the cluster. See below.
- list_
steps_ Sequence[str]states List of step states used to filter returned steps
- log_
encryption_ strkms_ key_ id AWS KMS customer master key (CMK) key ID or arn used for encrypting log files. This attribute is only available with EMR version 5.30.0 and later, excluding EMR 6.0.0.
- log_
uri str S3 bucket to write the log files of the job flow. If a value is not provided, logs are not created.
- master_
instance_ Clusterfleet Master Instance Fleet Args Configuration block to use an Instance Fleet for the master node type. Cannot be specified if any
master_instance_groupconfiguration blocks are set. Detailed below.- master_
instance_ Clustergroup Master Instance Group Args Configuration block to use an Instance Group for the master node type.
- name str
Name of the step.
- scale_
down_ strbehavior Way that individual Amazon EC2 instances terminate when an automatic scale-in activity occurs or an
instance groupis resized.- security_
configuration str Security configuration name to attach to the EMR cluster. Only valid for EMR clusters with
release_label4.8.0 or greater.- step_
concurrency_ intlevel Number of steps that can be executed concurrently. You can specify a maximum of 256 steps. Only valid for EMR clusters with
release_label5.28.0 or greater (default is 1).- steps
Sequence[Cluster
Step Args] List of steps to run when creating the cluster. See below. It is highly recommended to utilize the lifecycle resource options block with
ignoreChangesif other steps are being managed outside of this provider.- Mapping[str, str]
list of tags to apply to the EMR Cluster. If configured with a provider
default_tagsconfiguration block present, tags with matching keys will overwrite those defined at the provider-level.- termination_
protection bool Switch on/off termination protection (default is
false, except when using multiple master nodes). Before attempting to destroy the resource when termination protection is enabled, this configuration must be applied with its value set tofalse.- visible_
to_ boolall_ users Whether the job flow is visible to all IAM users of the AWS account associated with the job flow. Default value is
true.
- release
Label String Release label for the Amazon EMR release.
- service
Role String IAM role that will be assumed by the Amazon EMR service to access AWS resources.
- additional
Info String - applications List<String>
A case-insensitive list of applications for Amazon EMR to install and configure when launching the cluster. For a list of applications available for each Amazon EMR release version, see the Amazon EMR Release Guide.
- auto
Termination Property MapPolicy An auto-termination policy for an Amazon EMR cluster. An auto-termination policy defines the amount of idle time in seconds after which a cluster automatically terminates. See Auto Termination Policy Below.
- autoscaling
Role String IAM role for automatic scaling policies. The IAM role provides permissions that the automatic scaling feature requires to launch and terminate EC2 instances in an instance group.
- bootstrap
Actions List<Property Map> Ordered list of bootstrap actions that will be run before Hadoop is started on the cluster nodes. See below.
- configurations String
Configuration classification that applies when provisioning cluster instances, which can include configurations for applications and software that run on the cluster. List of
configurationblocks.- configurations
Json String JSON string for supplying list of configurations for the EMR cluster.
- core
Instance Property MapFleet Configuration block to use an Instance Fleet for the core node type. Cannot be specified if any
core_instance_groupconfiguration blocks are set. Detailed below.- core
Instance Property MapGroup Configuration block to use an Instance Group for the core node type.
- custom
Ami StringId Custom Amazon Linux AMI for the cluster (instead of an EMR-owned AMI). Available in Amazon EMR version 5.7.0 and later.
- ebs
Root NumberVolume Size Size in GiB of the EBS root device volume of the Linux AMI that is used for each EC2 instance. Available in Amazon EMR version 4.x and later.
- ec2Attributes Property Map
Attributes for the EC2 instances running the job flow. See below.
- keep
Job BooleanFlow Alive When No Steps Switch on/off run cluster with no steps or when all steps are complete (default is on)
- kerberos
Attributes Property Map Kerberos configuration for the cluster. See below.
- list
Steps List<String>States List of step states used to filter returned steps
- log
Encryption StringKms Key Id AWS KMS customer master key (CMK) key ID or arn used for encrypting log files. This attribute is only available with EMR version 5.30.0 and later, excluding EMR 6.0.0.
- log
Uri String S3 bucket to write the log files of the job flow. If a value is not provided, logs are not created.
- master
Instance Property MapFleet Configuration block to use an Instance Fleet for the master node type. Cannot be specified if any
master_instance_groupconfiguration blocks are set. Detailed below.- master
Instance Property MapGroup Configuration block to use an Instance Group for the master node type.
- name String
Name of the step.
- scale
Down StringBehavior Way that individual Amazon EC2 instances terminate when an automatic scale-in activity occurs or an
instance groupis resized.- security
Configuration String Security configuration name to attach to the EMR cluster. Only valid for EMR clusters with
release_label4.8.0 or greater.- step
Concurrency NumberLevel Number of steps that can be executed concurrently. You can specify a maximum of 256 steps. Only valid for EMR clusters with
release_label5.28.0 or greater (default is 1).- steps List<Property Map>
List of steps to run when creating the cluster. See below. It is highly recommended to utilize the lifecycle resource options block with
ignoreChangesif other steps are being managed outside of this provider.- Map<String>
list of tags to apply to the EMR Cluster. If configured with a provider
default_tagsconfiguration block present, tags with matching keys will overwrite those defined at the provider-level.- termination
Protection Boolean Switch on/off termination protection (default is
false, except when using multiple master nodes). Before attempting to destroy the resource when termination protection is enabled, this configuration must be applied with its value set tofalse.- visible
To BooleanAll Users Whether the job flow is visible to all IAM users of the AWS account associated with the job flow. Default value is
true.
Outputs
All input properties are implicitly available as output properties. Additionally, the Cluster resource produces the following output properties:
- Arn string
- Id string
The provider-assigned unique ID for this managed resource.
- Master
Public stringDns The DNS name of the master node. If the cluster is on a private subnet, this is the private DNS name. On a public subnet, this is the public DNS name.
- State string
- Dictionary<string, string>
Map of tags assigned to the resource, including those inherited from the provider
default_tagsconfiguration block.
- Arn string
- Cluster
State string - Id string
The provider-assigned unique ID for this managed resource.
- Master
Public stringDns The DNS name of the master node. If the cluster is on a private subnet, this is the private DNS name. On a public subnet, this is the public DNS name.
- map[string]string
Map of tags assigned to the resource, including those inherited from the provider
default_tagsconfiguration block.
- arn String
- cluster
State String - id String
The provider-assigned unique ID for this managed resource.
- master
Public StringDns The DNS name of the master node. If the cluster is on a private subnet, this is the private DNS name. On a public subnet, this is the public DNS name.
- Map<String,String>
Map of tags assigned to the resource, including those inherited from the provider
default_tagsconfiguration block.
- arn string
- cluster
State string - id string
The provider-assigned unique ID for this managed resource.
- master
Public stringDns The DNS name of the master node. If the cluster is on a private subnet, this is the private DNS name. On a public subnet, this is the public DNS name.
- {[key: string]: string}
Map of tags assigned to the resource, including those inherited from the provider
default_tagsconfiguration block.
- arn str
- cluster_
state str - id str
The provider-assigned unique ID for this managed resource.
- master_
public_ strdns The DNS name of the master node. If the cluster is on a private subnet, this is the private DNS name. On a public subnet, this is the public DNS name.
- Mapping[str, str]
Map of tags assigned to the resource, including those inherited from the provider
default_tagsconfiguration block.
- arn String
- cluster
State String - id String
The provider-assigned unique ID for this managed resource.
- master
Public StringDns The DNS name of the master node. If the cluster is on a private subnet, this is the private DNS name. On a public subnet, this is the public DNS name.
- Map<String>
Map of tags assigned to the resource, including those inherited from the provider
default_tagsconfiguration block.
Look up an Existing Cluster Resource
Get an existing Cluster resource’s state with the given name, ID, and optional extra properties used to qualify the lookup.
public static get(name: string, id: Input<ID>, state?: ClusterState, opts?: CustomResourceOptions): Cluster@staticmethod
def get(resource_name: str,
id: str,
opts: Optional[ResourceOptions] = None,
additional_info: Optional[str] = None,
applications: Optional[Sequence[str]] = None,
arn: Optional[str] = None,
auto_termination_policy: Optional[ClusterAutoTerminationPolicyArgs] = None,
autoscaling_role: Optional[str] = None,
bootstrap_actions: Optional[Sequence[ClusterBootstrapActionArgs]] = None,
cluster_state: Optional[str] = None,
configurations: Optional[str] = None,
configurations_json: Optional[str] = None,
core_instance_fleet: Optional[ClusterCoreInstanceFleetArgs] = None,
core_instance_group: Optional[ClusterCoreInstanceGroupArgs] = None,
custom_ami_id: Optional[str] = None,
ebs_root_volume_size: Optional[int] = None,
ec2_attributes: Optional[ClusterEc2AttributesArgs] = None,
keep_job_flow_alive_when_no_steps: Optional[bool] = None,
kerberos_attributes: Optional[ClusterKerberosAttributesArgs] = None,
list_steps_states: Optional[Sequence[str]] = None,
log_encryption_kms_key_id: Optional[str] = None,
log_uri: Optional[str] = None,
master_instance_fleet: Optional[ClusterMasterInstanceFleetArgs] = None,
master_instance_group: Optional[ClusterMasterInstanceGroupArgs] = None,
master_public_dns: Optional[str] = None,
name: Optional[str] = None,
release_label: Optional[str] = None,
scale_down_behavior: Optional[str] = None,
security_configuration: Optional[str] = None,
service_role: Optional[str] = None,
step_concurrency_level: Optional[int] = None,
steps: Optional[Sequence[ClusterStepArgs]] = None,
tags: Optional[Mapping[str, str]] = None,
tags_all: Optional[Mapping[str, str]] = None,
termination_protection: Optional[bool] = None,
visible_to_all_users: Optional[bool] = None) -> Clusterfunc GetCluster(ctx *Context, name string, id IDInput, state *ClusterState, opts ...ResourceOption) (*Cluster, error)public static Cluster Get(string name, Input<string> id, ClusterState? state, CustomResourceOptions? opts = null)public static Cluster get(String name, Output<String> id, ClusterState state, CustomResourceOptions options)Resource lookup is not supported in YAML- name
- The unique name of the resulting resource.
- id
- The unique provider ID of the resource to lookup.
- state
- Any extra arguments used during the lookup.
- opts
- A bag of options that control this resource's behavior.
- resource_name
- The unique name of the resulting resource.
- id
- The unique provider ID of the resource to lookup.
- name
- The unique name of the resulting resource.
- id
- The unique provider ID of the resource to lookup.
- state
- Any extra arguments used during the lookup.
- opts
- A bag of options that control this resource's behavior.
- name
- The unique name of the resulting resource.
- id
- The unique provider ID of the resource to lookup.
- state
- Any extra arguments used during the lookup.
- opts
- A bag of options that control this resource's behavior.
- name
- The unique name of the resulting resource.
- id
- The unique provider ID of the resource to lookup.
- state
- Any extra arguments used during the lookup.
- opts
- A bag of options that control this resource's behavior.
- Additional
Info string - Applications List<string>
A case-insensitive list of applications for Amazon EMR to install and configure when launching the cluster. For a list of applications available for each Amazon EMR release version, see the Amazon EMR Release Guide.
- Arn string
- Auto
Termination ClusterPolicy Auto Termination Policy Args An auto-termination policy for an Amazon EMR cluster. An auto-termination policy defines the amount of idle time in seconds after which a cluster automatically terminates. See Auto Termination Policy Below.
- Autoscaling
Role string IAM role for automatic scaling policies. The IAM role provides permissions that the automatic scaling feature requires to launch and terminate EC2 instances in an instance group.
- Bootstrap
Actions List<ClusterBootstrap Action Args> Ordered list of bootstrap actions that will be run before Hadoop is started on the cluster nodes. See below.
- Configurations string
Configuration classification that applies when provisioning cluster instances, which can include configurations for applications and software that run on the cluster. List of
configurationblocks.- Configurations
Json string JSON string for supplying list of configurations for the EMR cluster.
- Core
Instance ClusterFleet Core Instance Fleet Args Configuration block to use an Instance Fleet for the core node type. Cannot be specified if any
core_instance_groupconfiguration blocks are set. Detailed below.- Core
Instance ClusterGroup Core Instance Group Args Configuration block to use an Instance Group for the core node type.
- Custom
Ami stringId Custom Amazon Linux AMI for the cluster (instead of an EMR-owned AMI). Available in Amazon EMR version 5.7.0 and later.
- Ebs
Root intVolume Size Size in GiB of the EBS root device volume of the Linux AMI that is used for each EC2 instance. Available in Amazon EMR version 4.x and later.
- Ec2Attributes
Cluster
Ec2Attributes Args Attributes for the EC2 instances running the job flow. See below.
- Keep
Job boolFlow Alive When No Steps Switch on/off run cluster with no steps or when all steps are complete (default is on)
- Kerberos
Attributes ClusterKerberos Attributes Args Kerberos configuration for the cluster. See below.
- List
Steps List<string>States List of step states used to filter returned steps
- Log
Encryption stringKms Key Id AWS KMS customer master key (CMK) key ID or arn used for encrypting log files. This attribute is only available with EMR version 5.30.0 and later, excluding EMR 6.0.0.
- Log
Uri string S3 bucket to write the log files of the job flow. If a value is not provided, logs are not created.
- Master
Instance ClusterFleet Master Instance Fleet Args Configuration block to use an Instance Fleet for the master node type. Cannot be specified if any
master_instance_groupconfiguration blocks are set. Detailed below.- Master
Instance ClusterGroup Master Instance Group Args Configuration block to use an Instance Group for the master node type.
- Master
Public stringDns The DNS name of the master node. If the cluster is on a private subnet, this is the private DNS name. On a public subnet, this is the public DNS name.
- Name string
Name of the step.
- Release
Label string Release label for the Amazon EMR release.
- Scale
Down stringBehavior Way that individual Amazon EC2 instances terminate when an automatic scale-in activity occurs or an
instance groupis resized.- Security
Configuration string Security configuration name to attach to the EMR cluster. Only valid for EMR clusters with
release_label4.8.0 or greater.- Service
Role string IAM role that will be assumed by the Amazon EMR service to access AWS resources.
- State string
- Step
Concurrency intLevel Number of steps that can be executed concurrently. You can specify a maximum of 256 steps. Only valid for EMR clusters with
release_label5.28.0 or greater (default is 1).- Steps
List<Cluster
Step Args> List of steps to run when creating the cluster. See below. It is highly recommended to utilize the lifecycle resource options block with
ignoreChangesif other steps are being managed outside of this provider.- Dictionary<string, string>
list of tags to apply to the EMR Cluster. If configured with a provider
default_tagsconfiguration block present, tags with matching keys will overwrite those defined at the provider-level.- Dictionary<string, string>
Map of tags assigned to the resource, including those inherited from the provider
default_tagsconfiguration block.- Termination
Protection bool Switch on/off termination protection (default is
false, except when using multiple master nodes). Before attempting to destroy the resource when termination protection is enabled, this configuration must be applied with its value set tofalse.- Visible
To boolAll Users Whether the job flow is visible to all IAM users of the AWS account associated with the job flow. Default value is
true.
- Additional
Info string - Applications []string
A case-insensitive list of applications for Amazon EMR to install and configure when launching the cluster. For a list of applications available for each Amazon EMR release version, see the Amazon EMR Release Guide.
- Arn string
- Auto
Termination ClusterPolicy Auto Termination Policy Args An auto-termination policy for an Amazon EMR cluster. An auto-termination policy defines the amount of idle time in seconds after which a cluster automatically terminates. See Auto Termination Policy Below.
- Autoscaling
Role string IAM role for automatic scaling policies. The IAM role provides permissions that the automatic scaling feature requires to launch and terminate EC2 instances in an instance group.
- Bootstrap
Actions []ClusterBootstrap Action Args Ordered list of bootstrap actions that will be run before Hadoop is started on the cluster nodes. See below.
- Cluster
State string - Configurations string
Configuration classification that applies when provisioning cluster instances, which can include configurations for applications and software that run on the cluster. List of
configurationblocks.- Configurations
Json string JSON string for supplying list of configurations for the EMR cluster.
- Core
Instance ClusterFleet Core Instance Fleet Args Configuration block to use an Instance Fleet for the core node type. Cannot be specified if any
core_instance_groupconfiguration blocks are set. Detailed below.- Core
Instance ClusterGroup Core Instance Group Args Configuration block to use an Instance Group for the core node type.
- Custom
Ami stringId Custom Amazon Linux AMI for the cluster (instead of an EMR-owned AMI). Available in Amazon EMR version 5.7.0 and later.
- Ebs
Root intVolume Size Size in GiB of the EBS root device volume of the Linux AMI that is used for each EC2 instance. Available in Amazon EMR version 4.x and later.
- Ec2Attributes
Cluster
Ec2Attributes Args Attributes for the EC2 instances running the job flow. See below.
- Keep
Job boolFlow Alive When No Steps Switch on/off run cluster with no steps or when all steps are complete (default is on)
- Kerberos
Attributes ClusterKerberos Attributes Args Kerberos configuration for the cluster. See below.
- List
Steps []stringStates List of step states used to filter returned steps
- Log
Encryption stringKms Key Id AWS KMS customer master key (CMK) key ID or arn used for encrypting log files. This attribute is only available with EMR version 5.30.0 and later, excluding EMR 6.0.0.
- Log
Uri string S3 bucket to write the log files of the job flow. If a value is not provided, logs are not created.
- Master
Instance ClusterFleet Master Instance Fleet Args Configuration block to use an Instance Fleet for the master node type. Cannot be specified if any
master_instance_groupconfiguration blocks are set. Detailed below.- Master
Instance ClusterGroup Master Instance Group Args Configuration block to use an Instance Group for the master node type.
- Master
Public stringDns The DNS name of the master node. If the cluster is on a private subnet, this is the private DNS name. On a public subnet, this is the public DNS name.
- Name string
Name of the step.
- Release
Label string Release label for the Amazon EMR release.
- Scale
Down stringBehavior Way that individual Amazon EC2 instances terminate when an automatic scale-in activity occurs or an
instance groupis resized.- Security
Configuration string Security configuration name to attach to the EMR cluster. Only valid for EMR clusters with
release_label4.8.0 or greater.- Service
Role string IAM role that will be assumed by the Amazon EMR service to access AWS resources.
- Step
Concurrency intLevel Number of steps that can be executed concurrently. You can specify a maximum of 256 steps. Only valid for EMR clusters with
release_label5.28.0 or greater (default is 1).- Steps
[]Cluster
Step Args List of steps to run when creating the cluster. See below. It is highly recommended to utilize the lifecycle resource options block with
ignoreChangesif other steps are being managed outside of this provider.- map[string]string
list of tags to apply to the EMR Cluster. If configured with a provider
default_tagsconfiguration block present, tags with matching keys will overwrite those defined at the provider-level.- map[string]string
Map of tags assigned to the resource, including those inherited from the provider
default_tagsconfiguration block.- Termination
Protection bool Switch on/off termination protection (default is
false, except when using multiple master nodes). Before attempting to destroy the resource when termination protection is enabled, this configuration must be applied with its value set tofalse.- Visible
To boolAll Users Whether the job flow is visible to all IAM users of the AWS account associated with the job flow. Default value is
true.
- additional
Info String - applications List<String>
A case-insensitive list of applications for Amazon EMR to install and configure when launching the cluster. For a list of applications available for each Amazon EMR release version, see the Amazon EMR Release Guide.
- arn String
- auto
Termination ClusterPolicy Auto Termination Policy Args An auto-termination policy for an Amazon EMR cluster. An auto-termination policy defines the amount of idle time in seconds after which a cluster automatically terminates. See Auto Termination Policy Below.
- autoscaling
Role String IAM role for automatic scaling policies. The IAM role provides permissions that the automatic scaling feature requires to launch and terminate EC2 instances in an instance group.
- bootstrap
Actions List<ClusterBootstrap Action Args> Ordered list of bootstrap actions that will be run before Hadoop is started on the cluster nodes. See below.
- cluster
State String - configurations String
Configuration classification that applies when provisioning cluster instances, which can include configurations for applications and software that run on the cluster. List of
configurationblocks.- configurations
Json String JSON string for supplying list of configurations for the EMR cluster.
- core
Instance ClusterFleet Core Instance Fleet Args Configuration block to use an Instance Fleet for the core node type. Cannot be specified if any
core_instance_groupconfiguration blocks are set. Detailed below.- core
Instance ClusterGroup Core Instance Group Args Configuration block to use an Instance Group for the core node type.
- custom
Ami StringId Custom Amazon Linux AMI for the cluster (instead of an EMR-owned AMI). Available in Amazon EMR version 5.7.0 and later.
- ebs
Root IntegerVolume Size Size in GiB of the EBS root device volume of the Linux AMI that is used for each EC2 instance. Available in Amazon EMR version 4.x and later.
- ec2Attributes
Cluster
Ec2Attributes Args Attributes for the EC2 instances running the job flow. See below.
- keep
Job BooleanFlow Alive When No Steps Switch on/off run cluster with no steps or when all steps are complete (default is on)
- kerberos
Attributes ClusterKerberos Attributes Args Kerberos configuration for the cluster. See below.
- list
Steps List<String>States List of step states used to filter returned steps
- log
Encryption StringKms Key Id AWS KMS customer master key (CMK) key ID or arn used for encrypting log files. This attribute is only available with EMR version 5.30.0 and later, excluding EMR 6.0.0.
- log
Uri String S3 bucket to write the log files of the job flow. If a value is not provided, logs are not created.
- master
Instance ClusterFleet Master Instance Fleet Args Configuration block to use an Instance Fleet for the master node type. Cannot be specified if any
master_instance_groupconfiguration blocks are set. Detailed below.- master
Instance ClusterGroup Master Instance Group Args Configuration block to use an Instance Group for the master node type.
- master
Public StringDns The DNS name of the master node. If the cluster is on a private subnet, this is the private DNS name. On a public subnet, this is the public DNS name.
- name String
Name of the step.
- release
Label String Release label for the Amazon EMR release.
- scale
Down StringBehavior Way that individual Amazon EC2 instances terminate when an automatic scale-in activity occurs or an
instance groupis resized.- security
Configuration String Security configuration name to attach to the EMR cluster. Only valid for EMR clusters with
release_label4.8.0 or greater.- service
Role String IAM role that will be assumed by the Amazon EMR service to access AWS resources.
- step
Concurrency IntegerLevel Number of steps that can be executed concurrently. You can specify a maximum of 256 steps. Only valid for EMR clusters with
release_label5.28.0 or greater (default is 1).- steps
List<Cluster
Step Args> List of steps to run when creating the cluster. See below. It is highly recommended to utilize the lifecycle resource options block with
ignoreChangesif other steps are being managed outside of this provider.- Map<String,String>
list of tags to apply to the EMR Cluster. If configured with a provider
default_tagsconfiguration block present, tags with matching keys will overwrite those defined at the provider-level.- Map<String,String>
Map of tags assigned to the resource, including those inherited from the provider
default_tagsconfiguration block.- termination
Protection Boolean Switch on/off termination protection (default is
false, except when using multiple master nodes). Before attempting to destroy the resource when termination protection is enabled, this configuration must be applied with its value set tofalse.- visible
To BooleanAll Users Whether the job flow is visible to all IAM users of the AWS account associated with the job flow. Default value is
true.
- additional
Info string - applications string[]
A case-insensitive list of applications for Amazon EMR to install and configure when launching the cluster. For a list of applications available for each Amazon EMR release version, see the Amazon EMR Release Guide.
- arn string
- auto
Termination ClusterPolicy Auto Termination Policy Args An auto-termination policy for an Amazon EMR cluster. An auto-termination policy defines the amount of idle time in seconds after which a cluster automatically terminates. See Auto Termination Policy Below.
- autoscaling
Role string IAM role for automatic scaling policies. The IAM role provides permissions that the automatic scaling feature requires to launch and terminate EC2 instances in an instance group.
- bootstrap
Actions ClusterBootstrap Action Args[] Ordered list of bootstrap actions that will be run before Hadoop is started on the cluster nodes. See below.
- cluster
State string - configurations string
Configuration classification that applies when provisioning cluster instances, which can include configurations for applications and software that run on the cluster. List of
configurationblocks.- configurations
Json string JSON string for supplying list of configurations for the EMR cluster.
- core
Instance ClusterFleet Core Instance Fleet Args Configuration block to use an Instance Fleet for the core node type. Cannot be specified if any
core_instance_groupconfiguration blocks are set. Detailed below.- core
Instance ClusterGroup Core Instance Group Args Configuration block to use an Instance Group for the core node type.
- custom
Ami stringId Custom Amazon Linux AMI for the cluster (instead of an EMR-owned AMI). Available in Amazon EMR version 5.7.0 and later.
- ebs
Root numberVolume Size Size in GiB of the EBS root device volume of the Linux AMI that is used for each EC2 instance. Available in Amazon EMR version 4.x and later.
- ec2Attributes
Cluster
Ec2Attributes Args Attributes for the EC2 instances running the job flow. See below.
- keep
Job booleanFlow Alive When No Steps Switch on/off run cluster with no steps or when all steps are complete (default is on)
- kerberos
Attributes ClusterKerberos Attributes Args Kerberos configuration for the cluster. See below.
- list
Steps string[]States List of step states used to filter returned steps
- log
Encryption stringKms Key Id AWS KMS customer master key (CMK) key ID or arn used for encrypting log files. This attribute is only available with EMR version 5.30.0 and later, excluding EMR 6.0.0.
- log
Uri string S3 bucket to write the log files of the job flow. If a value is not provided, logs are not created.
- master
Instance ClusterFleet Master Instance Fleet Args Configuration block to use an Instance Fleet for the master node type. Cannot be specified if any
master_instance_groupconfiguration blocks are set. Detailed below.- master
Instance ClusterGroup Master Instance Group Args Configuration block to use an Instance Group for the master node type.
- master
Public stringDns The DNS name of the master node. If the cluster is on a private subnet, this is the private DNS name. On a public subnet, this is the public DNS name.
- name string
Name of the step.
- release
Label string Release label for the Amazon EMR release.
- scale
Down stringBehavior Way that individual Amazon EC2 instances terminate when an automatic scale-in activity occurs or an
instance groupis resized.- security
Configuration string Security configuration name to attach to the EMR cluster. Only valid for EMR clusters with
release_label4.8.0 or greater.- service
Role string IAM role that will be assumed by the Amazon EMR service to access AWS resources.
- step
Concurrency numberLevel Number of steps that can be executed concurrently. You can specify a maximum of 256 steps. Only valid for EMR clusters with
release_label5.28.0 or greater (default is 1).- steps
Cluster
Step Args[] List of steps to run when creating the cluster. See below. It is highly recommended to utilize the lifecycle resource options block with
ignoreChangesif other steps are being managed outside of this provider.- {[key: string]: string}
list of tags to apply to the EMR Cluster. If configured with a provider
default_tagsconfiguration block present, tags with matching keys will overwrite those defined at the provider-level.- {[key: string]: string}
Map of tags assigned to the resource, including those inherited from the provider
default_tagsconfiguration block.- termination
Protection boolean Switch on/off termination protection (default is
false, except when using multiple master nodes). Before attempting to destroy the resource when termination protection is enabled, this configuration must be applied with its value set tofalse.- visible
To booleanAll Users Whether the job flow is visible to all IAM users of the AWS account associated with the job flow. Default value is
true.
- additional_
info str - applications Sequence[str]
A case-insensitive list of applications for Amazon EMR to install and configure when launching the cluster. For a list of applications available for each Amazon EMR release version, see the Amazon EMR Release Guide.
- arn str
- auto_
termination_ Clusterpolicy Auto Termination Policy Args An auto-termination policy for an Amazon EMR cluster. An auto-termination policy defines the amount of idle time in seconds after which a cluster automatically terminates. See Auto Termination Policy Below.
- autoscaling_
role str IAM role for automatic scaling policies. The IAM role provides permissions that the automatic scaling feature requires to launch and terminate EC2 instances in an instance group.
- bootstrap_
actions Sequence[ClusterBootstrap Action Args] Ordered list of bootstrap actions that will be run before Hadoop is started on the cluster nodes. See below.
- cluster_
state str - configurations str
Configuration classification that applies when provisioning cluster instances, which can include configurations for applications and software that run on the cluster. List of
configurationblocks.- configurations_
json str JSON string for supplying list of configurations for the EMR cluster.
- core_
instance_ Clusterfleet Core Instance Fleet Args Configuration block to use an Instance Fleet for the core node type. Cannot be specified if any
core_instance_groupconfiguration blocks are set. Detailed below.- core_
instance_ Clustergroup Core Instance Group Args Configuration block to use an Instance Group for the core node type.
- custom_
ami_ strid Custom Amazon Linux AMI for the cluster (instead of an EMR-owned AMI). Available in Amazon EMR version 5.7.0 and later.
- ebs_
root_ intvolume_ size Size in GiB of the EBS root device volume of the Linux AMI that is used for each EC2 instance. Available in Amazon EMR version 4.x and later.
- ec2_
attributes ClusterEc2Attributes Args Attributes for the EC2 instances running the job flow. See below.
- keep_
job_ boolflow_ alive_ when_ no_ steps Switch on/off run cluster with no steps or when all steps are complete (default is on)
- kerberos_
attributes ClusterKerberos Attributes Args Kerberos configuration for the cluster. See below.
- list_
steps_ Sequence[str]states List of step states used to filter returned steps
- log_
encryption_ strkms_ key_ id AWS KMS customer master key (CMK) key ID or arn used for encrypting log files. This attribute is only available with EMR version 5.30.0 and later, excluding EMR 6.0.0.
- log_
uri str S3 bucket to write the log files of the job flow. If a value is not provided, logs are not created.
- master_
instance_ Clusterfleet Master Instance Fleet Args Configuration block to use an Instance Fleet for the master node type. Cannot be specified if any
master_instance_groupconfiguration blocks are set. Detailed below.- master_
instance_ Clustergroup Master Instance Group Args Configuration block to use an Instance Group for the master node type.
- master_
public_ strdns The DNS name of the master node. If the cluster is on a private subnet, this is the private DNS name. On a public subnet, this is the public DNS name.
- name str
Name of the step.
- release_
label str Release label for the Amazon EMR release.
- scale_
down_ strbehavior Way that individual Amazon EC2 instances terminate when an automatic scale-in activity occurs or an
instance groupis resized.- security_
configuration str Security configuration name to attach to the EMR cluster. Only valid for EMR clusters with
release_label4.8.0 or greater.- service_
role str IAM role that will be assumed by the Amazon EMR service to access AWS resources.
- step_
concurrency_ intlevel Number of steps that can be executed concurrently. You can specify a maximum of 256 steps. Only valid for EMR clusters with
release_label5.28.0 or greater (default is 1).- steps
Sequence[Cluster
Step Args] List of steps to run when creating the cluster. See below. It is highly recommended to utilize the lifecycle resource options block with
ignoreChangesif other steps are being managed outside of this provider.- Mapping[str, str]
list of tags to apply to the EMR Cluster. If configured with a provider
default_tagsconfiguration block present, tags with matching keys will overwrite those defined at the provider-level.- Mapping[str, str]
Map of tags assigned to the resource, including those inherited from the provider
default_tagsconfiguration block.- termination_
protection bool Switch on/off termination protection (default is
false, except when using multiple master nodes). Before attempting to destroy the resource when termination protection is enabled, this configuration must be applied with its value set tofalse.- visible_
to_ boolall_ users Whether the job flow is visible to all IAM users of the AWS account associated with the job flow. Default value is
true.
- additional
Info String - applications List<String>
A case-insensitive list of applications for Amazon EMR to install and configure when launching the cluster. For a list of applications available for each Amazon EMR release version, see the Amazon EMR Release Guide.
- arn String
- auto
Termination Property MapPolicy An auto-termination policy for an Amazon EMR cluster. An auto-termination policy defines the amount of idle time in seconds after which a cluster automatically terminates. See Auto Termination Policy Below.
- autoscaling
Role String IAM role for automatic scaling policies. The IAM role provides permissions that the automatic scaling feature requires to launch and terminate EC2 instances in an instance group.
- bootstrap
Actions List<Property Map> Ordered list of bootstrap actions that will be run before Hadoop is started on the cluster nodes. See below.
- cluster
State String - configurations String
Configuration classification that applies when provisioning cluster instances, which can include configurations for applications and software that run on the cluster. List of
configurationblocks.- configurations
Json String JSON string for supplying list of configurations for the EMR cluster.
- core
Instance Property MapFleet Configuration block to use an Instance Fleet for the core node type. Cannot be specified if any
core_instance_groupconfiguration blocks are set. Detailed below.- core
Instance Property MapGroup Configuration block to use an Instance Group for the core node type.
- custom
Ami StringId Custom Amazon Linux AMI for the cluster (instead of an EMR-owned AMI). Available in Amazon EMR version 5.7.0 and later.
- ebs
Root NumberVolume Size Size in GiB of the EBS root device volume of the Linux AMI that is used for each EC2 instance. Available in Amazon EMR version 4.x and later.
- ec2Attributes Property Map
Attributes for the EC2 instances running the job flow. See below.
- keep
Job BooleanFlow Alive When No Steps Switch on/off run cluster with no steps or when all steps are complete (default is on)
- kerberos
Attributes Property Map Kerberos configuration for the cluster. See below.
- list
Steps List<String>States List of step states used to filter returned steps
- log
Encryption StringKms Key Id AWS KMS customer master key (CMK) key ID or arn used for encrypting log files. This attribute is only available with EMR version 5.30.0 and later, excluding EMR 6.0.0.
- log
Uri String S3 bucket to write the log files of the job flow. If a value is not provided, logs are not created.
- master
Instance Property MapFleet Configuration block to use an Instance Fleet for the master node type. Cannot be specified if any
master_instance_groupconfiguration blocks are set. Detailed below.- master
Instance Property MapGroup Configuration block to use an Instance Group for the master node type.
- master
Public StringDns The DNS name of the master node. If the cluster is on a private subnet, this is the private DNS name. On a public subnet, this is the public DNS name.
- name String
Name of the step.
- release
Label String Release label for the Amazon EMR release.
- scale
Down StringBehavior Way that individual Amazon EC2 instances terminate when an automatic scale-in activity occurs or an
instance groupis resized.- security
Configuration String Security configuration name to attach to the EMR cluster. Only valid for EMR clusters with
release_label4.8.0 or greater.- service
Role String IAM role that will be assumed by the Amazon EMR service to access AWS resources.
- step
Concurrency NumberLevel Number of steps that can be executed concurrently. You can specify a maximum of 256 steps. Only valid for EMR clusters with
release_label5.28.0 or greater (default is 1).- steps List<Property Map>
List of steps to run when creating the cluster. See below. It is highly recommended to utilize the lifecycle resource options block with
ignoreChangesif other steps are being managed outside of this provider.- Map<String>
list of tags to apply to the EMR Cluster. If configured with a provider
default_tagsconfiguration block present, tags with matching keys will overwrite those defined at the provider-level.- Map<String>
Map of tags assigned to the resource, including those inherited from the provider
default_tagsconfiguration block.- termination
Protection Boolean Switch on/off termination protection (default is
false, except when using multiple master nodes). Before attempting to destroy the resource when termination protection is enabled, this configuration must be applied with its value set tofalse.- visible
To BooleanAll Users Whether the job flow is visible to all IAM users of the AWS account associated with the job flow. Default value is
true.
Supporting Types
ClusterAutoTerminationPolicy
- Idle
Timeout int Specifies the amount of idle time in seconds after which the cluster automatically terminates. You can specify a minimum of
60seconds and a maximum of604800seconds (seven days).
- Idle
Timeout int Specifies the amount of idle time in seconds after which the cluster automatically terminates. You can specify a minimum of
60seconds and a maximum of604800seconds (seven days).
- idle
Timeout Integer Specifies the amount of idle time in seconds after which the cluster automatically terminates. You can specify a minimum of
60seconds and a maximum of604800seconds (seven days).
- idle
Timeout number Specifies the amount of idle time in seconds after which the cluster automatically terminates. You can specify a minimum of
60seconds and a maximum of604800seconds (seven days).
- idle_
timeout int Specifies the amount of idle time in seconds after which the cluster automatically terminates. You can specify a minimum of
60seconds and a maximum of604800seconds (seven days).
- idle
Timeout Number Specifies the amount of idle time in seconds after which the cluster automatically terminates. You can specify a minimum of
60seconds and a maximum of604800seconds (seven days).
ClusterBootstrapAction
ClusterCoreInstanceFleet
- Id string
ID of the cluster.
- Instance
Type List<ClusterConfigs Core Instance Fleet Instance Type Config> Configuration block for instance fleet.
- Launch
Specifications ClusterCore Instance Fleet Launch Specifications Configuration block for launch specification.
- Name string
Name of the step.
- Provisioned
On intDemand Capacity - Provisioned
Spot intCapacity - Target
On intDemand Capacity Target capacity of On-Demand units for the instance fleet, which determines how many On-Demand instances to provision.
- Target
Spot intCapacity Target capacity of Spot units for the instance fleet, which determines how many Spot instances to provision.
- Id string
ID of the cluster.
- Instance
Type []ClusterConfigs Core Instance Fleet Instance Type Config Configuration block for instance fleet.
- Launch
Specifications ClusterCore Instance Fleet Launch Specifications Configuration block for launch specification.
- Name string
Name of the step.
- Provisioned
On intDemand Capacity - Provisioned
Spot intCapacity - Target
On intDemand Capacity Target capacity of On-Demand units for the instance fleet, which determines how many On-Demand instances to provision.
- Target
Spot intCapacity Target capacity of Spot units for the instance fleet, which determines how many Spot instances to provision.
- id String
ID of the cluster.
- instance
Type List<ClusterConfigs Core Instance Fleet Instance Type Config> Configuration block for instance fleet.
- launch
Specifications ClusterCore Instance Fleet Launch Specifications Configuration block for launch specification.
- name String
Name of the step.
- provisioned
On IntegerDemand Capacity - provisioned
Spot IntegerCapacity - target
On IntegerDemand Capacity Target capacity of On-Demand units for the instance fleet, which determines how many On-Demand instances to provision.
- target
Spot IntegerCapacity Target capacity of Spot units for the instance fleet, which determines how many Spot instances to provision.
- id string
ID of the cluster.
- instance
Type ClusterConfigs Core Instance Fleet Instance Type Config[] Configuration block for instance fleet.
- launch
Specifications ClusterCore Instance Fleet Launch Specifications Configuration block for launch specification.
- name string
Name of the step.
- provisioned
On numberDemand Capacity - provisioned
Spot numberCapacity - target
On numberDemand Capacity Target capacity of On-Demand units for the instance fleet, which determines how many On-Demand instances to provision.
- target
Spot numberCapacity Target capacity of Spot units for the instance fleet, which determines how many Spot instances to provision.
- id str
ID of the cluster.
- instance_
type_ Sequence[Clusterconfigs Core Instance Fleet Instance Type Config] Configuration block for instance fleet.
- launch_
specifications ClusterCore Instance Fleet Launch Specifications Configuration block for launch specification.
- name str
Name of the step.
- provisioned_
on_ intdemand_ capacity - provisioned_
spot_ intcapacity - target_
on_ intdemand_ capacity Target capacity of On-Demand units for the instance fleet, which determines how many On-Demand instances to provision.
- target_
spot_ intcapacity Target capacity of Spot units for the instance fleet, which determines how many Spot instances to provision.
- id String
ID of the cluster.
- instance
Type List<Property Map>Configs Configuration block for instance fleet.
- launch
Specifications Property Map Configuration block for launch specification.
- name String
Name of the step.
- provisioned
On NumberDemand Capacity - provisioned
Spot NumberCapacity - target
On NumberDemand Capacity Target capacity of On-Demand units for the instance fleet, which determines how many On-Demand instances to provision.
- target
Spot NumberCapacity Target capacity of Spot units for the instance fleet, which determines how many Spot instances to provision.
ClusterCoreInstanceFleetInstanceTypeConfig
- Instance
Type string EC2 instance type for all instances in the instance group.
- Bid
Price string Bid price for each EC2 instance in the instance group, expressed in USD. By setting this attribute, the instance group is being declared as a Spot Instance, and will implicitly create a Spot request. Leave this blank to use On-Demand Instances.
- Bid
Price doubleAs Percentage Of On Demand Price Bid price, as a percentage of On-Demand price, for each EC2 Spot instance as defined by
instance_type. Expressed as a number (for example, 20 specifies 20%). If neitherbid_pricenorbid_price_as_percentage_of_on_demand_priceis provided,bid_price_as_percentage_of_on_demand_pricedefaults to 100%.- Configurations
List<Cluster
Core Instance Fleet Instance Type Config Configuration> Configuration classification that applies when provisioning cluster instances, which can include configurations for applications and software that run on the cluster. List of
configurationblocks.- Ebs
Configs List<ClusterCore Instance Fleet Instance Type Config Ebs Config> Configuration block(s) for EBS volumes attached to each instance in the instance group. Detailed below.
- Weighted
Capacity int Number of units that a provisioned instance of this type provides toward fulfilling the target capacities defined in
aws.emr.InstanceFleet.
- Instance
Type string EC2 instance type for all instances in the instance group.
- Bid
Price string Bid price for each EC2 instance in the instance group, expressed in USD. By setting this attribute, the instance group is being declared as a Spot Instance, and will implicitly create a Spot request. Leave this blank to use On-Demand Instances.
- Bid
Price float64As Percentage Of On Demand Price Bid price, as a percentage of On-Demand price, for each EC2 Spot instance as defined by
instance_type. Expressed as a number (for example, 20 specifies 20%). If neitherbid_pricenorbid_price_as_percentage_of_on_demand_priceis provided,bid_price_as_percentage_of_on_demand_pricedefaults to 100%.- Configurations
[]Cluster
Core Instance Fleet Instance Type Config Configuration Configuration classification that applies when provisioning cluster instances, which can include configurations for applications and software that run on the cluster. List of
configurationblocks.- Ebs
Configs []ClusterCore Instance Fleet Instance Type Config Ebs Config Configuration block(s) for EBS volumes attached to each instance in the instance group. Detailed below.
- Weighted
Capacity int Number of units that a provisioned instance of this type provides toward fulfilling the target capacities defined in
aws.emr.InstanceFleet.
- instance
Type String EC2 instance type for all instances in the instance group.
- bid
Price String Bid price for each EC2 instance in the instance group, expressed in USD. By setting this attribute, the instance group is being declared as a Spot Instance, and will implicitly create a Spot request. Leave this blank to use On-Demand Instances.
- bid
Price DoubleAs Percentage Of On Demand Price Bid price, as a percentage of On-Demand price, for each EC2 Spot instance as defined by
instance_type. Expressed as a number (for example, 20 specifies 20%). If neitherbid_pricenorbid_price_as_percentage_of_on_demand_priceis provided,bid_price_as_percentage_of_on_demand_pricedefaults to 100%.- configurations
List<Cluster
Core Instance Fleet Instance Type Config Configuration> Configuration classification that applies when provisioning cluster instances, which can include configurations for applications and software that run on the cluster. List of
configurationblocks.- ebs
Configs List<ClusterCore Instance Fleet Instance Type Config Ebs Config> Configuration block(s) for EBS volumes attached to each instance in the instance group. Detailed below.
- weighted
Capacity Integer Number of units that a provisioned instance of this type provides toward fulfilling the target capacities defined in
aws.emr.InstanceFleet.
- instance
Type string EC2 instance type for all instances in the instance group.
- bid
Price string Bid price for each EC2 instance in the instance group, expressed in USD. By setting this attribute, the instance group is being declared as a Spot Instance, and will implicitly create a Spot request. Leave this blank to use On-Demand Instances.
- bid
Price numberAs Percentage Of On Demand Price Bid price, as a percentage of On-Demand price, for each EC2 Spot instance as defined by
instance_type. Expressed as a number (for example, 20 specifies 20%). If neitherbid_pricenorbid_price_as_percentage_of_on_demand_priceis provided,bid_price_as_percentage_of_on_demand_pricedefaults to 100%.- configurations
Cluster
Core Instance Fleet Instance Type Config Configuration[] Configuration classification that applies when provisioning cluster instances, which can include configurations for applications and software that run on the cluster. List of
configurationblocks.- ebs
Configs ClusterCore Instance Fleet Instance Type Config Ebs Config[] Configuration block(s) for EBS volumes attached to each instance in the instance group. Detailed below.
- weighted
Capacity number Number of units that a provisioned instance of this type provides toward fulfilling the target capacities defined in
aws.emr.InstanceFleet.
- instance_
type str EC2 instance type for all instances in the instance group.
- bid_
price str Bid price for each EC2 instance in the instance group, expressed in USD. By setting this attribute, the instance group is being declared as a Spot Instance, and will implicitly create a Spot request. Leave this blank to use On-Demand Instances.
- bid_
price_ floatas_ percentage_ of_ on_ demand_ price Bid price, as a percentage of On-Demand price, for each EC2 Spot instance as defined by
instance_type. Expressed as a number (for example, 20 specifies 20%). If neitherbid_pricenorbid_price_as_percentage_of_on_demand_priceis provided,bid_price_as_percentage_of_on_demand_pricedefaults to 100%.- configurations
Sequence[Cluster
Core Instance Fleet Instance Type Config Configuration] Configuration classification that applies when provisioning cluster instances, which can include configurations for applications and software that run on the cluster. List of
configurationblocks.- ebs_
configs Sequence[ClusterCore Instance Fleet Instance Type Config Ebs Config] Configuration block(s) for EBS volumes attached to each instance in the instance group. Detailed below.
- weighted_
capacity int Number of units that a provisioned instance of this type provides toward fulfilling the target capacities defined in
aws.emr.InstanceFleet.
- instance
Type String EC2 instance type for all instances in the instance group.
- bid
Price String Bid price for each EC2 instance in the instance group, expressed in USD. By setting this attribute, the instance group is being declared as a Spot Instance, and will implicitly create a Spot request. Leave this blank to use On-Demand Instances.
- bid
Price NumberAs Percentage Of On Demand Price Bid price, as a percentage of On-Demand price, for each EC2 Spot instance as defined by
instance_type. Expressed as a number (for example, 20 specifies 20%). If neitherbid_pricenorbid_price_as_percentage_of_on_demand_priceis provided,bid_price_as_percentage_of_on_demand_pricedefaults to 100%.- configurations List<Property Map>
Configuration classification that applies when provisioning cluster instances, which can include configurations for applications and software that run on the cluster. List of
configurationblocks.- ebs
Configs List<Property Map> Configuration block(s) for EBS volumes attached to each instance in the instance group. Detailed below.
- weighted
Capacity Number Number of units that a provisioned instance of this type provides toward fulfilling the target capacities defined in
aws.emr.InstanceFleet.
ClusterCoreInstanceFleetInstanceTypeConfigConfiguration
- Classification string
Classification within a configuration.
- Properties Dictionary<string, object>
Key-Value map of Java properties that are set when the step runs. You can use these properties to pass key value pairs to your main function.
- Classification string
Classification within a configuration.
- Properties map[string]interface{}
Key-Value map of Java properties that are set when the step runs. You can use these properties to pass key value pairs to your main function.
- classification String
Classification within a configuration.
- properties Map<String,Object>
Key-Value map of Java properties that are set when the step runs. You can use these properties to pass key value pairs to your main function.
- classification string
Classification within a configuration.
- properties {[key: string]: any}
Key-Value map of Java properties that are set when the step runs. You can use these properties to pass key value pairs to your main function.
- classification str
Classification within a configuration.
- properties Mapping[str, Any]
Key-Value map of Java properties that are set when the step runs. You can use these properties to pass key value pairs to your main function.
- classification String
Classification within a configuration.
- properties Map<Any>
Key-Value map of Java properties that are set when the step runs. You can use these properties to pass key value pairs to your main function.
ClusterCoreInstanceFleetInstanceTypeConfigEbsConfig
- Size int
Volume size, in gibibytes (GiB).
- Type string
Volume type. Valid options are
gp3,gp2,io1,standard,st1andsc1. See EBS Volume Types.- Iops int
Number of I/O operations per second (IOPS) that the volume supports.
- Volumes
Per intInstance Number of EBS volumes with this configuration to attach to each EC2 instance in the instance group (default is 1).
- Size int
Volume size, in gibibytes (GiB).
- Type string
Volume type. Valid options are
gp3,gp2,io1,standard,st1andsc1. See EBS Volume Types.- Iops int
Number of I/O operations per second (IOPS) that the volume supports.
- Volumes
Per intInstance Number of EBS volumes with this configuration to attach to each EC2 instance in the instance group (default is 1).
- size Integer
Volume size, in gibibytes (GiB).
- type String
Volume type. Valid options are
gp3,gp2,io1,standard,st1andsc1. See EBS Volume Types.- iops Integer
Number of I/O operations per second (IOPS) that the volume supports.
- volumes
Per IntegerInstance Number of EBS volumes with this configuration to attach to each EC2 instance in the instance group (default is 1).
- size number
Volume size, in gibibytes (GiB).
- type string
Volume type. Valid options are
gp3,gp2,io1,standard,st1andsc1. See EBS Volume Types.- iops number
Number of I/O operations per second (IOPS) that the volume supports.
- volumes
Per numberInstance Number of EBS volumes with this configuration to attach to each EC2 instance in the instance group (default is 1).
- size int
Volume size, in gibibytes (GiB).
- type str
Volume type. Valid options are
gp3,gp2,io1,standard,st1andsc1. See EBS Volume Types.- iops int
Number of I/O operations per second (IOPS) that the volume supports.
- volumes_
per_ intinstance Number of EBS volumes with this configuration to attach to each EC2 instance in the instance group (default is 1).
- size Number
Volume size, in gibibytes (GiB).
- type String
Volume type. Valid options are
gp3,gp2,io1,standard,st1andsc1. See EBS Volume Types.- iops Number
Number of I/O operations per second (IOPS) that the volume supports.
- volumes
Per NumberInstance Number of EBS volumes with this configuration to attach to each EC2 instance in the instance group (default is 1).
ClusterCoreInstanceFleetLaunchSpecifications
- On
Demand List<ClusterSpecifications Core Instance Fleet Launch Specifications On Demand Specification> Configuration block for on demand instances launch specifications.
- Spot
Specifications List<ClusterCore Instance Fleet Launch Specifications Spot Specification> Configuration block for spot instances launch specifications.
- On
Demand []ClusterSpecifications Core Instance Fleet Launch Specifications On Demand Specification Configuration block for on demand instances launch specifications.
- Spot
Specifications []ClusterCore Instance Fleet Launch Specifications Spot Specification Configuration block for spot instances launch specifications.
- on
Demand List<ClusterSpecifications Core Instance Fleet Launch Specifications On Demand Specification> Configuration block for on demand instances launch specifications.
- spot
Specifications List<ClusterCore Instance Fleet Launch Specifications Spot Specification> Configuration block for spot instances launch specifications.
- on
Demand ClusterSpecifications Core Instance Fleet Launch Specifications On Demand Specification[] Configuration block for on demand instances launch specifications.
- spot
Specifications ClusterCore Instance Fleet Launch Specifications Spot Specification[] Configuration block for spot instances launch specifications.
- on_
demand_ Sequence[Clusterspecifications Core Instance Fleet Launch Specifications On Demand Specification] Configuration block for on demand instances launch specifications.
- spot_
specifications Sequence[ClusterCore Instance Fleet Launch Specifications Spot Specification] Configuration block for spot instances launch specifications.
- on
Demand List<Property Map>Specifications Configuration block for on demand instances launch specifications.
- spot
Specifications List<Property Map> Configuration block for spot instances launch specifications.
ClusterCoreInstanceFleetLaunchSpecificationsOnDemandSpecification
- Allocation
Strategy string Specifies the strategy to use in launching Spot instance fleets. Currently, the only option is
capacity-optimized(the default), which launches instances from Spot instance pools with optimal capacity for the number of instances that are launching.
- Allocation
Strategy string Specifies the strategy to use in launching Spot instance fleets. Currently, the only option is
capacity-optimized(the default), which launches instances from Spot instance pools with optimal capacity for the number of instances that are launching.
- allocation
Strategy String Specifies the strategy to use in launching Spot instance fleets. Currently, the only option is
capacity-optimized(the default), which launches instances from Spot instance pools with optimal capacity for the number of instances that are launching.
- allocation
Strategy string Specifies the strategy to use in launching Spot instance fleets. Currently, the only option is
capacity-optimized(the default), which launches instances from Spot instance pools with optimal capacity for the number of instances that are launching.
- allocation_
strategy str Specifies the strategy to use in launching Spot instance fleets. Currently, the only option is
capacity-optimized(the default), which launches instances from Spot instance pools with optimal capacity for the number of instances that are launching.
- allocation
Strategy String Specifies the strategy to use in launching Spot instance fleets. Currently, the only option is
capacity-optimized(the default), which launches instances from Spot instance pools with optimal capacity for the number of instances that are launching.
ClusterCoreInstanceFleetLaunchSpecificationsSpotSpecification
- Allocation
Strategy string Specifies the strategy to use in launching Spot instance fleets. Currently, the only option is
capacity-optimized(the default), which launches instances from Spot instance pools with optimal capacity for the number of instances that are launching.- Timeout
Action string Action to take when TargetSpotCapacity has not been fulfilled when the TimeoutDurationMinutes has expired; that is, when all Spot instances could not be provisioned within the Spot provisioning timeout. Valid values are
TERMINATE_CLUSTERandSWITCH_TO_ON_DEMAND. SWITCH_TO_ON_DEMAND specifies that if no Spot instances are available, On-Demand Instances should be provisioned to fulfill any remaining Spot capacity.- Timeout
Duration intMinutes Spot provisioning timeout period in minutes. If Spot instances are not provisioned within this time period, the TimeOutAction is taken. Minimum value is 5 and maximum value is 1440. The timeout applies only during initial provisioning, when the cluster is first created.
- Block
Duration intMinutes Defined duration for Spot instances (also known as Spot blocks) in minutes. When specified, the Spot instance does not terminate before the defined duration expires, and defined duration pricing for Spot instances applies. Valid values are 60, 120, 180, 240, 300, or 360. The duration period starts as soon as a Spot instance receives its instance ID. At the end of the duration, Amazon EC2 marks the Spot instance for termination and provides a Spot instance termination notice, which gives the instance a two-minute warning before it terminates.
- Allocation
Strategy string Specifies the strategy to use in launching Spot instance fleets. Currently, the only option is
capacity-optimized(the default), which launches instances from Spot instance pools with optimal capacity for the number of instances that are launching.- Timeout
Action string Action to take when TargetSpotCapacity has not been fulfilled when the TimeoutDurationMinutes has expired; that is, when all Spot instances could not be provisioned within the Spot provisioning timeout. Valid values are
TERMINATE_CLUSTERandSWITCH_TO_ON_DEMAND. SWITCH_TO_ON_DEMAND specifies that if no Spot instances are available, On-Demand Instances should be provisioned to fulfill any remaining Spot capacity.- Timeout
Duration intMinutes Spot provisioning timeout period in minutes. If Spot instances are not provisioned within this time period, the TimeOutAction is taken. Minimum value is 5 and maximum value is 1440. The timeout applies only during initial provisioning, when the cluster is first created.
- Block
Duration intMinutes Defined duration for Spot instances (also known as Spot blocks) in minutes. When specified, the Spot instance does not terminate before the defined duration expires, and defined duration pricing for Spot instances applies. Valid values are 60, 120, 180, 240, 300, or 360. The duration period starts as soon as a Spot instance receives its instance ID. At the end of the duration, Amazon EC2 marks the Spot instance for termination and provides a Spot instance termination notice, which gives the instance a two-minute warning before it terminates.
- allocation
Strategy String Specifies the strategy to use in launching Spot instance fleets. Currently, the only option is
capacity-optimized(the default), which launches instances from Spot instance pools with optimal capacity for the number of instances that are launching.- timeout
Action String Action to take when TargetSpotCapacity has not been fulfilled when the TimeoutDurationMinutes has expired; that is, when all Spot instances could not be provisioned within the Spot provisioning timeout. Valid values are
TERMINATE_CLUSTERandSWITCH_TO_ON_DEMAND. SWITCH_TO_ON_DEMAND specifies that if no Spot instances are available, On-Demand Instances should be provisioned to fulfill any remaining Spot capacity.- timeout
Duration IntegerMinutes Spot provisioning timeout period in minutes. If Spot instances are not provisioned within this time period, the TimeOutAction is taken. Minimum value is 5 and maximum value is 1440. The timeout applies only during initial provisioning, when the cluster is first created.
- block
Duration IntegerMinutes Defined duration for Spot instances (also known as Spot blocks) in minutes. When specified, the Spot instance does not terminate before the defined duration expires, and defined duration pricing for Spot instances applies. Valid values are 60, 120, 180, 240, 300, or 360. The duration period starts as soon as a Spot instance receives its instance ID. At the end of the duration, Amazon EC2 marks the Spot instance for termination and provides a Spot instance termination notice, which gives the instance a two-minute warning before it terminates.
- allocation
Strategy string Specifies the strategy to use in launching Spot instance fleets. Currently, the only option is
capacity-optimized(the default), which launches instances from Spot instance pools with optimal capacity for the number of instances that are launching.- timeout
Action string Action to take when TargetSpotCapacity has not been fulfilled when the TimeoutDurationMinutes has expired; that is, when all Spot instances could not be provisioned within the Spot provisioning timeout. Valid values are
TERMINATE_CLUSTERandSWITCH_TO_ON_DEMAND. SWITCH_TO_ON_DEMAND specifies that if no Spot instances are available, On-Demand Instances should be provisioned to fulfill any remaining Spot capacity.- timeout
Duration numberMinutes Spot provisioning timeout period in minutes. If Spot instances are not provisioned within this time period, the TimeOutAction is taken. Minimum value is 5 and maximum value is 1440. The timeout applies only during initial provisioning, when the cluster is first created.
- block
Duration numberMinutes Defined duration for Spot instances (also known as Spot blocks) in minutes. When specified, the Spot instance does not terminate before the defined duration expires, and defined duration pricing for Spot instances applies. Valid values are 60, 120, 180, 240, 300, or 360. The duration period starts as soon as a Spot instance receives its instance ID. At the end of the duration, Amazon EC2 marks the Spot instance for termination and provides a Spot instance termination notice, which gives the instance a two-minute warning before it terminates.
- allocation_
strategy str Specifies the strategy to use in launching Spot instance fleets. Currently, the only option is
capacity-optimized(the default), which launches instances from Spot instance pools with optimal capacity for the number of instances that are launching.- timeout_
action str Action to take when TargetSpotCapacity has not been fulfilled when the TimeoutDurationMinutes has expired; that is, when all Spot instances could not be provisioned within the Spot provisioning timeout. Valid values are
TERMINATE_CLUSTERandSWITCH_TO_ON_DEMAND. SWITCH_TO_ON_DEMAND specifies that if no Spot instances are available, On-Demand Instances should be provisioned to fulfill any remaining Spot capacity.- timeout_
duration_ intminutes Spot provisioning timeout period in minutes. If Spot instances are not provisioned within this time period, the TimeOutAction is taken. Minimum value is 5 and maximum value is 1440. The timeout applies only during initial provisioning, when the cluster is first created.
- block_
duration_ intminutes Defined duration for Spot instances (also known as Spot blocks) in minutes. When specified, the Spot instance does not terminate before the defined duration expires, and defined duration pricing for Spot instances applies. Valid values are 60, 120, 180, 240, 300, or 360. The duration period starts as soon as a Spot instance receives its instance ID. At the end of the duration, Amazon EC2 marks the Spot instance for termination and provides a Spot instance termination notice, which gives the instance a two-minute warning before it terminates.
- allocation
Strategy String Specifies the strategy to use in launching Spot instance fleets. Currently, the only option is
capacity-optimized(the default), which launches instances from Spot instance pools with optimal capacity for the number of instances that are launching.- timeout
Action String Action to take when TargetSpotCapacity has not been fulfilled when the TimeoutDurationMinutes has expired; that is, when all Spot instances could not be provisioned within the Spot provisioning timeout. Valid values are
TERMINATE_CLUSTERandSWITCH_TO_ON_DEMAND. SWITCH_TO_ON_DEMAND specifies that if no Spot instances are available, On-Demand Instances should be provisioned to fulfill any remaining Spot capacity.- timeout
Duration NumberMinutes Spot provisioning timeout period in minutes. If Spot instances are not provisioned within this time period, the TimeOutAction is taken. Minimum value is 5 and maximum value is 1440. The timeout applies only during initial provisioning, when the cluster is first created.
- block
Duration NumberMinutes Defined duration for Spot instances (also known as Spot blocks) in minutes. When specified, the Spot instance does not terminate before the defined duration expires, and defined duration pricing for Spot instances applies. Valid values are 60, 120, 180, 240, 300, or 360. The duration period starts as soon as a Spot instance receives its instance ID. At the end of the duration, Amazon EC2 marks the Spot instance for termination and provides a Spot instance termination notice, which gives the instance a two-minute warning before it terminates.
ClusterCoreInstanceGroup
- Instance
Type string EC2 instance type for all instances in the instance group.
- Autoscaling
Policy string String containing the EMR Auto Scaling Policy JSON.
- Bid
Price string Bid price for each EC2 instance in the instance group, expressed in USD. By setting this attribute, the instance group is being declared as a Spot Instance, and will implicitly create a Spot request. Leave this blank to use On-Demand Instances.
- Ebs
Configs List<ClusterCore Instance Group Ebs Config> Configuration block(s) for EBS volumes attached to each instance in the instance group. Detailed below.
- Id string
ID of the cluster.
- Instance
Count int Target number of instances for the instance group. Must be 1 or 3. Defaults to 1. Launching with multiple master nodes is only supported in EMR version 5.23.0+, and requires this resource's
core_instance_groupto be configured. Public (Internet accessible) instances must be created in VPC subnets that have map public IP on launch enabled. Termination protection is automatically enabled when launched with multiple master nodes and this provider must have thetermination_protection = falseconfiguration applied before destroying this resource.- Name string
Name of the step.
- Instance
Type string EC2 instance type for all instances in the instance group.
- Autoscaling
Policy string String containing the EMR Auto Scaling Policy JSON.
- Bid
Price string Bid price for each EC2 instance in the instance group, expressed in USD. By setting this attribute, the instance group is being declared as a Spot Instance, and will implicitly create a Spot request. Leave this blank to use On-Demand Instances.
- Ebs
Configs []ClusterCore Instance Group Ebs Config Configuration block(s) for EBS volumes attached to each instance in the instance group. Detailed below.
- Id string
ID of the cluster.
- Instance
Count int Target number of instances for the instance group. Must be 1 or 3. Defaults to 1. Launching with multiple master nodes is only supported in EMR version 5.23.0+, and requires this resource's
core_instance_groupto be configured. Public (Internet accessible) instances must be created in VPC subnets that have map public IP on launch enabled. Termination protection is automatically enabled when launched with multiple master nodes and this provider must have thetermination_protection = falseconfiguration applied before destroying this resource.- Name string
Name of the step.
- instance
Type String EC2 instance type for all instances in the instance group.
- autoscaling
Policy String String containing the EMR Auto Scaling Policy JSON.
- bid
Price String Bid price for each EC2 instance in the instance group, expressed in USD. By setting this attribute, the instance group is being declared as a Spot Instance, and will implicitly create a Spot request. Leave this blank to use On-Demand Instances.
- ebs
Configs List<ClusterCore Instance Group Ebs Config> Configuration block(s) for EBS volumes attached to each instance in the instance group. Detailed below.
- id String
ID of the cluster.
- instance
Count Integer Target number of instances for the instance group. Must be 1 or 3. Defaults to 1. Launching with multiple master nodes is only supported in EMR version 5.23.0+, and requires this resource's
core_instance_groupto be configured. Public (Internet accessible) instances must be created in VPC subnets that have map public IP on launch enabled. Termination protection is automatically enabled when launched with multiple master nodes and this provider must have thetermination_protection = falseconfiguration applied before destroying this resource.- name String
Name of the step.
- instance
Type string EC2 instance type for all instances in the instance group.
- autoscaling
Policy string String containing the EMR Auto Scaling Policy JSON.
- bid
Price string Bid price for each EC2 instance in the instance group, expressed in USD. By setting this attribute, the instance group is being declared as a Spot Instance, and will implicitly create a Spot request. Leave this blank to use On-Demand Instances.
- ebs
Configs ClusterCore Instance Group Ebs Config[] Configuration block(s) for EBS volumes attached to each instance in the instance group. Detailed below.
- id string
ID of the cluster.
- instance
Count number Target number of instances for the instance group. Must be 1 or 3. Defaults to 1. Launching with multiple master nodes is only supported in EMR version 5.23.0+, and requires this resource's
core_instance_groupto be configured. Public (Internet accessible) instances must be created in VPC subnets that have map public IP on launch enabled. Termination protection is automatically enabled when launched with multiple master nodes and this provider must have thetermination_protection = falseconfiguration applied before destroying this resource.- name string
Name of the step.
- instance_
type str EC2 instance type for all instances in the instance group.
- autoscaling_
policy str String containing the EMR Auto Scaling Policy JSON.
- bid_
price str Bid price for each EC2 instance in the instance group, expressed in USD. By setting this attribute, the instance group is being declared as a Spot Instance, and will implicitly create a Spot request. Leave this blank to use On-Demand Instances.
- ebs_
configs Sequence[ClusterCore Instance Group Ebs Config] Configuration block(s) for EBS volumes attached to each instance in the instance group. Detailed below.
- id str
ID of the cluster.
- instance_
count int Target number of instances for the instance group. Must be 1 or 3. Defaults to 1. Launching with multiple master nodes is only supported in EMR version 5.23.0+, and requires this resource's
core_instance_groupto be configured. Public (Internet accessible) instances must be created in VPC subnets that have map public IP on launch enabled. Termination protection is automatically enabled when launched with multiple master nodes and this provider must have thetermination_protection = falseconfiguration applied before destroying this resource.- name str
Name of the step.
- instance
Type String EC2 instance type for all instances in the instance group.
- autoscaling
Policy String String containing the EMR Auto Scaling Policy JSON.
- bid
Price String Bid price for each EC2 instance in the instance group, expressed in USD. By setting this attribute, the instance group is being declared as a Spot Instance, and will implicitly create a Spot request. Leave this blank to use On-Demand Instances.
- ebs
Configs List<Property Map> Configuration block(s) for EBS volumes attached to each instance in the instance group. Detailed below.
- id String
ID of the cluster.
- instance
Count Number Target number of instances for the instance group. Must be 1 or 3. Defaults to 1. Launching with multiple master nodes is only supported in EMR version 5.23.0+, and requires this resource's
core_instance_groupto be configured. Public (Internet accessible) instances must be created in VPC subnets that have map public IP on launch enabled. Termination protection is automatically enabled when launched with multiple master nodes and this provider must have thetermination_protection = falseconfiguration applied before destroying this resource.- name String
Name of the step.
ClusterCoreInstanceGroupEbsConfig
- Size int
Volume size, in gibibytes (GiB).
- Type string
Volume type. Valid options are
gp3,gp2,io1,standard,st1andsc1. See EBS Volume Types.- Iops int
Number of I/O operations per second (IOPS) that the volume supports.
- Throughput int
The throughput, in mebibyte per second (MiB/s).
- Volumes
Per intInstance Number of EBS volumes with this configuration to attach to each EC2 instance in the instance group (default is 1).
- Size int
Volume size, in gibibytes (GiB).
- Type string
Volume type. Valid options are
gp3,gp2,io1,standard,st1andsc1. See EBS Volume Types.- Iops int
Number of I/O operations per second (IOPS) that the volume supports.
- Throughput int
The throughput, in mebibyte per second (MiB/s).
- Volumes
Per intInstance Number of EBS volumes with this configuration to attach to each EC2 instance in the instance group (default is 1).
- size Integer
Volume size, in gibibytes (GiB).
- type String
Volume type. Valid options are
gp3,gp2,io1,standard,st1andsc1. See EBS Volume Types.- iops Integer
Number of I/O operations per second (IOPS) that the volume supports.
- throughput Integer
The throughput, in mebibyte per second (MiB/s).
- volumes
Per IntegerInstance Number of EBS volumes with this configuration to attach to each EC2 instance in the instance group (default is 1).
- size number
Volume size, in gibibytes (GiB).
- type string
Volume type. Valid options are
gp3,gp2,io1,standard,st1andsc1. See EBS Volume Types.- iops number
Number of I/O operations per second (IOPS) that the volume supports.
- throughput number
The throughput, in mebibyte per second (MiB/s).
- volumes
Per numberInstance Number of EBS volumes with this configuration to attach to each EC2 instance in the instance group (default is 1).
- size int
Volume size, in gibibytes (GiB).
- type str
Volume type. Valid options are
gp3,gp2,io1,standard,st1andsc1. See EBS Volume Types.- iops int
Number of I/O operations per second (IOPS) that the volume supports.
- throughput int
The throughput, in mebibyte per second (MiB/s).
- volumes_
per_ intinstance Number of EBS volumes with this configuration to attach to each EC2 instance in the instance group (default is 1).
- size Number
Volume size, in gibibytes (GiB).
- type String
Volume type. Valid options are
gp3,gp2,io1,standard,st1andsc1. See EBS Volume Types.- iops Number
Number of I/O operations per second (IOPS) that the volume supports.
- throughput Number
The throughput, in mebibyte per second (MiB/s).
- volumes
Per NumberInstance Number of EBS volumes with this configuration to attach to each EC2 instance in the instance group (default is 1).
ClusterEc2Attributes
- Instance
Profile string Instance Profile for EC2 instances of the cluster assume this role.
- Additional
Master stringSecurity Groups String containing a comma separated list of additional Amazon EC2 security group IDs for the master node.
- Additional
Slave stringSecurity Groups String containing a comma separated list of additional Amazon EC2 security group IDs for the slave nodes as a comma separated string.
- Emr
Managed stringMaster Security Group Identifier of the Amazon EC2 EMR-Managed security group for the master node.
- Emr
Managed stringSlave Security Group Identifier of the Amazon EC2 EMR-Managed security group for the slave nodes.
- Key
Name string Amazon EC2 key pair that can be used to ssh to the master node as the user called
hadoop.- Service
Access stringSecurity Group Identifier of the Amazon EC2 service-access security group - required when the cluster runs on a private subnet.
- Subnet
Id string VPC subnet id where you want the job flow to launch. Cannot specify the
cc1.4xlargeinstance type for nodes of a job flow launched in an Amazon VPC.- Subnet
Ids List<string> List of VPC subnet id-s where you want the job flow to launch. Amazon EMR identifies the best Availability Zone to launch instances according to your fleet specifications.
- Instance
Profile string Instance Profile for EC2 instances of the cluster assume this role.
- Additional
Master stringSecurity Groups String containing a comma separated list of additional Amazon EC2 security group IDs for the master node.
- Additional
Slave stringSecurity Groups String containing a comma separated list of additional Amazon EC2 security group IDs for the slave nodes as a comma separated string.
- Emr
Managed stringMaster Security Group Identifier of the Amazon EC2 EMR-Managed security group for the master node.
- Emr
Managed stringSlave Security Group Identifier of the Amazon EC2 EMR-Managed security group for the slave nodes.
- Key
Name string Amazon EC2 key pair that can be used to ssh to the master node as the user called
hadoop.- Service
Access stringSecurity Group Identifier of the Amazon EC2 service-access security group - required when the cluster runs on a private subnet.
- Subnet
Id string VPC subnet id where you want the job flow to launch. Cannot specify the
cc1.4xlargeinstance type for nodes of a job flow launched in an Amazon VPC.- Subnet
Ids []string List of VPC subnet id-s where you want the job flow to launch. Amazon EMR identifies the best Availability Zone to launch instances according to your fleet specifications.
- instance
Profile String Instance Profile for EC2 instances of the cluster assume this role.
- additional
Master StringSecurity Groups String containing a comma separated list of additional Amazon EC2 security group IDs for the master node.
- additional
Slave StringSecurity Groups String containing a comma separated list of additional Amazon EC2 security group IDs for the slave nodes as a comma separated string.
- emr
Managed StringMaster Security Group Identifier of the Amazon EC2 EMR-Managed security group for the master node.
- emr
Managed StringSlave Security Group Identifier of the Amazon EC2 EMR-Managed security group for the slave nodes.
- key
Name String Amazon EC2 key pair that can be used to ssh to the master node as the user called
hadoop.- service
Access StringSecurity Group Identifier of the Amazon EC2 service-access security group - required when the cluster runs on a private subnet.
- subnet
Id String VPC subnet id where you want the job flow to launch. Cannot specify the
cc1.4xlargeinstance type for nodes of a job flow launched in an Amazon VPC.- subnet
Ids List<String> List of VPC subnet id-s where you want the job flow to launch. Amazon EMR identifies the best Availability Zone to launch instances according to your fleet specifications.
- instance
Profile string Instance Profile for EC2 instances of the cluster assume this role.
- additional
Master stringSecurity Groups String containing a comma separated list of additional Amazon EC2 security group IDs for the master node.
- additional
Slave stringSecurity Groups String containing a comma separated list of additional Amazon EC2 security group IDs for the slave nodes as a comma separated string.
- emr
Managed stringMaster Security Group Identifier of the Amazon EC2 EMR-Managed security group for the master node.
- emr
Managed stringSlave Security Group Identifier of the Amazon EC2 EMR-Managed security group for the slave nodes.
- key
Name string Amazon EC2 key pair that can be used to ssh to the master node as the user called
hadoop.- service
Access stringSecurity Group Identifier of the Amazon EC2 service-access security group - required when the cluster runs on a private subnet.
- subnet
Id string VPC subnet id where you want the job flow to launch. Cannot specify the
cc1.4xlargeinstance type for nodes of a job flow launched in an Amazon VPC.- subnet
Ids string[] List of VPC subnet id-s where you want the job flow to launch. Amazon EMR identifies the best Availability Zone to launch instances according to your fleet specifications.
- instance_
profile str Instance Profile for EC2 instances of the cluster assume this role.
- additional_
master_ strsecurity_ groups String containing a comma separated list of additional Amazon EC2 security group IDs for the master node.
- additional_
slave_ strsecurity_ groups String containing a comma separated list of additional Amazon EC2 security group IDs for the slave nodes as a comma separated string.
- emr_
managed_ strmaster_ security_ group Identifier of the Amazon EC2 EMR-Managed security group for the master node.
- emr_
managed_ strslave_ security_ group Identifier of the Amazon EC2 EMR-Managed security group for the slave nodes.
- key_
name str Amazon EC2 key pair that can be used to ssh to the master node as the user called
hadoop.- service_
access_ strsecurity_ group Identifier of the Amazon EC2 service-access security group - required when the cluster runs on a private subnet.
- subnet_
id str VPC subnet id where you want the job flow to launch. Cannot specify the
cc1.4xlargeinstance type for nodes of a job flow launched in an Amazon VPC.- subnet_
ids Sequence[str] List of VPC subnet id-s where you want the job flow to launch. Amazon EMR identifies the best Availability Zone to launch instances according to your fleet specifications.
- instance
Profile String Instance Profile for EC2 instances of the cluster assume this role.
- additional
Master StringSecurity Groups String containing a comma separated list of additional Amazon EC2 security group IDs for the master node.
- additional
Slave StringSecurity Groups String containing a comma separated list of additional Amazon EC2 security group IDs for the slave nodes as a comma separated string.
- emr
Managed StringMaster Security Group Identifier of the Amazon EC2 EMR-Managed security group for the master node.
- emr
Managed StringSlave Security Group Identifier of the Amazon EC2 EMR-Managed security group for the slave nodes.
- key
Name String Amazon EC2 key pair that can be used to ssh to the master node as the user called
hadoop.- service
Access StringSecurity Group Identifier of the Amazon EC2 service-access security group - required when the cluster runs on a private subnet.
- subnet
Id String VPC subnet id where you want the job flow to launch. Cannot specify the
cc1.4xlargeinstance type for nodes of a job flow launched in an Amazon VPC.- subnet
Ids List<String> List of VPC subnet id-s where you want the job flow to launch. Amazon EMR identifies the best Availability Zone to launch instances according to your fleet specifications.
ClusterKerberosAttributes
- Kdc
Admin stringPassword Password used within the cluster for the kadmin service on the cluster-dedicated KDC, which maintains Kerberos principals, password policies, and keytabs for the cluster. This provider cannot perform drift detection of this configuration.
- Realm string
Name of the Kerberos realm to which all nodes in a cluster belong. For example,
EC2.INTERNAL- Ad
Domain stringJoin Password Active Directory password for
ad_domain_join_user. This provider cannot perform drift detection of this configuration.- Ad
Domain stringJoin User Required only when establishing a cross-realm trust with an Active Directory domain. A user with sufficient privileges to join resources to the domain. This provider cannot perform drift detection of this configuration.
- Cross
Realm stringTrust Principal Password Required only when establishing a cross-realm trust with a KDC in a different realm. The cross-realm principal password, which must be identical across realms. This provider cannot perform drift detection of this configuration.
- Kdc
Admin stringPassword Password used within the cluster for the kadmin service on the cluster-dedicated KDC, which maintains Kerberos principals, password policies, and keytabs for the cluster. This provider cannot perform drift detection of this configuration.
- Realm string
Name of the Kerberos realm to which all nodes in a cluster belong. For example,
EC2.INTERNAL- Ad
Domain stringJoin Password Active Directory password for
ad_domain_join_user. This provider cannot perform drift detection of this configuration.- Ad
Domain stringJoin User Required only when establishing a cross-realm trust with an Active Directory domain. A user with sufficient privileges to join resources to the domain. This provider cannot perform drift detection of this configuration.
- Cross
Realm stringTrust Principal Password Required only when establishing a cross-realm trust with a KDC in a different realm. The cross-realm principal password, which must be identical across realms. This provider cannot perform drift detection of this configuration.
- kdc
Admin StringPassword Password used within the cluster for the kadmin service on the cluster-dedicated KDC, which maintains Kerberos principals, password policies, and keytabs for the cluster. This provider cannot perform drift detection of this configuration.
- realm String
Name of the Kerberos realm to which all nodes in a cluster belong. For example,
EC2.INTERNAL- ad
Domain StringJoin Password Active Directory password for
ad_domain_join_user. This provider cannot perform drift detection of this configuration.- ad
Domain StringJoin User Required only when establishing a cross-realm trust with an Active Directory domain. A user with sufficient privileges to join resources to the domain. This provider cannot perform drift detection of this configuration.
- cross
Realm StringTrust Principal Password Required only when establishing a cross-realm trust with a KDC in a different realm. The cross-realm principal password, which must be identical across realms. This provider cannot perform drift detection of this configuration.
- kdc
Admin stringPassword Password used within the cluster for the kadmin service on the cluster-dedicated KDC, which maintains Kerberos principals, password policies, and keytabs for the cluster. This provider cannot perform drift detection of this configuration.
- realm string
Name of the Kerberos realm to which all nodes in a cluster belong. For example,
EC2.INTERNAL- ad
Domain stringJoin Password Active Directory password for
ad_domain_join_user. This provider cannot perform drift detection of this configuration.- ad
Domain stringJoin User Required only when establishing a cross-realm trust with an Active Directory domain. A user with sufficient privileges to join resources to the domain. This provider cannot perform drift detection of this configuration.
- cross
Realm stringTrust Principal Password Required only when establishing a cross-realm trust with a KDC in a different realm. The cross-realm principal password, which must be identical across realms. This provider cannot perform drift detection of this configuration.
- kdc_
admin_ strpassword Password used within the cluster for the kadmin service on the cluster-dedicated KDC, which maintains Kerberos principals, password policies, and keytabs for the cluster. This provider cannot perform drift detection of this configuration.
- realm str
Name of the Kerberos realm to which all nodes in a cluster belong. For example,
EC2.INTERNAL- ad_
domain_ strjoin_ password Active Directory password for
ad_domain_join_user. This provider cannot perform drift detection of this configuration.- ad_
domain_ strjoin_ user Required only when establishing a cross-realm trust with an Active Directory domain. A user with sufficient privileges to join resources to the domain. This provider cannot perform drift detection of this configuration.
- cross_
realm_ strtrust_ principal_ password Required only when establishing a cross-realm trust with a KDC in a different realm. The cross-realm principal password, which must be identical across realms. This provider cannot perform drift detection of this configuration.
- kdc
Admin StringPassword Password used within the cluster for the kadmin service on the cluster-dedicated KDC, which maintains Kerberos principals, password policies, and keytabs for the cluster. This provider cannot perform drift detection of this configuration.
- realm String
Name of the Kerberos realm to which all nodes in a cluster belong. For example,
EC2.INTERNAL- ad
Domain StringJoin Password Active Directory password for
ad_domain_join_user. This provider cannot perform drift detection of this configuration.- ad
Domain StringJoin User Required only when establishing a cross-realm trust with an Active Directory domain. A user with sufficient privileges to join resources to the domain. This provider cannot perform drift detection of this configuration.
- cross
Realm StringTrust Principal Password Required only when establishing a cross-realm trust with a KDC in a different realm. The cross-realm principal password, which must be identical across realms. This provider cannot perform drift detection of this configuration.
ClusterMasterInstanceFleet
- Id string
ID of the cluster.
- Instance
Type List<ClusterConfigs Master Instance Fleet Instance Type Config> Configuration block for instance fleet.
- Launch
Specifications ClusterMaster Instance Fleet Launch Specifications Configuration block for launch specification.
- Name string
Name of the step.
- Provisioned
On intDemand Capacity - Provisioned
Spot intCapacity - Target
On intDemand Capacity Target capacity of On-Demand units for the instance fleet, which determines how many On-Demand instances to provision.
- Target
Spot intCapacity Target capacity of Spot units for the instance fleet, which determines how many Spot instances to provision.
- Id string
ID of the cluster.
- Instance
Type []ClusterConfigs Master Instance Fleet Instance Type Config Configuration block for instance fleet.
- Launch
Specifications ClusterMaster Instance Fleet Launch Specifications Configuration block for launch specification.
- Name string
Name of the step.
- Provisioned
On intDemand Capacity - Provisioned
Spot intCapacity - Target
On intDemand Capacity Target capacity of On-Demand units for the instance fleet, which determines how many On-Demand instances to provision.
- Target
Spot intCapacity Target capacity of Spot units for the instance fleet, which determines how many Spot instances to provision.
- id String
ID of the cluster.
- instance
Type List<ClusterConfigs Master Instance Fleet Instance Type Config> Configuration block for instance fleet.
- launch
Specifications ClusterMaster Instance Fleet Launch Specifications Configuration block for launch specification.
- name String
Name of the step.
- provisioned
On IntegerDemand Capacity - provisioned
Spot IntegerCapacity - target
On IntegerDemand Capacity Target capacity of On-Demand units for the instance fleet, which determines how many On-Demand instances to provision.
- target
Spot IntegerCapacity Target capacity of Spot units for the instance fleet, which determines how many Spot instances to provision.
- id string
ID of the cluster.
- instance
Type ClusterConfigs Master Instance Fleet Instance Type Config[] Configuration block for instance fleet.
- launch
Specifications ClusterMaster Instance Fleet Launch Specifications Configuration block for launch specification.
- name string
Name of the step.
- provisioned
On numberDemand Capacity - provisioned
Spot numberCapacity - target
On numberDemand Capacity Target capacity of On-Demand units for the instance fleet, which determines how many On-Demand instances to provision.
- target
Spot numberCapacity Target capacity of Spot units for the instance fleet, which determines how many Spot instances to provision.
- id str
ID of the cluster.
- instance_
type_ Sequence[Clusterconfigs Master Instance Fleet Instance Type Config] Configuration block for instance fleet.
- launch_
specifications ClusterMaster Instance Fleet Launch Specifications Configuration block for launch specification.
- name str
Name of the step.
- provisioned_
on_ intdemand_ capacity - provisioned_
spot_ intcapacity - target_
on_ intdemand_ capacity Target capacity of On-Demand units for the instance fleet, which determines how many On-Demand instances to provision.
- target_
spot_ intcapacity Target capacity of Spot units for the instance fleet, which determines how many Spot instances to provision.
- id String
ID of the cluster.
- instance
Type List<Property Map>Configs Configuration block for instance fleet.
- launch
Specifications Property Map Configuration block for launch specification.
- name String
Name of the step.
- provisioned
On NumberDemand Capacity - provisioned
Spot NumberCapacity - target
On NumberDemand Capacity Target capacity of On-Demand units for the instance fleet, which determines how many On-Demand instances to provision.
- target
Spot NumberCapacity Target capacity of Spot units for the instance fleet, which determines how many Spot instances to provision.
ClusterMasterInstanceFleetInstanceTypeConfig
- Instance
Type string EC2 instance type for all instances in the instance group.
- Bid
Price string Bid price for each EC2 instance in the instance group, expressed in USD. By setting this attribute, the instance group is being declared as a Spot Instance, and will implicitly create a Spot request. Leave this blank to use On-Demand Instances.
- Bid
Price doubleAs Percentage Of On Demand Price Bid price, as a percentage of On-Demand price, for each EC2 Spot instance as defined by
instance_type. Expressed as a number (for example, 20 specifies 20%). If neitherbid_pricenorbid_price_as_percentage_of_on_demand_priceis provided,bid_price_as_percentage_of_on_demand_pricedefaults to 100%.- Configurations
List<Cluster
Master Instance Fleet Instance Type Config Configuration> Configuration classification that applies when provisioning cluster instances, which can include configurations for applications and software that run on the cluster. List of
configurationblocks.- Ebs
Configs List<ClusterMaster Instance Fleet Instance Type Config Ebs Config> Configuration block(s) for EBS volumes attached to each instance in the instance group. Detailed below.
- Weighted
Capacity int Number of units that a provisioned instance of this type provides toward fulfilling the target capacities defined in
aws.emr.InstanceFleet.
- Instance
Type string EC2 instance type for all instances in the instance group.
- Bid
Price string Bid price for each EC2 instance in the instance group, expressed in USD. By setting this attribute, the instance group is being declared as a Spot Instance, and will implicitly create a Spot request. Leave this blank to use On-Demand Instances.
- Bid
Price float64As Percentage Of On Demand Price Bid price, as a percentage of On-Demand price, for each EC2 Spot instance as defined by
instance_type. Expressed as a number (for example, 20 specifies 20%). If neitherbid_pricenorbid_price_as_percentage_of_on_demand_priceis provided,bid_price_as_percentage_of_on_demand_pricedefaults to 100%.- Configurations
[]Cluster
Master Instance Fleet Instance Type Config Configuration Configuration classification that applies when provisioning cluster instances, which can include configurations for applications and software that run on the cluster. List of
configurationblocks.- Ebs
Configs []ClusterMaster Instance Fleet Instance Type Config Ebs Config Configuration block(s) for EBS volumes attached to each instance in the instance group. Detailed below.
- Weighted
Capacity int Number of units that a provisioned instance of this type provides toward fulfilling the target capacities defined in
aws.emr.InstanceFleet.
- instance
Type String EC2 instance type for all instances in the instance group.
- bid
Price String Bid price for each EC2 instance in the instance group, expressed in USD. By setting this attribute, the instance group is being declared as a Spot Instance, and will implicitly create a Spot request. Leave this blank to use On-Demand Instances.
- bid
Price DoubleAs Percentage Of On Demand Price Bid price, as a percentage of On-Demand price, for each EC2 Spot instance as defined by
instance_type. Expressed as a number (for example, 20 specifies 20%). If neitherbid_pricenorbid_price_as_percentage_of_on_demand_priceis provided,bid_price_as_percentage_of_on_demand_pricedefaults to 100%.- configurations
List<Cluster
Master Instance Fleet Instance Type Config Configuration> Configuration classification that applies when provisioning cluster instances, which can include configurations for applications and software that run on the cluster. List of
configurationblocks.- ebs
Configs List<ClusterMaster Instance Fleet Instance Type Config Ebs Config> Configuration block(s) for EBS volumes attached to each instance in the instance group. Detailed below.
- weighted
Capacity Integer Number of units that a provisioned instance of this type provides toward fulfilling the target capacities defined in
aws.emr.InstanceFleet.
- instance
Type string EC2 instance type for all instances in the instance group.
- bid
Price string Bid price for each EC2 instance in the instance group, expressed in USD. By setting this attribute, the instance group is being declared as a Spot Instance, and will implicitly create a Spot request. Leave this blank to use On-Demand Instances.
- bid
Price numberAs Percentage Of On Demand Price Bid price, as a percentage of On-Demand price, for each EC2 Spot instance as defined by
instance_type. Expressed as a number (for example, 20 specifies 20%). If neitherbid_pricenorbid_price_as_percentage_of_on_demand_priceis provided,bid_price_as_percentage_of_on_demand_pricedefaults to 100%.- configurations
Cluster
Master Instance Fleet Instance Type Config Configuration[] Configuration classification that applies when provisioning cluster instances, which can include configurations for applications and software that run on the cluster. List of
configurationblocks.- ebs
Configs ClusterMaster Instance Fleet Instance Type Config Ebs Config[] Configuration block(s) for EBS volumes attached to each instance in the instance group. Detailed below.
- weighted
Capacity number Number of units that a provisioned instance of this type provides toward fulfilling the target capacities defined in
aws.emr.InstanceFleet.
- instance_
type str EC2 instance type for all instances in the instance group.
- bid_
price str Bid price for each EC2 instance in the instance group, expressed in USD. By setting this attribute, the instance group is being declared as a Spot Instance, and will implicitly create a Spot request. Leave this blank to use On-Demand Instances.
- bid_
price_ floatas_ percentage_ of_ on_ demand_ price Bid price, as a percentage of On-Demand price, for each EC2 Spot instance as defined by
instance_type. Expressed as a number (for example, 20 specifies 20%). If neitherbid_pricenorbid_price_as_percentage_of_on_demand_priceis provided,bid_price_as_percentage_of_on_demand_pricedefaults to 100%.- configurations
Sequence[Cluster
Master Instance Fleet Instance Type Config Configuration] Configuration classification that applies when provisioning cluster instances, which can include configurations for applications and software that run on the cluster. List of
configurationblocks.- ebs_
configs Sequence[ClusterMaster Instance Fleet Instance Type Config Ebs Config] Configuration block(s) for EBS volumes attached to each instance in the instance group. Detailed below.
- weighted_
capacity int Number of units that a provisioned instance of this type provides toward fulfilling the target capacities defined in
aws.emr.InstanceFleet.
- instance
Type String EC2 instance type for all instances in the instance group.
- bid
Price String Bid price for each EC2 instance in the instance group, expressed in USD. By setting this attribute, the instance group is being declared as a Spot Instance, and will implicitly create a Spot request. Leave this blank to use On-Demand Instances.
- bid
Price NumberAs Percentage Of On Demand Price Bid price, as a percentage of On-Demand price, for each EC2 Spot instance as defined by
instance_type. Expressed as a number (for example, 20 specifies 20%). If neitherbid_pricenorbid_price_as_percentage_of_on_demand_priceis provided,bid_price_as_percentage_of_on_demand_pricedefaults to 100%.- configurations List<Property Map>
Configuration classification that applies when provisioning cluster instances, which can include configurations for applications and software that run on the cluster. List of
configurationblocks.- ebs
Configs List<Property Map> Configuration block(s) for EBS volumes attached to each instance in the instance group. Detailed below.
- weighted
Capacity Number Number of units that a provisioned instance of this type provides toward fulfilling the target capacities defined in
aws.emr.InstanceFleet.
ClusterMasterInstanceFleetInstanceTypeConfigConfiguration
- Classification string
Classification within a configuration.
- Properties Dictionary<string, object>
Key-Value map of Java properties that are set when the step runs. You can use these properties to pass key value pairs to your main function.
- Classification string
Classification within a configuration.
- Properties map[string]interface{}
Key-Value map of Java properties that are set when the step runs. You can use these properties to pass key value pairs to your main function.
- classification String
Classification within a configuration.
- properties Map<String,Object>
Key-Value map of Java properties that are set when the step runs. You can use these properties to pass key value pairs to your main function.
- classification string
Classification within a configuration.
- properties {[key: string]: any}
Key-Value map of Java properties that are set when the step runs. You can use these properties to pass key value pairs to your main function.
- classification str
Classification within a configuration.
- properties Mapping[str, Any]
Key-Value map of Java properties that are set when the step runs. You can use these properties to pass key value pairs to your main function.
- classification String
Classification within a configuration.
- properties Map<Any>
Key-Value map of Java properties that are set when the step runs. You can use these properties to pass key value pairs to your main function.
ClusterMasterInstanceFleetInstanceTypeConfigEbsConfig
- Size int
Volume size, in gibibytes (GiB).
- Type string
Volume type. Valid options are
gp3,gp2,io1,standard,st1andsc1. See EBS Volume Types.- Iops int
Number of I/O operations per second (IOPS) that the volume supports.
- Volumes
Per intInstance Number of EBS volumes with this configuration to attach to each EC2 instance in the instance group (default is 1).
- Size int
Volume size, in gibibytes (GiB).
- Type string
Volume type. Valid options are
gp3,gp2,io1,standard,st1andsc1. See EBS Volume Types.- Iops int
Number of I/O operations per second (IOPS) that the volume supports.
- Volumes
Per intInstance Number of EBS volumes with this configuration to attach to each EC2 instance in the instance group (default is 1).
- size Integer
Volume size, in gibibytes (GiB).
- type String
Volume type. Valid options are
gp3,gp2,io1,standard,st1andsc1. See EBS Volume Types.- iops Integer
Number of I/O operations per second (IOPS) that the volume supports.
- volumes
Per IntegerInstance Number of EBS volumes with this configuration to attach to each EC2 instance in the instance group (default is 1).
- size number
Volume size, in gibibytes (GiB).
- type string
Volume type. Valid options are
gp3,gp2,io1,standard,st1andsc1. See EBS Volume Types.- iops number
Number of I/O operations per second (IOPS) that the volume supports.
- volumes
Per numberInstance Number of EBS volumes with this configuration to attach to each EC2 instance in the instance group (default is 1).
- size int
Volume size, in gibibytes (GiB).
- type str
Volume type. Valid options are
gp3,gp2,io1,standard,st1andsc1. See EBS Volume Types.- iops int
Number of I/O operations per second (IOPS) that the volume supports.
- volumes_
per_ intinstance Number of EBS volumes with this configuration to attach to each EC2 instance in the instance group (default is 1).
- size Number
Volume size, in gibibytes (GiB).
- type String
Volume type. Valid options are
gp3,gp2,io1,standard,st1andsc1. See EBS Volume Types.- iops Number
Number of I/O operations per second (IOPS) that the volume supports.
- volumes
Per NumberInstance Number of EBS volumes with this configuration to attach to each EC2 instance in the instance group (default is 1).
ClusterMasterInstanceFleetLaunchSpecifications
- On
Demand List<ClusterSpecifications Master Instance Fleet Launch Specifications On Demand Specification> Configuration block for on demand instances launch specifications.
- Spot
Specifications List<ClusterMaster Instance Fleet Launch Specifications Spot Specification> Configuration block for spot instances launch specifications.
- On
Demand []ClusterSpecifications Master Instance Fleet Launch Specifications On Demand Specification Configuration block for on demand instances launch specifications.
- Spot
Specifications []ClusterMaster Instance Fleet Launch Specifications Spot Specification Configuration block for spot instances launch specifications.
- on
Demand List<ClusterSpecifications Master Instance Fleet Launch Specifications On Demand Specification> Configuration block for on demand instances launch specifications.
- spot
Specifications List<ClusterMaster Instance Fleet Launch Specifications Spot Specification> Configuration block for spot instances launch specifications.
- on
Demand ClusterSpecifications Master Instance Fleet Launch Specifications On Demand Specification[] Configuration block for on demand instances launch specifications.
- spot
Specifications ClusterMaster Instance Fleet Launch Specifications Spot Specification[] Configuration block for spot instances launch specifications.
- on_
demand_ Sequence[Clusterspecifications Master Instance Fleet Launch Specifications On Demand Specification] Configuration block for on demand instances launch specifications.
- spot_
specifications Sequence[ClusterMaster Instance Fleet Launch Specifications Spot Specification] Configuration block for spot instances launch specifications.
- on
Demand List<Property Map>Specifications Configuration block for on demand instances launch specifications.
- spot
Specifications List<Property Map> Configuration block for spot instances launch specifications.
ClusterMasterInstanceFleetLaunchSpecificationsOnDemandSpecification
- Allocation
Strategy string Specifies the strategy to use in launching Spot instance fleets. Currently, the only option is
capacity-optimized(the default), which launches instances from Spot instance pools with optimal capacity for the number of instances that are launching.
- Allocation
Strategy string Specifies the strategy to use in launching Spot instance fleets. Currently, the only option is
capacity-optimized(the default), which launches instances from Spot instance pools with optimal capacity for the number of instances that are launching.
- allocation
Strategy String Specifies the strategy to use in launching Spot instance fleets. Currently, the only option is
capacity-optimized(the default), which launches instances from Spot instance pools with optimal capacity for the number of instances that are launching.
- allocation
Strategy string Specifies the strategy to use in launching Spot instance fleets. Currently, the only option is
capacity-optimized(the default), which launches instances from Spot instance pools with optimal capacity for the number of instances that are launching.
- allocation_
strategy str Specifies the strategy to use in launching Spot instance fleets. Currently, the only option is
capacity-optimized(the default), which launches instances from Spot instance pools with optimal capacity for the number of instances that are launching.
- allocation
Strategy String Specifies the strategy to use in launching Spot instance fleets. Currently, the only option is
capacity-optimized(the default), which launches instances from Spot instance pools with optimal capacity for the number of instances that are launching.
ClusterMasterInstanceFleetLaunchSpecificationsSpotSpecification
- Allocation
Strategy string Specifies the strategy to use in launching Spot instance fleets. Currently, the only option is
capacity-optimized(the default), which launches instances from Spot instance pools with optimal capacity for the number of instances that are launching.- Timeout
Action string Action to take when TargetSpotCapacity has not been fulfilled when the TimeoutDurationMinutes has expired; that is, when all Spot instances could not be provisioned within the Spot provisioning timeout. Valid values are
TERMINATE_CLUSTERandSWITCH_TO_ON_DEMAND. SWITCH_TO_ON_DEMAND specifies that if no Spot instances are available, On-Demand Instances should be provisioned to fulfill any remaining Spot capacity.- Timeout
Duration intMinutes Spot provisioning timeout period in minutes. If Spot instances are not provisioned within this time period, the TimeOutAction is taken. Minimum value is 5 and maximum value is 1440. The timeout applies only during initial provisioning, when the cluster is first created.
- Block
Duration intMinutes Defined duration for Spot instances (also known as Spot blocks) in minutes. When specified, the Spot instance does not terminate before the defined duration expires, and defined duration pricing for Spot instances applies. Valid values are 60, 120, 180, 240, 300, or 360. The duration period starts as soon as a Spot instance receives its instance ID. At the end of the duration, Amazon EC2 marks the Spot instance for termination and provides a Spot instance termination notice, which gives the instance a two-minute warning before it terminates.
- Allocation
Strategy string Specifies the strategy to use in launching Spot instance fleets. Currently, the only option is
capacity-optimized(the default), which launches instances from Spot instance pools with optimal capacity for the number of instances that are launching.- Timeout
Action string Action to take when TargetSpotCapacity has not been fulfilled when the TimeoutDurationMinutes has expired; that is, when all Spot instances could not be provisioned within the Spot provisioning timeout. Valid values are
TERMINATE_CLUSTERandSWITCH_TO_ON_DEMAND. SWITCH_TO_ON_DEMAND specifies that if no Spot instances are available, On-Demand Instances should be provisioned to fulfill any remaining Spot capacity.- Timeout
Duration intMinutes Spot provisioning timeout period in minutes. If Spot instances are not provisioned within this time period, the TimeOutAction is taken. Minimum value is 5 and maximum value is 1440. The timeout applies only during initial provisioning, when the cluster is first created.
- Block
Duration intMinutes Defined duration for Spot instances (also known as Spot blocks) in minutes. When specified, the Spot instance does not terminate before the defined duration expires, and defined duration pricing for Spot instances applies. Valid values are 60, 120, 180, 240, 300, or 360. The duration period starts as soon as a Spot instance receives its instance ID. At the end of the duration, Amazon EC2 marks the Spot instance for termination and provides a Spot instance termination notice, which gives the instance a two-minute warning before it terminates.
- allocation
Strategy String Specifies the strategy to use in launching Spot instance fleets. Currently, the only option is
capacity-optimized(the default), which launches instances from Spot instance pools with optimal capacity for the number of instances that are launching.- timeout
Action String Action to take when TargetSpotCapacity has not been fulfilled when the TimeoutDurationMinutes has expired; that is, when all Spot instances could not be provisioned within the Spot provisioning timeout. Valid values are
TERMINATE_CLUSTERandSWITCH_TO_ON_DEMAND. SWITCH_TO_ON_DEMAND specifies that if no Spot instances are available, On-Demand Instances should be provisioned to fulfill any remaining Spot capacity.- timeout
Duration IntegerMinutes Spot provisioning timeout period in minutes. If Spot instances are not provisioned within this time period, the TimeOutAction is taken. Minimum value is 5 and maximum value is 1440. The timeout applies only during initial provisioning, when the cluster is first created.
- block
Duration IntegerMinutes Defined duration for Spot instances (also known as Spot blocks) in minutes. When specified, the Spot instance does not terminate before the defined duration expires, and defined duration pricing for Spot instances applies. Valid values are 60, 120, 180, 240, 300, or 360. The duration period starts as soon as a Spot instance receives its instance ID. At the end of the duration, Amazon EC2 marks the Spot instance for termination and provides a Spot instance termination notice, which gives the instance a two-minute warning before it terminates.
- allocation
Strategy string Specifies the strategy to use in launching Spot instance fleets. Currently, the only option is
capacity-optimized(the default), which launches instances from Spot instance pools with optimal capacity for the number of instances that are launching.- timeout
Action string Action to take when TargetSpotCapacity has not been fulfilled when the TimeoutDurationMinutes has expired; that is, when all Spot instances could not be provisioned within the Spot provisioning timeout. Valid values are
TERMINATE_CLUSTERandSWITCH_TO_ON_DEMAND. SWITCH_TO_ON_DEMAND specifies that if no Spot instances are available, On-Demand Instances should be provisioned to fulfill any remaining Spot capacity.- timeout
Duration numberMinutes Spot provisioning timeout period in minutes. If Spot instances are not provisioned within this time period, the TimeOutAction is taken. Minimum value is 5 and maximum value is 1440. The timeout applies only during initial provisioning, when the cluster is first created.
- block
Duration numberMinutes Defined duration for Spot instances (also known as Spot blocks) in minutes. When specified, the Spot instance does not terminate before the defined duration expires, and defined duration pricing for Spot instances applies. Valid values are 60, 120, 180, 240, 300, or 360. The duration period starts as soon as a Spot instance receives its instance ID. At the end of the duration, Amazon EC2 marks the Spot instance for termination and provides a Spot instance termination notice, which gives the instance a two-minute warning before it terminates.
- allocation_
strategy str Specifies the strategy to use in launching Spot instance fleets. Currently, the only option is
capacity-optimized(the default), which launches instances from Spot instance pools with optimal capacity for the number of instances that are launching.- timeout_
action str Action to take when TargetSpotCapacity has not been fulfilled when the TimeoutDurationMinutes has expired; that is, when all Spot instances could not be provisioned within the Spot provisioning timeout. Valid values are
TERMINATE_CLUSTERandSWITCH_TO_ON_DEMAND. SWITCH_TO_ON_DEMAND specifies that if no Spot instances are available, On-Demand Instances should be provisioned to fulfill any remaining Spot capacity.- timeout_
duration_ intminutes Spot provisioning timeout period in minutes. If Spot instances are not provisioned within this time period, the TimeOutAction is taken. Minimum value is 5 and maximum value is 1440. The timeout applies only during initial provisioning, when the cluster is first created.
- block_
duration_ intminutes Defined duration for Spot instances (also known as Spot blocks) in minutes. When specified, the Spot instance does not terminate before the defined duration expires, and defined duration pricing for Spot instances applies. Valid values are 60, 120, 180, 240, 300, or 360. The duration period starts as soon as a Spot instance receives its instance ID. At the end of the duration, Amazon EC2 marks the Spot instance for termination and provides a Spot instance termination notice, which gives the instance a two-minute warning before it terminates.
- allocation
Strategy String Specifies the strategy to use in launching Spot instance fleets. Currently, the only option is
capacity-optimized(the default), which launches instances from Spot instance pools with optimal capacity for the number of instances that are launching.- timeout
Action String Action to take when TargetSpotCapacity has not been fulfilled when the TimeoutDurationMinutes has expired; that is, when all Spot instances could not be provisioned within the Spot provisioning timeout. Valid values are
TERMINATE_CLUSTERandSWITCH_TO_ON_DEMAND. SWITCH_TO_ON_DEMAND specifies that if no Spot instances are available, On-Demand Instances should be provisioned to fulfill any remaining Spot capacity.- timeout
Duration NumberMinutes Spot provisioning timeout period in minutes. If Spot instances are not provisioned within this time period, the TimeOutAction is taken. Minimum value is 5 and maximum value is 1440. The timeout applies only during initial provisioning, when the cluster is first created.
- block
Duration NumberMinutes Defined duration for Spot instances (also known as Spot blocks) in minutes. When specified, the Spot instance does not terminate before the defined duration expires, and defined duration pricing for Spot instances applies. Valid values are 60, 120, 180, 240, 300, or 360. The duration period starts as soon as a Spot instance receives its instance ID. At the end of the duration, Amazon EC2 marks the Spot instance for termination and provides a Spot instance termination notice, which gives the instance a two-minute warning before it terminates.
ClusterMasterInstanceGroup
- Instance
Type string EC2 instance type for all instances in the instance group.
- Bid
Price string Bid price for each EC2 instance in the instance group, expressed in USD. By setting this attribute, the instance group is being declared as a Spot Instance, and will implicitly create a Spot request. Leave this blank to use On-Demand Instances.
- Ebs
Configs List<ClusterMaster Instance Group Ebs Config> Configuration block(s) for EBS volumes attached to each instance in the instance group. Detailed below.
- Id string
ID of the cluster.
- Instance
Count int Target number of instances for the instance group. Must be 1 or 3. Defaults to 1. Launching with multiple master nodes is only supported in EMR version 5.23.0+, and requires this resource's
core_instance_groupto be configured. Public (Internet accessible) instances must be created in VPC subnets that have map public IP on launch enabled. Termination protection is automatically enabled when launched with multiple master nodes and this provider must have thetermination_protection = falseconfiguration applied before destroying this resource.- Name string
Name of the step.
- Instance
Type string EC2 instance type for all instances in the instance group.
- Bid
Price string Bid price for each EC2 instance in the instance group, expressed in USD. By setting this attribute, the instance group is being declared as a Spot Instance, and will implicitly create a Spot request. Leave this blank to use On-Demand Instances.
- Ebs
Configs []ClusterMaster Instance Group Ebs Config Configuration block(s) for EBS volumes attached to each instance in the instance group. Detailed below.
- Id string
ID of the cluster.
- Instance
Count int Target number of instances for the instance group. Must be 1 or 3. Defaults to 1. Launching with multiple master nodes is only supported in EMR version 5.23.0+, and requires this resource's
core_instance_groupto be configured. Public (Internet accessible) instances must be created in VPC subnets that have map public IP on launch enabled. Termination protection is automatically enabled when launched with multiple master nodes and this provider must have thetermination_protection = falseconfiguration applied before destroying this resource.- Name string
Name of the step.
- instance
Type String EC2 instance type for all instances in the instance group.
- bid
Price String Bid price for each EC2 instance in the instance group, expressed in USD. By setting this attribute, the instance group is being declared as a Spot Instance, and will implicitly create a Spot request. Leave this blank to use On-Demand Instances.
- ebs
Configs List<ClusterMaster Instance Group Ebs Config> Configuration block(s) for EBS volumes attached to each instance in the instance group. Detailed below.
- id String
ID of the cluster.
- instance
Count Integer Target number of instances for the instance group. Must be 1 or 3. Defaults to 1. Launching with multiple master nodes is only supported in EMR version 5.23.0+, and requires this resource's
core_instance_groupto be configured. Public (Internet accessible) instances must be created in VPC subnets that have map public IP on launch enabled. Termination protection is automatically enabled when launched with multiple master nodes and this provider must have thetermination_protection = falseconfiguration applied before destroying this resource.- name String
Name of the step.
- instance
Type string EC2 instance type for all instances in the instance group.
- bid
Price string Bid price for each EC2 instance in the instance group, expressed in USD. By setting this attribute, the instance group is being declared as a Spot Instance, and will implicitly create a Spot request. Leave this blank to use On-Demand Instances.
- ebs
Configs ClusterMaster Instance Group Ebs Config[] Configuration block(s) for EBS volumes attached to each instance in the instance group. Detailed below.
- id string
ID of the cluster.
- instance
Count number Target number of instances for the instance group. Must be 1 or 3. Defaults to 1. Launching with multiple master nodes is only supported in EMR version 5.23.0+, and requires this resource's
core_instance_groupto be configured. Public (Internet accessible) instances must be created in VPC subnets that have map public IP on launch enabled. Termination protection is automatically enabled when launched with multiple master nodes and this provider must have thetermination_protection = falseconfiguration applied before destroying this resource.- name string
Name of the step.
- instance_
type str EC2 instance type for all instances in the instance group.