The azure-native:hdinsight:ClusterPoolCluster resource, part of the Pulumi Azure Native provider, defines an HDInsight on AKS cluster within a cluster pool: its type (Trino, Spark, Ranger), compute nodes, and service-specific configuration. This guide focuses on four capabilities: autoscaling configuration, Apache Ranger integration, service-level tuning, and network access control.
Clusters run within cluster pools and reference managed identities, storage accounts, SQL databases, and Key Vault secrets that must exist separately. The examples are intentionally small. Combine them with your own cluster pool, identity, and storage infrastructure.
Deploy a Trino cluster with autoscaling
Analytics workloads often need clusters that scale automatically based on demand. HDInsight on AKS supports both load-based and schedule-based autoscaling to optimize costs while maintaining performance.
import * as pulumi from "@pulumi/pulumi";
import * as azure_native from "@pulumi/azure-native";
const clusterPoolCluster = new azure_native.hdinsight.ClusterPoolCluster("clusterPoolCluster", {
clusterName: "cluster1",
clusterPoolName: "clusterpool1",
clusterProfile: {
authorizationProfile: {
userIds: [
"testuser1",
"testuser2",
],
},
autoscaleProfile: {
autoscaleType: azure_native.hdinsight.AutoscaleType.ScheduleBased,
enabled: true,
gracefulDecommissionTimeout: 3600,
loadBasedConfig: {
cooldownPeriod: 300,
maxNodes: 20,
minNodes: 10,
pollInterval: 60,
scalingRules: [
{
actionType: azure_native.hdinsight.ScaleActionType.Scaleup,
comparisonRule: {
operator: azure_native.hdinsight.ComparisonOperator.GreaterThan,
threshold: 90,
},
evaluationCount: 3,
scalingMetric: "cpu",
},
{
actionType: azure_native.hdinsight.ScaleActionType.Scaledown,
comparisonRule: {
operator: azure_native.hdinsight.ComparisonOperator.LessThan,
threshold: 20,
},
evaluationCount: 3,
scalingMetric: "cpu",
},
],
},
scheduleBasedConfig: {
defaultCount: 10,
schedules: [
{
count: 20,
days: [azure_native.hdinsight.ScheduleDay.Monday],
endTime: "12:00",
startTime: "00:00",
},
{
count: 25,
days: [azure_native.hdinsight.ScheduleDay.Sunday],
endTime: "12:00",
startTime: "00:00",
},
],
timeZone: "Cen. Australia Standard Time",
},
},
clusterVersion: "1.0.6",
managedIdentityProfile: {
identityList: [{
clientId: "de91f1d8-767f-460a-ac11-3cf103f74b34",
objectId: "40491351-c240-4042-91e0-f644a1d2b441",
resourceId: "/subscriptions/subid/resourceGroups/hiloResourcegroup/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-msi",
type: azure_native.hdinsight.ManagedIdentityType.Cluster,
}],
},
ossVersion: "0.410.0",
sshProfile: {
count: 2,
vmSize: "Standard_E8as_v5",
},
trinoProfile: {},
},
clusterType: "Trino",
computeProfile: {
availabilityZones: [
"1",
"2",
"3",
],
nodes: [
{
count: 2,
type: "Head",
vmSize: "Standard_E8as_v5",
},
{
count: 3,
type: "Worker",
vmSize: "Standard_E8as_v5",
},
],
},
location: "West US 2",
resourceGroupName: "hiloResourcegroup",
});
import pulumi
import pulumi_azure_native as azure_native
cluster_pool_cluster = azure_native.hdinsight.ClusterPoolCluster("clusterPoolCluster",
cluster_name="cluster1",
cluster_pool_name="clusterpool1",
cluster_profile={
"authorization_profile": {
"user_ids": [
"testuser1",
"testuser2",
],
},
"autoscale_profile": {
"autoscale_type": azure_native.hdinsight.AutoscaleType.SCHEDULE_BASED,
"enabled": True,
"graceful_decommission_timeout": 3600,
"load_based_config": {
"cooldown_period": 300,
"max_nodes": 20,
"min_nodes": 10,
"poll_interval": 60,
"scaling_rules": [
{
"action_type": azure_native.hdinsight.ScaleActionType.SCALEUP,
"comparison_rule": {
"operator": azure_native.hdinsight.ComparisonOperator.GREATER_THAN,
"threshold": 90,
},
"evaluation_count": 3,
"scaling_metric": "cpu",
},
{
"action_type": azure_native.hdinsight.ScaleActionType.SCALEDOWN,
"comparison_rule": {
"operator": azure_native.hdinsight.ComparisonOperator.LESS_THAN,
"threshold": 20,
},
"evaluation_count": 3,
"scaling_metric": "cpu",
},
],
},
"schedule_based_config": {
"default_count": 10,
"schedules": [
{
"count": 20,
"days": [azure_native.hdinsight.ScheduleDay.MONDAY],
"end_time": "12:00",
"start_time": "00:00",
},
{
"count": 25,
"days": [azure_native.hdinsight.ScheduleDay.SUNDAY],
"end_time": "12:00",
"start_time": "00:00",
},
],
"time_zone": "Cen. Australia Standard Time",
},
},
"cluster_version": "1.0.6",
"managed_identity_profile": {
"identity_list": [{
"client_id": "de91f1d8-767f-460a-ac11-3cf103f74b34",
"object_id": "40491351-c240-4042-91e0-f644a1d2b441",
"resource_id": "/subscriptions/subid/resourceGroups/hiloResourcegroup/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-msi",
"type": azure_native.hdinsight.ManagedIdentityType.CLUSTER,
}],
},
"oss_version": "0.410.0",
"ssh_profile": {
"count": 2,
"vm_size": "Standard_E8as_v5",
},
"trino_profile": {},
},
cluster_type="Trino",
compute_profile={
"availability_zones": [
"1",
"2",
"3",
],
"nodes": [
{
"count": 2,
"type": "Head",
"vm_size": "Standard_E8as_v5",
},
{
"count": 3,
"type": "Worker",
"vm_size": "Standard_E8as_v5",
},
],
},
location="West US 2",
resource_group_name="hiloResourcegroup")
package main
import (
hdinsight "github.com/pulumi/pulumi-azure-native-sdk/hdinsight/v3"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
_, err := hdinsight.NewClusterPoolCluster(ctx, "clusterPoolCluster", &hdinsight.ClusterPoolClusterArgs{
ClusterName: pulumi.String("cluster1"),
ClusterPoolName: pulumi.String("clusterpool1"),
ClusterProfile: &hdinsight.ClusterProfileArgs{
AuthorizationProfile: &hdinsight.AuthorizationProfileArgs{
UserIds: pulumi.StringArray{
pulumi.String("testuser1"),
pulumi.String("testuser2"),
},
},
AutoscaleProfile: &hdinsight.AutoscaleProfileArgs{
AutoscaleType: pulumi.String(hdinsight.AutoscaleTypeScheduleBased),
Enabled: pulumi.Bool(true),
GracefulDecommissionTimeout: pulumi.Int(3600),
LoadBasedConfig: &hdinsight.LoadBasedConfigArgs{
CooldownPeriod: pulumi.Int(300),
MaxNodes: pulumi.Int(20),
MinNodes: pulumi.Int(10),
PollInterval: pulumi.Int(60),
ScalingRules: hdinsight.ScalingRuleArray{
&hdinsight.ScalingRuleArgs{
ActionType: pulumi.String(hdinsight.ScaleActionTypeScaleup),
ComparisonRule: &hdinsight.ComparisonRuleArgs{
Operator: pulumi.String(hdinsight.ComparisonOperatorGreaterThan),
Threshold: pulumi.Float64(90),
},
EvaluationCount: pulumi.Int(3),
ScalingMetric: pulumi.String("cpu"),
},
&hdinsight.ScalingRuleArgs{
ActionType: pulumi.String(hdinsight.ScaleActionTypeScaledown),
ComparisonRule: &hdinsight.ComparisonRuleArgs{
Operator: pulumi.String(hdinsight.ComparisonOperatorLessThan),
Threshold: pulumi.Float64(20),
},
EvaluationCount: pulumi.Int(3),
ScalingMetric: pulumi.String("cpu"),
},
},
},
ScheduleBasedConfig: &hdinsight.ScheduleBasedConfigArgs{
DefaultCount: pulumi.Int(10),
Schedules: hdinsight.ScheduleArray{
&hdinsight.ScheduleArgs{
Count: pulumi.Int(20),
Days: pulumi.StringArray{
pulumi.String(hdinsight.ScheduleDayMonday),
},
EndTime: pulumi.String("12:00"),
StartTime: pulumi.String("00:00"),
},
&hdinsight.ScheduleArgs{
Count: pulumi.Int(25),
Days: pulumi.StringArray{
pulumi.String(hdinsight.ScheduleDaySunday),
},
EndTime: pulumi.String("12:00"),
StartTime: pulumi.String("00:00"),
},
},
TimeZone: pulumi.String("Cen. Australia Standard Time"),
},
},
ClusterVersion: pulumi.String("1.0.6"),
ManagedIdentityProfile: &hdinsight.ManagedIdentityProfileArgs{
IdentityList: hdinsight.ManagedIdentitySpecArray{
&hdinsight.ManagedIdentitySpecArgs{
ClientId: pulumi.String("de91f1d8-767f-460a-ac11-3cf103f74b34"),
ObjectId: pulumi.String("40491351-c240-4042-91e0-f644a1d2b441"),
ResourceId: pulumi.String("/subscriptions/subid/resourceGroups/hiloResourcegroup/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-msi"),
Type: pulumi.String(hdinsight.ManagedIdentityTypeCluster),
},
},
},
OssVersion: pulumi.String("0.410.0"),
SshProfile: &hdinsight.ClusterPoolSshProfileArgs{
Count: pulumi.Int(2),
VmSize: pulumi.String("Standard_E8as_v5"),
},
TrinoProfile: &hdinsight.TrinoProfileArgs{},
},
ClusterType: pulumi.String("Trino"),
ComputeProfile: &hdinsight.ClusterPoolComputeProfileArgs{
AvailabilityZones: pulumi.StringArray{
pulumi.String("1"),
pulumi.String("2"),
pulumi.String("3"),
},
Nodes: hdinsight.NodeProfileArray{
&hdinsight.NodeProfileArgs{
Count: pulumi.Int(2),
Type: pulumi.String("Head"),
VmSize: pulumi.String("Standard_E8as_v5"),
},
&hdinsight.NodeProfileArgs{
Count: pulumi.Int(3),
Type: pulumi.String("Worker"),
VmSize: pulumi.String("Standard_E8as_v5"),
},
},
},
Location: pulumi.String("West US 2"),
ResourceGroupName: pulumi.String("hiloResourcegroup"),
})
if err != nil {
return err
}
return nil
})
}
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using AzureNative = Pulumi.AzureNative;
return await Deployment.RunAsync(() =>
{
var clusterPoolCluster = new AzureNative.HDInsight.ClusterPoolCluster("clusterPoolCluster", new()
{
ClusterName = "cluster1",
ClusterPoolName = "clusterpool1",
ClusterProfile = new AzureNative.HDInsight.Inputs.ClusterProfileArgs
{
AuthorizationProfile = new AzureNative.HDInsight.Inputs.AuthorizationProfileArgs
{
UserIds = new[]
{
"testuser1",
"testuser2",
},
},
AutoscaleProfile = new AzureNative.HDInsight.Inputs.AutoscaleProfileArgs
{
AutoscaleType = AzureNative.HDInsight.AutoscaleType.ScheduleBased,
Enabled = true,
GracefulDecommissionTimeout = 3600,
LoadBasedConfig = new AzureNative.HDInsight.Inputs.LoadBasedConfigArgs
{
CooldownPeriod = 300,
MaxNodes = 20,
MinNodes = 10,
PollInterval = 60,
ScalingRules = new[]
{
new AzureNative.HDInsight.Inputs.ScalingRuleArgs
{
ActionType = AzureNative.HDInsight.ScaleActionType.Scaleup,
ComparisonRule = new AzureNative.HDInsight.Inputs.ComparisonRuleArgs
{
Operator = AzureNative.HDInsight.ComparisonOperator.GreaterThan,
Threshold = 90,
},
EvaluationCount = 3,
ScalingMetric = "cpu",
},
new AzureNative.HDInsight.Inputs.ScalingRuleArgs
{
ActionType = AzureNative.HDInsight.ScaleActionType.Scaledown,
ComparisonRule = new AzureNative.HDInsight.Inputs.ComparisonRuleArgs
{
Operator = AzureNative.HDInsight.ComparisonOperator.LessThan,
Threshold = 20,
},
EvaluationCount = 3,
ScalingMetric = "cpu",
},
},
},
ScheduleBasedConfig = new AzureNative.HDInsight.Inputs.ScheduleBasedConfigArgs
{
DefaultCount = 10,
Schedules = new[]
{
new AzureNative.HDInsight.Inputs.ScheduleArgs
{
Count = 20,
Days = new[]
{
AzureNative.HDInsight.ScheduleDay.Monday,
},
EndTime = "12:00",
StartTime = "00:00",
},
new AzureNative.HDInsight.Inputs.ScheduleArgs
{
Count = 25,
Days = new[]
{
AzureNative.HDInsight.ScheduleDay.Sunday,
},
EndTime = "12:00",
StartTime = "00:00",
},
},
TimeZone = "Cen. Australia Standard Time",
},
},
ClusterVersion = "1.0.6",
ManagedIdentityProfile = new AzureNative.HDInsight.Inputs.ManagedIdentityProfileArgs
{
IdentityList = new[]
{
new AzureNative.HDInsight.Inputs.ManagedIdentitySpecArgs
{
ClientId = "de91f1d8-767f-460a-ac11-3cf103f74b34",
ObjectId = "40491351-c240-4042-91e0-f644a1d2b441",
ResourceId = "/subscriptions/subid/resourceGroups/hiloResourcegroup/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-msi",
Type = AzureNative.HDInsight.ManagedIdentityType.Cluster,
},
},
},
OssVersion = "0.410.0",
SshProfile = new AzureNative.HDInsight.Inputs.ClusterPoolSshProfileArgs
{
Count = 2,
VmSize = "Standard_E8as_v5",
},
TrinoProfile = null,
},
ClusterType = "Trino",
ComputeProfile = new AzureNative.HDInsight.Inputs.ClusterPoolComputeProfileArgs
{
AvailabilityZones = new[]
{
"1",
"2",
"3",
},
Nodes = new[]
{
new AzureNative.HDInsight.Inputs.NodeProfileArgs
{
Count = 2,
Type = "Head",
VmSize = "Standard_E8as_v5",
},
new AzureNative.HDInsight.Inputs.NodeProfileArgs
{
Count = 3,
Type = "Worker",
VmSize = "Standard_E8as_v5",
},
},
},
Location = "West US 2",
ResourceGroupName = "hiloResourcegroup",
});
});
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.azurenative.hdinsight.ClusterPoolCluster;
import com.pulumi.azurenative.hdinsight.ClusterPoolClusterArgs;
import com.pulumi.azurenative.hdinsight.inputs.ClusterProfileArgs;
import com.pulumi.azurenative.hdinsight.inputs.AuthorizationProfileArgs;
import com.pulumi.azurenative.hdinsight.inputs.AutoscaleProfileArgs;
import com.pulumi.azurenative.hdinsight.inputs.LoadBasedConfigArgs;
import com.pulumi.azurenative.hdinsight.inputs.ScheduleBasedConfigArgs;
import com.pulumi.azurenative.hdinsight.inputs.ManagedIdentityProfileArgs;
import com.pulumi.azurenative.hdinsight.inputs.ClusterPoolSshProfileArgs;
import com.pulumi.azurenative.hdinsight.inputs.TrinoProfileArgs;
import com.pulumi.azurenative.hdinsight.inputs.ClusterPoolComputeProfileArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var clusterPoolCluster = new ClusterPoolCluster("clusterPoolCluster", ClusterPoolClusterArgs.builder()
.clusterName("cluster1")
.clusterPoolName("clusterpool1")
.clusterProfile(ClusterProfileArgs.builder()
.authorizationProfile(AuthorizationProfileArgs.builder()
.userIds(
"testuser1",
"testuser2")
.build())
.autoscaleProfile(AutoscaleProfileArgs.builder()
.autoscaleType("ScheduleBased")
.enabled(true)
.gracefulDecommissionTimeout(3600)
.loadBasedConfig(LoadBasedConfigArgs.builder()
.cooldownPeriod(300)
.maxNodes(20)
.minNodes(10)
.pollInterval(60)
.scalingRules(
ScalingRuleArgs.builder()
.actionType("scaleup")
.comparisonRule(ComparisonRuleArgs.builder()
.operator("greaterThan")
.threshold(90.0)
.build())
.evaluationCount(3)
.scalingMetric("cpu")
.build(),
ScalingRuleArgs.builder()
.actionType("scaledown")
.comparisonRule(ComparisonRuleArgs.builder()
.operator("lessThan")
.threshold(20.0)
.build())
.evaluationCount(3)
.scalingMetric("cpu")
.build())
.build())
.scheduleBasedConfig(ScheduleBasedConfigArgs.builder()
.defaultCount(10)
.schedules(
ScheduleArgs.builder()
.count(20)
.days("Monday")
.endTime("12:00")
.startTime("00:00")
.build(),
ScheduleArgs.builder()
.count(25)
.days("Sunday")
.endTime("12:00")
.startTime("00:00")
.build())
.timeZone("Cen. Australia Standard Time")
.build())
.build())
.clusterVersion("1.0.6")
.managedIdentityProfile(ManagedIdentityProfileArgs.builder()
.identityList(ManagedIdentitySpecArgs.builder()
.clientId("de91f1d8-767f-460a-ac11-3cf103f74b34")
.objectId("40491351-c240-4042-91e0-f644a1d2b441")
.resourceId("/subscriptions/subid/resourceGroups/hiloResourcegroup/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-msi")
.type("cluster")
.build())
.build())
.ossVersion("0.410.0")
.sshProfile(ClusterPoolSshProfileArgs.builder()
.count(2)
.vmSize("Standard_E8as_v5")
.build())
.trinoProfile(TrinoProfileArgs.builder()
.build())
.build())
.clusterType("Trino")
.computeProfile(ClusterPoolComputeProfileArgs.builder()
.availabilityZones(
"1",
"2",
"3")
.nodes(
NodeProfileArgs.builder()
.count(2)
.type("Head")
.vmSize("Standard_E8as_v5")
.build(),
NodeProfileArgs.builder()
.count(3)
.type("Worker")
.vmSize("Standard_E8as_v5")
.build())
.build())
.location("West US 2")
.resourceGroupName("hiloResourcegroup")
.build());
}
}
resources:
clusterPoolCluster:
type: azure-native:hdinsight:ClusterPoolCluster
properties:
clusterName: cluster1
clusterPoolName: clusterpool1
clusterProfile:
authorizationProfile:
userIds:
- testuser1
- testuser2
autoscaleProfile:
autoscaleType: ScheduleBased
enabled: true
gracefulDecommissionTimeout: 3600
loadBasedConfig:
cooldownPeriod: 300
maxNodes: 20
minNodes: 10
pollInterval: 60
scalingRules:
- actionType: scaleup
comparisonRule:
operator: greaterThan
threshold: 90
evaluationCount: 3
scalingMetric: cpu
- actionType: scaledown
comparisonRule:
operator: lessThan
threshold: 20
evaluationCount: 3
scalingMetric: cpu
scheduleBasedConfig:
defaultCount: 10
schedules:
- count: 20
days:
- Monday
endTime: 12:00
startTime: 00:00
- count: 25
days:
- Sunday
endTime: 12:00
startTime: 00:00
timeZone: Cen. Australia Standard Time
clusterVersion: 1.0.6
managedIdentityProfile:
identityList:
- clientId: de91f1d8-767f-460a-ac11-3cf103f74b34
objectId: 40491351-c240-4042-91e0-f644a1d2b441
resourceId: /subscriptions/subid/resourceGroups/hiloResourcegroup/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-msi
type: cluster
ossVersion: 0.410.0
sshProfile:
count: 2
vmSize: Standard_E8as_v5
trinoProfile: {}
clusterType: Trino
computeProfile:
availabilityZones:
- '1'
- '2'
- '3'
nodes:
- count: 2
type: Head
vmSize: Standard_E8as_v5
- count: 3
type: Worker
vmSize: Standard_E8as_v5
location: West US 2
resourceGroupName: hiloResourcegroup
The autoscaleProfile enables automatic capacity adjustment. The loadBasedConfig defines scaling rules that trigger when CPU crosses thresholds; the scheduleBasedConfig sets node counts for specific days and times. The computeProfile specifies the node types (Head, Worker) and VM sizes that form the cluster’s compute capacity.
Configure Apache Ranger for access control
Data governance requirements often mandate centralized access control and audit logging. Apache Ranger provides policy-based authorization and tracks all data access for compliance.
import * as pulumi from "@pulumi/pulumi";
import * as azure_native from "@pulumi/azure-native";
const clusterPoolCluster = new azure_native.hdinsight.ClusterPoolCluster("clusterPoolCluster", {
clusterName: "cluster1",
clusterPoolName: "clusterpool1",
clusterProfile: {
authorizationProfile: {
userIds: [
"testuser1",
"testuser2",
],
},
clusterVersion: "0.0.1",
managedIdentityProfile: {
identityList: [{
clientId: "de91f1d8-767f-460a-ac11-3cf103f74b34",
objectId: "40491351-c240-4042-91e0-f644a1d2b441",
resourceId: "/subscriptions/subid/resourceGroups/hiloResourcegroup/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-msi",
type: azure_native.hdinsight.ManagedIdentityType.Cluster,
}],
},
ossVersion: "2.2.3",
rangerProfile: {
rangerAdmin: {
admins: [
"testuser1@contoso.com",
"testuser2@contoso.com",
],
database: {
host: "testsqlserver.database.windows.net",
name: "testdb",
passwordSecretRef: "https://testkv.vault.azure.net/secrets/mysecret/5df6584d9c25418c8d900240aa6c3452",
username: "admin",
},
},
rangerAudit: {
storageAccount: "https://teststorage.blob.core.windows.net/testblob",
},
rangerUsersync: {
enabled: true,
groups: [
"0a53828f-36c9-44c3-be3d-99a7fce977ad",
"13be6971-79db-4f33-9d41-b25589ca25ac",
],
mode: azure_native.hdinsight.RangerUsersyncMode.Automatic,
users: [
"testuser1@contoso.com",
"testuser2@contoso.com",
],
},
},
},
clusterType: "ranger",
computeProfile: {
availabilityZones: [
"1",
"2",
"3",
],
nodes: [{
count: 2,
type: "head",
vmSize: "Standard_D3_v2",
}],
},
location: "West US 2",
resourceGroupName: "hiloResourcegroup",
});
import pulumi
import pulumi_azure_native as azure_native
cluster_pool_cluster = azure_native.hdinsight.ClusterPoolCluster("clusterPoolCluster",
cluster_name="cluster1",
cluster_pool_name="clusterpool1",
cluster_profile={
"authorization_profile": {
"user_ids": [
"testuser1",
"testuser2",
],
},
"cluster_version": "0.0.1",
"managed_identity_profile": {
"identity_list": [{
"client_id": "de91f1d8-767f-460a-ac11-3cf103f74b34",
"object_id": "40491351-c240-4042-91e0-f644a1d2b441",
"resource_id": "/subscriptions/subid/resourceGroups/hiloResourcegroup/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-msi",
"type": azure_native.hdinsight.ManagedIdentityType.CLUSTER,
}],
},
"oss_version": "2.2.3",
"ranger_profile": {
"ranger_admin": {
"admins": [
"testuser1@contoso.com",
"testuser2@contoso.com",
],
"database": {
"host": "testsqlserver.database.windows.net",
"name": "testdb",
"password_secret_ref": "https://testkv.vault.azure.net/secrets/mysecret/5df6584d9c25418c8d900240aa6c3452",
"username": "admin",
},
},
"ranger_audit": {
"storage_account": "https://teststorage.blob.core.windows.net/testblob",
},
"ranger_usersync": {
"enabled": True,
"groups": [
"0a53828f-36c9-44c3-be3d-99a7fce977ad",
"13be6971-79db-4f33-9d41-b25589ca25ac",
],
"mode": azure_native.hdinsight.RangerUsersyncMode.AUTOMATIC,
"users": [
"testuser1@contoso.com",
"testuser2@contoso.com",
],
},
},
},
cluster_type="ranger",
compute_profile={
"availability_zones": [
"1",
"2",
"3",
],
"nodes": [{
"count": 2,
"type": "head",
"vm_size": "Standard_D3_v2",
}],
},
location="West US 2",
resource_group_name="hiloResourcegroup")
package main
import (
hdinsight "github.com/pulumi/pulumi-azure-native-sdk/hdinsight/v3"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
_, err := hdinsight.NewClusterPoolCluster(ctx, "clusterPoolCluster", &hdinsight.ClusterPoolClusterArgs{
ClusterName: pulumi.String("cluster1"),
ClusterPoolName: pulumi.String("clusterpool1"),
ClusterProfile: &hdinsight.ClusterProfileArgs{
AuthorizationProfile: &hdinsight.AuthorizationProfileArgs{
UserIds: pulumi.StringArray{
pulumi.String("testuser1"),
pulumi.String("testuser2"),
},
},
ClusterVersion: pulumi.String("0.0.1"),
ManagedIdentityProfile: &hdinsight.ManagedIdentityProfileArgs{
IdentityList: hdinsight.ManagedIdentitySpecArray{
&hdinsight.ManagedIdentitySpecArgs{
ClientId: pulumi.String("de91f1d8-767f-460a-ac11-3cf103f74b34"),
ObjectId: pulumi.String("40491351-c240-4042-91e0-f644a1d2b441"),
ResourceId: pulumi.String("/subscriptions/subid/resourceGroups/hiloResourcegroup/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-msi"),
Type: pulumi.String(hdinsight.ManagedIdentityTypeCluster),
},
},
},
OssVersion: pulumi.String("2.2.3"),
RangerProfile: &hdinsight.RangerProfileArgs{
RangerAdmin: &hdinsight.RangerAdminSpecArgs{
Admins: pulumi.StringArray{
pulumi.String("testuser1@contoso.com"),
pulumi.String("testuser2@contoso.com"),
},
Database: &hdinsight.RangerAdminSpecDatabaseArgs{
Host: pulumi.String("testsqlserver.database.windows.net"),
Name: pulumi.String("testdb"),
PasswordSecretRef: pulumi.String("https://testkv.vault.azure.net/secrets/mysecret/5df6584d9c25418c8d900240aa6c3452"),
Username: pulumi.String("admin"),
},
},
RangerAudit: &hdinsight.RangerAuditSpecArgs{
StorageAccount: pulumi.String("https://teststorage.blob.core.windows.net/testblob"),
},
RangerUsersync: &hdinsight.RangerUsersyncSpecArgs{
Enabled: pulumi.Bool(true),
Groups: pulumi.StringArray{
pulumi.String("0a53828f-36c9-44c3-be3d-99a7fce977ad"),
pulumi.String("13be6971-79db-4f33-9d41-b25589ca25ac"),
},
Mode: pulumi.String(hdinsight.RangerUsersyncModeAutomatic),
Users: pulumi.StringArray{
pulumi.String("testuser1@contoso.com"),
pulumi.String("testuser2@contoso.com"),
},
},
},
},
ClusterType: pulumi.String("ranger"),
ComputeProfile: &hdinsight.ClusterPoolComputeProfileArgs{
AvailabilityZones: pulumi.StringArray{
pulumi.String("1"),
pulumi.String("2"),
pulumi.String("3"),
},
Nodes: hdinsight.NodeProfileArray{
&hdinsight.NodeProfileArgs{
Count: pulumi.Int(2),
Type: pulumi.String("head"),
VmSize: pulumi.String("Standard_D3_v2"),
},
},
},
Location: pulumi.String("West US 2"),
ResourceGroupName: pulumi.String("hiloResourcegroup"),
})
if err != nil {
return err
}
return nil
})
}
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using AzureNative = Pulumi.AzureNative;
return await Deployment.RunAsync(() =>
{
var clusterPoolCluster = new AzureNative.HDInsight.ClusterPoolCluster("clusterPoolCluster", new()
{
ClusterName = "cluster1",
ClusterPoolName = "clusterpool1",
ClusterProfile = new AzureNative.HDInsight.Inputs.ClusterProfileArgs
{
AuthorizationProfile = new AzureNative.HDInsight.Inputs.AuthorizationProfileArgs
{
UserIds = new[]
{
"testuser1",
"testuser2",
},
},
ClusterVersion = "0.0.1",
ManagedIdentityProfile = new AzureNative.HDInsight.Inputs.ManagedIdentityProfileArgs
{
IdentityList = new[]
{
new AzureNative.HDInsight.Inputs.ManagedIdentitySpecArgs
{
ClientId = "de91f1d8-767f-460a-ac11-3cf103f74b34",
ObjectId = "40491351-c240-4042-91e0-f644a1d2b441",
ResourceId = "/subscriptions/subid/resourceGroups/hiloResourcegroup/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-msi",
Type = AzureNative.HDInsight.ManagedIdentityType.Cluster,
},
},
},
OssVersion = "2.2.3",
RangerProfile = new AzureNative.HDInsight.Inputs.RangerProfileArgs
{
RangerAdmin = new AzureNative.HDInsight.Inputs.RangerAdminSpecArgs
{
Admins = new[]
{
"testuser1@contoso.com",
"testuser2@contoso.com",
},
Database = new AzureNative.HDInsight.Inputs.RangerAdminSpecDatabaseArgs
{
Host = "testsqlserver.database.windows.net",
Name = "testdb",
PasswordSecretRef = "https://testkv.vault.azure.net/secrets/mysecret/5df6584d9c25418c8d900240aa6c3452",
Username = "admin",
},
},
RangerAudit = new AzureNative.HDInsight.Inputs.RangerAuditSpecArgs
{
StorageAccount = "https://teststorage.blob.core.windows.net/testblob",
},
RangerUsersync = new AzureNative.HDInsight.Inputs.RangerUsersyncSpecArgs
{
Enabled = true,
Groups = new[]
{
"0a53828f-36c9-44c3-be3d-99a7fce977ad",
"13be6971-79db-4f33-9d41-b25589ca25ac",
},
Mode = AzureNative.HDInsight.RangerUsersyncMode.Automatic,
Users = new[]
{
"testuser1@contoso.com",
"testuser2@contoso.com",
},
},
},
},
ClusterType = "ranger",
ComputeProfile = new AzureNative.HDInsight.Inputs.ClusterPoolComputeProfileArgs
{
AvailabilityZones = new[]
{
"1",
"2",
"3",
},
Nodes = new[]
{
new AzureNative.HDInsight.Inputs.NodeProfileArgs
{
Count = 2,
Type = "head",
VmSize = "Standard_D3_v2",
},
},
},
Location = "West US 2",
ResourceGroupName = "hiloResourcegroup",
});
});
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.azurenative.hdinsight.ClusterPoolCluster;
import com.pulumi.azurenative.hdinsight.ClusterPoolClusterArgs;
import com.pulumi.azurenative.hdinsight.inputs.ClusterProfileArgs;
import com.pulumi.azurenative.hdinsight.inputs.AuthorizationProfileArgs;
import com.pulumi.azurenative.hdinsight.inputs.ManagedIdentityProfileArgs;
import com.pulumi.azurenative.hdinsight.inputs.RangerProfileArgs;
import com.pulumi.azurenative.hdinsight.inputs.RangerAdminSpecArgs;
import com.pulumi.azurenative.hdinsight.inputs.RangerAdminSpecDatabaseArgs;
import com.pulumi.azurenative.hdinsight.inputs.RangerAuditSpecArgs;
import com.pulumi.azurenative.hdinsight.inputs.RangerUsersyncSpecArgs;
import com.pulumi.azurenative.hdinsight.inputs.ClusterPoolComputeProfileArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var clusterPoolCluster = new ClusterPoolCluster("clusterPoolCluster", ClusterPoolClusterArgs.builder()
.clusterName("cluster1")
.clusterPoolName("clusterpool1")
.clusterProfile(ClusterProfileArgs.builder()
.authorizationProfile(AuthorizationProfileArgs.builder()
.userIds(
"testuser1",
"testuser2")
.build())
.clusterVersion("0.0.1")
.managedIdentityProfile(ManagedIdentityProfileArgs.builder()
.identityList(ManagedIdentitySpecArgs.builder()
.clientId("de91f1d8-767f-460a-ac11-3cf103f74b34")
.objectId("40491351-c240-4042-91e0-f644a1d2b441")
.resourceId("/subscriptions/subid/resourceGroups/hiloResourcegroup/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-msi")
.type("cluster")
.build())
.build())
.ossVersion("2.2.3")
.rangerProfile(RangerProfileArgs.builder()
.rangerAdmin(RangerAdminSpecArgs.builder()
.admins(
"testuser1@contoso.com",
"testuser2@contoso.com")
.database(RangerAdminSpecDatabaseArgs.builder()
.host("testsqlserver.database.windows.net")
.name("testdb")
.passwordSecretRef("https://testkv.vault.azure.net/secrets/mysecret/5df6584d9c25418c8d900240aa6c3452")
.username("admin")
.build())
.build())
.rangerAudit(RangerAuditSpecArgs.builder()
.storageAccount("https://teststorage.blob.core.windows.net/testblob")
.build())
.rangerUsersync(RangerUsersyncSpecArgs.builder()
.enabled(true)
.groups(
"0a53828f-36c9-44c3-be3d-99a7fce977ad",
"13be6971-79db-4f33-9d41-b25589ca25ac")
.mode("automatic")
.users(
"testuser1@contoso.com",
"testuser2@contoso.com")
.build())
.build())
.build())
.clusterType("ranger")
.computeProfile(ClusterPoolComputeProfileArgs.builder()
.availabilityZones(
"1",
"2",
"3")
.nodes(NodeProfileArgs.builder()
.count(2)
.type("head")
.vmSize("Standard_D3_v2")
.build())
.build())
.location("West US 2")
.resourceGroupName("hiloResourcegroup")
.build());
}
}
resources:
clusterPoolCluster:
type: azure-native:hdinsight:ClusterPoolCluster
properties:
clusterName: cluster1
clusterPoolName: clusterpool1
clusterProfile:
authorizationProfile:
userIds:
- testuser1
- testuser2
clusterVersion: 0.0.1
managedIdentityProfile:
identityList:
- clientId: de91f1d8-767f-460a-ac11-3cf103f74b34
objectId: 40491351-c240-4042-91e0-f644a1d2b441
resourceId: /subscriptions/subid/resourceGroups/hiloResourcegroup/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-msi
type: cluster
ossVersion: 2.2.3
rangerProfile:
rangerAdmin:
admins:
- testuser1@contoso.com
- testuser2@contoso.com
database:
host: testsqlserver.database.windows.net
name: testdb
passwordSecretRef: https://testkv.vault.azure.net/secrets/mysecret/5df6584d9c25418c8d900240aa6c3452
username: admin
rangerAudit:
storageAccount: https://teststorage.blob.core.windows.net/testblob
rangerUsersync:
enabled: true
groups:
- 0a53828f-36c9-44c3-be3d-99a7fce977ad
- 13be6971-79db-4f33-9d41-b25589ca25ac
mode: automatic
users:
- testuser1@contoso.com
- testuser2@contoso.com
clusterType: ranger
computeProfile:
availabilityZones:
- '1'
- '2'
- '3'
nodes:
- count: 2
type: head
vmSize: Standard_D3_v2
location: West US 2
resourceGroupName: hiloResourcegroup
The rangerProfile configures three components: rangerAdmin connects to a SQL database for policy storage, rangerAudit writes access logs to a storage account, and rangerUsersync synchronizes users and groups from Azure AD. The database passwordSecretRef points to a Key Vault secret rather than embedding credentials in code.
Deploy Spark with custom service configurations
Spark deployments typically require tuning service-level settings like event logging, storage integration, and YARN resource management.
import * as pulumi from "@pulumi/pulumi";
import * as azure_native from "@pulumi/azure-native";
const clusterPoolCluster = new azure_native.hdinsight.ClusterPoolCluster("clusterPoolCluster", {
clusterName: "cluster1",
clusterPoolName: "clusterpool1",
clusterProfile: {
authorizationProfile: {
userIds: [
"testuser1",
"testuser2",
],
},
clusterVersion: "0.0.1",
managedIdentityProfile: {
identityList: [{
clientId: "de91f1d8-767f-460a-ac11-3cf103f74b34",
objectId: "40491351-c240-4042-91e0-f644a1d2b441",
resourceId: "/subscriptions/subid/resourceGroups/hiloResourcegroup/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-msi",
type: azure_native.hdinsight.ManagedIdentityType.Cluster,
}],
},
ossVersion: "2.2.3",
serviceConfigsProfiles: [
{
configs: [{
component: "spark-config",
files: [{
fileName: "spark-defaults.conf",
values: {
"spark.eventLog.enabled": "true",
},
}],
}],
serviceName: "spark-service",
},
{
configs: [{
component: "yarn-config",
files: [
{
fileName: "core-site.xml",
values: {
"fs.defaultFS": "wasb://testcontainer@teststorage.dfs.core.windows.net/",
"storage.container": "testcontainer",
"storage.key": "test key",
"storage.name": "teststorage",
"storage.protocol": "wasb",
},
},
{
fileName: "yarn-site.xml",
values: {
"yarn.webapp.ui2.enable": "false",
},
},
],
}],
serviceName: "yarn-service",
},
],
sparkProfile: {},
sshProfile: {
count: 2,
vmSize: "Standard_D3_v2",
},
},
clusterType: "spark",
computeProfile: {
availabilityZones: [
"1",
"2",
"3",
],
nodes: [{
count: 4,
type: "worker",
vmSize: "Standard_D3_v2",
}],
},
location: "West US 2",
resourceGroupName: "hiloResourcegroup",
});
import pulumi
import pulumi_azure_native as azure_native
cluster_pool_cluster = azure_native.hdinsight.ClusterPoolCluster("clusterPoolCluster",
cluster_name="cluster1",
cluster_pool_name="clusterpool1",
cluster_profile={
"authorization_profile": {
"user_ids": [
"testuser1",
"testuser2",
],
},
"cluster_version": "0.0.1",
"managed_identity_profile": {
"identity_list": [{
"client_id": "de91f1d8-767f-460a-ac11-3cf103f74b34",
"object_id": "40491351-c240-4042-91e0-f644a1d2b441",
"resource_id": "/subscriptions/subid/resourceGroups/hiloResourcegroup/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-msi",
"type": azure_native.hdinsight.ManagedIdentityType.CLUSTER,
}],
},
"oss_version": "2.2.3",
"service_configs_profiles": [
{
"configs": [{
"component": "spark-config",
"files": [{
"file_name": "spark-defaults.conf",
"values": {
"spark.eventLog.enabled": "true",
},
}],
}],
"service_name": "spark-service",
},
{
"configs": [{
"component": "yarn-config",
"files": [
{
"file_name": "core-site.xml",
"values": {
"fs.defaultFS": "wasb://testcontainer@teststorage.dfs.core.windows.net/",
"storage.container": "testcontainer",
"storage.key": "test key",
"storage.name": "teststorage",
"storage.protocol": "wasb",
},
},
{
"file_name": "yarn-site.xml",
"values": {
"yarn.webapp.ui2.enable": "false",
},
},
],
}],
"service_name": "yarn-service",
},
],
"spark_profile": {},
"ssh_profile": {
"count": 2,
"vm_size": "Standard_D3_v2",
},
},
cluster_type="spark",
compute_profile={
"availability_zones": [
"1",
"2",
"3",
],
"nodes": [{
"count": 4,
"type": "worker",
"vm_size": "Standard_D3_v2",
}],
},
location="West US 2",
resource_group_name="hiloResourcegroup")
package main
import (
hdinsight "github.com/pulumi/pulumi-azure-native-sdk/hdinsight/v3"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
_, err := hdinsight.NewClusterPoolCluster(ctx, "clusterPoolCluster", &hdinsight.ClusterPoolClusterArgs{
ClusterName: pulumi.String("cluster1"),
ClusterPoolName: pulumi.String("clusterpool1"),
ClusterProfile: &hdinsight.ClusterProfileArgs{
AuthorizationProfile: &hdinsight.AuthorizationProfileArgs{
UserIds: pulumi.StringArray{
pulumi.String("testuser1"),
pulumi.String("testuser2"),
},
},
ClusterVersion: pulumi.String("0.0.1"),
ManagedIdentityProfile: &hdinsight.ManagedIdentityProfileArgs{
IdentityList: hdinsight.ManagedIdentitySpecArray{
&hdinsight.ManagedIdentitySpecArgs{
ClientId: pulumi.String("de91f1d8-767f-460a-ac11-3cf103f74b34"),
ObjectId: pulumi.String("40491351-c240-4042-91e0-f644a1d2b441"),
ResourceId: pulumi.String("/subscriptions/subid/resourceGroups/hiloResourcegroup/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-msi"),
Type: pulumi.String(hdinsight.ManagedIdentityTypeCluster),
},
},
},
OssVersion: pulumi.String("2.2.3"),
ServiceConfigsProfiles: hdinsight.ClusterServiceConfigsProfileArray{
&hdinsight.ClusterServiceConfigsProfileArgs{
Configs: hdinsight.ClusterServiceConfigArray{
&hdinsight.ClusterServiceConfigArgs{
Component: pulumi.String("spark-config"),
Files: hdinsight.ClusterConfigFileArray{
&hdinsight.ClusterConfigFileArgs{
FileName: pulumi.String("spark-defaults.conf"),
Values: pulumi.StringMap{
"spark.eventLog.enabled": pulumi.String("true"),
},
},
},
},
},
ServiceName: pulumi.String("spark-service"),
},
&hdinsight.ClusterServiceConfigsProfileArgs{
Configs: hdinsight.ClusterServiceConfigArray{
&hdinsight.ClusterServiceConfigArgs{
Component: pulumi.String("yarn-config"),
Files: hdinsight.ClusterConfigFileArray{
&hdinsight.ClusterConfigFileArgs{
FileName: pulumi.String("core-site.xml"),
Values: pulumi.StringMap{
"fs.defaultFS": pulumi.String("wasb://testcontainer@teststorage.dfs.core.windows.net/"),
"storage.container": pulumi.String("testcontainer"),
"storage.key": pulumi.String("test key"),
"storage.name": pulumi.String("teststorage"),
"storage.protocol": pulumi.String("wasb"),
},
},
&hdinsight.ClusterConfigFileArgs{
FileName: pulumi.String("yarn-site.xml"),
Values: pulumi.StringMap{
"yarn.webapp.ui2.enable": pulumi.String("false"),
},
},
},
},
},
ServiceName: pulumi.String("yarn-service"),
},
},
SparkProfile: &hdinsight.SparkProfileArgs{},
SshProfile: &hdinsight.ClusterPoolSshProfileArgs{
Count: pulumi.Int(2),
VmSize: pulumi.String("Standard_D3_v2"),
},
},
ClusterType: pulumi.String("spark"),
ComputeProfile: &hdinsight.ClusterPoolComputeProfileArgs{
AvailabilityZones: pulumi.StringArray{
pulumi.String("1"),
pulumi.String("2"),
pulumi.String("3"),
},
Nodes: hdinsight.NodeProfileArray{
&hdinsight.NodeProfileArgs{
Count: pulumi.Int(4),
Type: pulumi.String("worker"),
VmSize: pulumi.String("Standard_D3_v2"),
},
},
},
Location: pulumi.String("West US 2"),
ResourceGroupName: pulumi.String("hiloResourcegroup"),
})
if err != nil {
return err
}
return nil
})
}
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using AzureNative = Pulumi.AzureNative;
return await Deployment.RunAsync(() =>
{
var clusterPoolCluster = new AzureNative.HDInsight.ClusterPoolCluster("clusterPoolCluster", new()
{
ClusterName = "cluster1",
ClusterPoolName = "clusterpool1",
ClusterProfile = new AzureNative.HDInsight.Inputs.ClusterProfileArgs
{
AuthorizationProfile = new AzureNative.HDInsight.Inputs.AuthorizationProfileArgs
{
UserIds = new[]
{
"testuser1",
"testuser2",
},
},
ClusterVersion = "0.0.1",
ManagedIdentityProfile = new AzureNative.HDInsight.Inputs.ManagedIdentityProfileArgs
{
IdentityList = new[]
{
new AzureNative.HDInsight.Inputs.ManagedIdentitySpecArgs
{
ClientId = "de91f1d8-767f-460a-ac11-3cf103f74b34",
ObjectId = "40491351-c240-4042-91e0-f644a1d2b441",
ResourceId = "/subscriptions/subid/resourceGroups/hiloResourcegroup/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-msi",
Type = AzureNative.HDInsight.ManagedIdentityType.Cluster,
},
},
},
OssVersion = "2.2.3",
ServiceConfigsProfiles = new[]
{
new AzureNative.HDInsight.Inputs.ClusterServiceConfigsProfileArgs
{
Configs = new[]
{
new AzureNative.HDInsight.Inputs.ClusterServiceConfigArgs
{
Component = "spark-config",
Files = new[]
{
new AzureNative.HDInsight.Inputs.ClusterConfigFileArgs
{
FileName = "spark-defaults.conf",
Values =
{
{ "spark.eventLog.enabled", "true" },
},
},
},
},
},
ServiceName = "spark-service",
},
new AzureNative.HDInsight.Inputs.ClusterServiceConfigsProfileArgs
{
Configs = new[]
{
new AzureNative.HDInsight.Inputs.ClusterServiceConfigArgs
{
Component = "yarn-config",
Files = new[]
{
new AzureNative.HDInsight.Inputs.ClusterConfigFileArgs
{
FileName = "core-site.xml",
Values =
{
{ "fs.defaultFS", "wasb://testcontainer@teststorage.dfs.core.windows.net/" },
{ "storage.container", "testcontainer" },
{ "storage.key", "test key" },
{ "storage.name", "teststorage" },
{ "storage.protocol", "wasb" },
},
},
new AzureNative.HDInsight.Inputs.ClusterConfigFileArgs
{
FileName = "yarn-site.xml",
Values =
{
{ "yarn.webapp.ui2.enable", "false" },
},
},
},
},
},
ServiceName = "yarn-service",
},
},
SparkProfile = null,
SshProfile = new AzureNative.HDInsight.Inputs.ClusterPoolSshProfileArgs
{
Count = 2,
VmSize = "Standard_D3_v2",
},
},
ClusterType = "spark",
ComputeProfile = new AzureNative.HDInsight.Inputs.ClusterPoolComputeProfileArgs
{
AvailabilityZones = new[]
{
"1",
"2",
"3",
},
Nodes = new[]
{
new AzureNative.HDInsight.Inputs.NodeProfileArgs
{
Count = 4,
Type = "worker",
VmSize = "Standard_D3_v2",
},
},
},
Location = "West US 2",
ResourceGroupName = "hiloResourcegroup",
});
});
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.azurenative.hdinsight.ClusterPoolCluster;
import com.pulumi.azurenative.hdinsight.ClusterPoolClusterArgs;
import com.pulumi.azurenative.hdinsight.inputs.ClusterProfileArgs;
import com.pulumi.azurenative.hdinsight.inputs.AuthorizationProfileArgs;
import com.pulumi.azurenative.hdinsight.inputs.ManagedIdentityProfileArgs;
import com.pulumi.azurenative.hdinsight.inputs.SparkProfileArgs;
import com.pulumi.azurenative.hdinsight.inputs.ClusterPoolSshProfileArgs;
import com.pulumi.azurenative.hdinsight.inputs.ClusterPoolComputeProfileArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var clusterPoolCluster = new ClusterPoolCluster("clusterPoolCluster", ClusterPoolClusterArgs.builder()
.clusterName("cluster1")
.clusterPoolName("clusterpool1")
.clusterProfile(ClusterProfileArgs.builder()
.authorizationProfile(AuthorizationProfileArgs.builder()
.userIds(
"testuser1",
"testuser2")
.build())
.clusterVersion("0.0.1")
.managedIdentityProfile(ManagedIdentityProfileArgs.builder()
.identityList(ManagedIdentitySpecArgs.builder()
.clientId("de91f1d8-767f-460a-ac11-3cf103f74b34")
.objectId("40491351-c240-4042-91e0-f644a1d2b441")
.resourceId("/subscriptions/subid/resourceGroups/hiloResourcegroup/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-msi")
.type("cluster")
.build())
.build())
.ossVersion("2.2.3")
.serviceConfigsProfiles(
ClusterServiceConfigsProfileArgs.builder()
.configs(ClusterServiceConfigArgs.builder()
.component("spark-config")
.files(ClusterConfigFileArgs.builder()
.fileName("spark-defaults.conf")
.values(Map.of("spark.eventLog.enabled", "true"))
.build())
.build())
.serviceName("spark-service")
.build(),
ClusterServiceConfigsProfileArgs.builder()
.configs(ClusterServiceConfigArgs.builder()
.component("yarn-config")
.files(
ClusterConfigFileArgs.builder()
.fileName("core-site.xml")
.values(Map.ofEntries(
Map.entry("fs.defaultFS", "wasb://testcontainer@teststorage.dfs.core.windows.net/"),
Map.entry("storage.container", "testcontainer"),
Map.entry("storage.key", "test key"),
Map.entry("storage.name", "teststorage"),
Map.entry("storage.protocol", "wasb")
))
.build(),
ClusterConfigFileArgs.builder()
.fileName("yarn-site.xml")
.values(Map.of("yarn.webapp.ui2.enable", "false"))
.build())
.build())
.serviceName("yarn-service")
.build())
.sparkProfile(SparkProfileArgs.builder()
.build())
.sshProfile(ClusterPoolSshProfileArgs.builder()
.count(2)
.vmSize("Standard_D3_v2")
.build())
.build())
.clusterType("spark")
.computeProfile(ClusterPoolComputeProfileArgs.builder()
.availabilityZones(
"1",
"2",
"3")
.nodes(NodeProfileArgs.builder()
.count(4)
.type("worker")
.vmSize("Standard_D3_v2")
.build())
.build())
.location("West US 2")
.resourceGroupName("hiloResourcegroup")
.build());
}
}
resources:
clusterPoolCluster:
type: azure-native:hdinsight:ClusterPoolCluster
properties:
clusterName: cluster1
clusterPoolName: clusterpool1
clusterProfile:
authorizationProfile:
userIds:
- testuser1
- testuser2
clusterVersion: 0.0.1
managedIdentityProfile:
identityList:
- clientId: de91f1d8-767f-460a-ac11-3cf103f74b34
objectId: 40491351-c240-4042-91e0-f644a1d2b441
resourceId: /subscriptions/subid/resourceGroups/hiloResourcegroup/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-msi
type: cluster
ossVersion: 2.2.3
serviceConfigsProfiles:
- configs:
- component: spark-config
files:
- fileName: spark-defaults.conf
values:
spark.eventLog.enabled: 'true'
serviceName: spark-service
- configs:
- component: yarn-config
files:
- fileName: core-site.xml
values:
fs.defaultFS: wasb://testcontainer@teststorage.dfs.core.windows.net/
storage.container: testcontainer
storage.key: test key
storage.name: teststorage
storage.protocol: wasb
- fileName: yarn-site.xml
values:
yarn.webapp.ui2.enable: 'false'
serviceName: yarn-service
sparkProfile: {}
sshProfile:
count: 2
vmSize: Standard_D3_v2
clusterType: spark
computeProfile:
availabilityZones:
- '1'
- '2'
- '3'
nodes:
- count: 4
type: worker
vmSize: Standard_D3_v2
location: West US 2
resourceGroupName: hiloResourcegroup
The serviceConfigsProfiles array defines configuration files for each service. Each component (spark-config, yarn-config) maps to specific files (spark-defaults.conf, core-site.xml, yarn-site.xml) with key-value pairs. The core-site.xml configuration connects Spark to Azure storage by specifying the storage account, container, and access credentials.
Restrict cluster access to private networks
Security policies may require that cluster endpoints remain inaccessible from the public internet. Internal ingress restricts access to clients within the same virtual network.
import * as pulumi from "@pulumi/pulumi";
import * as azure_native from "@pulumi/azure-native";
const clusterPoolCluster = new azure_native.hdinsight.ClusterPoolCluster("clusterPoolCluster", {
clusterName: "cluster1",
clusterPoolName: "clusterpool1",
clusterProfile: {
authorizationProfile: {
userIds: [
"testuser1",
"testuser2",
],
},
clusterAccessProfile: {
enableInternalIngress: true,
},
clusterVersion: "0.0.1",
managedIdentityProfile: {
identityList: [{
clientId: "de91f1d8-767f-460a-ac11-3cf103f74b34",
objectId: "40491351-c240-4042-91e0-f644a1d2b441",
resourceId: "/subscriptions/subid/resourceGroups/hiloResourcegroup/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-msi",
type: azure_native.hdinsight.ManagedIdentityType.Cluster,
}],
},
ossVersion: "2.2.3",
serviceConfigsProfiles: [
{
configs: [{
component: "spark-config",
files: [{
fileName: "spark-defaults.conf",
values: {
"spark.eventLog.enabled": "true",
},
}],
}],
serviceName: "spark-service",
},
{
configs: [{
component: "yarn-config",
files: [
{
fileName: "core-site.xml",
values: {
"fs.defaultFS": "wasb://testcontainer@teststorage.dfs.core.windows.net/",
"storage.container": "testcontainer",
"storage.key": "test key",
"storage.name": "teststorage",
"storage.protocol": "wasb",
},
},
{
fileName: "yarn-site.xml",
values: {
"yarn.webapp.ui2.enable": "false",
},
},
],
}],
serviceName: "yarn-service",
},
],
sparkProfile: {},
sshProfile: {
count: 2,
vmSize: "Standard_D3_v2",
},
},
clusterType: "spark",
computeProfile: {
availabilityZones: [
"1",
"2",
"3",
],
nodes: [{
count: 4,
type: "worker",
vmSize: "Standard_D3_v2",
}],
},
location: "West US 2",
resourceGroupName: "hiloResourcegroup",
});
import pulumi
import pulumi_azure_native as azure_native
cluster_pool_cluster = azure_native.hdinsight.ClusterPoolCluster("clusterPoolCluster",
cluster_name="cluster1",
cluster_pool_name="clusterpool1",
cluster_profile={
"authorization_profile": {
"user_ids": [
"testuser1",
"testuser2",
],
},
"cluster_access_profile": {
"enable_internal_ingress": True,
},
"cluster_version": "0.0.1",
"managed_identity_profile": {
"identity_list": [{
"client_id": "de91f1d8-767f-460a-ac11-3cf103f74b34",
"object_id": "40491351-c240-4042-91e0-f644a1d2b441",
"resource_id": "/subscriptions/subid/resourceGroups/hiloResourcegroup/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-msi",
"type": azure_native.hdinsight.ManagedIdentityType.CLUSTER,
}],
},
"oss_version": "2.2.3",
"service_configs_profiles": [
{
"configs": [{
"component": "spark-config",
"files": [{
"file_name": "spark-defaults.conf",
"values": {
"spark.eventLog.enabled": "true",
},
}],
}],
"service_name": "spark-service",
},
{
"configs": [{
"component": "yarn-config",
"files": [
{
"file_name": "core-site.xml",
"values": {
"fs.defaultFS": "wasb://testcontainer@teststorage.dfs.core.windows.net/",
"storage.container": "testcontainer",
"storage.key": "test key",
"storage.name": "teststorage",
"storage.protocol": "wasb",
},
},
{
"file_name": "yarn-site.xml",
"values": {
"yarn.webapp.ui2.enable": "false",
},
},
],
}],
"service_name": "yarn-service",
},
],
"spark_profile": {},
"ssh_profile": {
"count": 2,
"vm_size": "Standard_D3_v2",
},
},
cluster_type="spark",
compute_profile={
"availability_zones": [
"1",
"2",
"3",
],
"nodes": [{
"count": 4,
"type": "worker",
"vm_size": "Standard_D3_v2",
}],
},
location="West US 2",
resource_group_name="hiloResourcegroup")
package main
import (
hdinsight "github.com/pulumi/pulumi-azure-native-sdk/hdinsight/v3"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
_, err := hdinsight.NewClusterPoolCluster(ctx, "clusterPoolCluster", &hdinsight.ClusterPoolClusterArgs{
ClusterName: pulumi.String("cluster1"),
ClusterPoolName: pulumi.String("clusterpool1"),
ClusterProfile: &hdinsight.ClusterProfileArgs{
AuthorizationProfile: &hdinsight.AuthorizationProfileArgs{
UserIds: pulumi.StringArray{
pulumi.String("testuser1"),
pulumi.String("testuser2"),
},
},
ClusterAccessProfile: &hdinsight.ClusterAccessProfileArgs{
EnableInternalIngress: pulumi.Bool(true),
},
ClusterVersion: pulumi.String("0.0.1"),
ManagedIdentityProfile: &hdinsight.ManagedIdentityProfileArgs{
IdentityList: hdinsight.ManagedIdentitySpecArray{
&hdinsight.ManagedIdentitySpecArgs{
ClientId: pulumi.String("de91f1d8-767f-460a-ac11-3cf103f74b34"),
ObjectId: pulumi.String("40491351-c240-4042-91e0-f644a1d2b441"),
ResourceId: pulumi.String("/subscriptions/subid/resourceGroups/hiloResourcegroup/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-msi"),
Type: pulumi.String(hdinsight.ManagedIdentityTypeCluster),
},
},
},
OssVersion: pulumi.String("2.2.3"),
ServiceConfigsProfiles: hdinsight.ClusterServiceConfigsProfileArray{
&hdinsight.ClusterServiceConfigsProfileArgs{
Configs: hdinsight.ClusterServiceConfigArray{
&hdinsight.ClusterServiceConfigArgs{
Component: pulumi.String("spark-config"),
Files: hdinsight.ClusterConfigFileArray{
&hdinsight.ClusterConfigFileArgs{
FileName: pulumi.String("spark-defaults.conf"),
Values: pulumi.StringMap{
"spark.eventLog.enabled": pulumi.String("true"),
},
},
},
},
},
ServiceName: pulumi.String("spark-service"),
},
&hdinsight.ClusterServiceConfigsProfileArgs{
Configs: hdinsight.ClusterServiceConfigArray{
&hdinsight.ClusterServiceConfigArgs{
Component: pulumi.String("yarn-config"),
Files: hdinsight.ClusterConfigFileArray{
&hdinsight.ClusterConfigFileArgs{
FileName: pulumi.String("core-site.xml"),
Values: pulumi.StringMap{
"fs.defaultFS": pulumi.String("wasb://testcontainer@teststorage.dfs.core.windows.net/"),
"storage.container": pulumi.String("testcontainer"),
"storage.key": pulumi.String("test key"),
"storage.name": pulumi.String("teststorage"),
"storage.protocol": pulumi.String("wasb"),
},
},
&hdinsight.ClusterConfigFileArgs{
FileName: pulumi.String("yarn-site.xml"),
Values: pulumi.StringMap{
"yarn.webapp.ui2.enable": pulumi.String("false"),
},
},
},
},
},
ServiceName: pulumi.String("yarn-service"),
},
},
SparkProfile: &hdinsight.SparkProfileArgs{},
SshProfile: &hdinsight.ClusterPoolSshProfileArgs{
Count: pulumi.Int(2),
VmSize: pulumi.String("Standard_D3_v2"),
},
},
ClusterType: pulumi.String("spark"),
ComputeProfile: &hdinsight.ClusterPoolComputeProfileArgs{
AvailabilityZones: pulumi.StringArray{
pulumi.String("1"),
pulumi.String("2"),
pulumi.String("3"),
},
Nodes: hdinsight.NodeProfileArray{
&hdinsight.NodeProfileArgs{
Count: pulumi.Int(4),
Type: pulumi.String("worker"),
VmSize: pulumi.String("Standard_D3_v2"),
},
},
},
Location: pulumi.String("West US 2"),
ResourceGroupName: pulumi.String("hiloResourcegroup"),
})
if err != nil {
return err
}
return nil
})
}
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using AzureNative = Pulumi.AzureNative;
return await Deployment.RunAsync(() =>
{
var clusterPoolCluster = new AzureNative.HDInsight.ClusterPoolCluster("clusterPoolCluster", new()
{
ClusterName = "cluster1",
ClusterPoolName = "clusterpool1",
ClusterProfile = new AzureNative.HDInsight.Inputs.ClusterProfileArgs
{
AuthorizationProfile = new AzureNative.HDInsight.Inputs.AuthorizationProfileArgs
{
UserIds = new[]
{
"testuser1",
"testuser2",
},
},
ClusterAccessProfile = new AzureNative.HDInsight.Inputs.ClusterAccessProfileArgs
{
EnableInternalIngress = true,
},
ClusterVersion = "0.0.1",
ManagedIdentityProfile = new AzureNative.HDInsight.Inputs.ManagedIdentityProfileArgs
{
IdentityList = new[]
{
new AzureNative.HDInsight.Inputs.ManagedIdentitySpecArgs
{
ClientId = "de91f1d8-767f-460a-ac11-3cf103f74b34",
ObjectId = "40491351-c240-4042-91e0-f644a1d2b441",
ResourceId = "/subscriptions/subid/resourceGroups/hiloResourcegroup/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-msi",
Type = AzureNative.HDInsight.ManagedIdentityType.Cluster,
},
},
},
OssVersion = "2.2.3",
ServiceConfigsProfiles = new[]
{
new AzureNative.HDInsight.Inputs.ClusterServiceConfigsProfileArgs
{
Configs = new[]
{
new AzureNative.HDInsight.Inputs.ClusterServiceConfigArgs
{
Component = "spark-config",
Files = new[]
{
new AzureNative.HDInsight.Inputs.ClusterConfigFileArgs
{
FileName = "spark-defaults.conf",
Values =
{
{ "spark.eventLog.enabled", "true" },
},
},
},
},
},
ServiceName = "spark-service",
},
new AzureNative.HDInsight.Inputs.ClusterServiceConfigsProfileArgs
{
Configs = new[]
{
new AzureNative.HDInsight.Inputs.ClusterServiceConfigArgs
{
Component = "yarn-config",
Files = new[]
{
new AzureNative.HDInsight.Inputs.ClusterConfigFileArgs
{
FileName = "core-site.xml",
Values =
{
{ "fs.defaultFS", "wasb://testcontainer@teststorage.dfs.core.windows.net/" },
{ "storage.container", "testcontainer" },
{ "storage.key", "test key" },
{ "storage.name", "teststorage" },
{ "storage.protocol", "wasb" },
},
},
new AzureNative.HDInsight.Inputs.ClusterConfigFileArgs
{
FileName = "yarn-site.xml",
Values =
{
{ "yarn.webapp.ui2.enable", "false" },
},
},
},
},
},
ServiceName = "yarn-service",
},
},
SparkProfile = null,
SshProfile = new AzureNative.HDInsight.Inputs.ClusterPoolSshProfileArgs
{
Count = 2,
VmSize = "Standard_D3_v2",
},
},
ClusterType = "spark",
ComputeProfile = new AzureNative.HDInsight.Inputs.ClusterPoolComputeProfileArgs
{
AvailabilityZones = new[]
{
"1",
"2",
"3",
},
Nodes = new[]
{
new AzureNative.HDInsight.Inputs.NodeProfileArgs
{
Count = 4,
Type = "worker",
VmSize = "Standard_D3_v2",
},
},
},
Location = "West US 2",
ResourceGroupName = "hiloResourcegroup",
});
});
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.azurenative.hdinsight.ClusterPoolCluster;
import com.pulumi.azurenative.hdinsight.ClusterPoolClusterArgs;
import com.pulumi.azurenative.hdinsight.inputs.ClusterProfileArgs;
import com.pulumi.azurenative.hdinsight.inputs.AuthorizationProfileArgs;
import com.pulumi.azurenative.hdinsight.inputs.ClusterAccessProfileArgs;
import com.pulumi.azurenative.hdinsight.inputs.ManagedIdentityProfileArgs;
import com.pulumi.azurenative.hdinsight.inputs.SparkProfileArgs;
import com.pulumi.azurenative.hdinsight.inputs.ClusterPoolSshProfileArgs;
import com.pulumi.azurenative.hdinsight.inputs.ClusterPoolComputeProfileArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var clusterPoolCluster = new ClusterPoolCluster("clusterPoolCluster", ClusterPoolClusterArgs.builder()
.clusterName("cluster1")
.clusterPoolName("clusterpool1")
.clusterProfile(ClusterProfileArgs.builder()
.authorizationProfile(AuthorizationProfileArgs.builder()
.userIds(
"testuser1",
"testuser2")
.build())
.clusterAccessProfile(ClusterAccessProfileArgs.builder()
.enableInternalIngress(true)
.build())
.clusterVersion("0.0.1")
.managedIdentityProfile(ManagedIdentityProfileArgs.builder()
.identityList(ManagedIdentitySpecArgs.builder()
.clientId("de91f1d8-767f-460a-ac11-3cf103f74b34")
.objectId("40491351-c240-4042-91e0-f644a1d2b441")
.resourceId("/subscriptions/subid/resourceGroups/hiloResourcegroup/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-msi")
.type("cluster")
.build())
.build())
.ossVersion("2.2.3")
.serviceConfigsProfiles(
ClusterServiceConfigsProfileArgs.builder()
.configs(ClusterServiceConfigArgs.builder()
.component("spark-config")
.files(ClusterConfigFileArgs.builder()
.fileName("spark-defaults.conf")
.values(Map.of("spark.eventLog.enabled", "true"))
.build())
.build())
.serviceName("spark-service")
.build(),
ClusterServiceConfigsProfileArgs.builder()
.configs(ClusterServiceConfigArgs.builder()
.component("yarn-config")
.files(
ClusterConfigFileArgs.builder()
.fileName("core-site.xml")
.values(Map.ofEntries(
Map.entry("fs.defaultFS", "wasb://testcontainer@teststorage.dfs.core.windows.net/"),
Map.entry("storage.container", "testcontainer"),
Map.entry("storage.key", "test key"),
Map.entry("storage.name", "teststorage"),
Map.entry("storage.protocol", "wasb")
))
.build(),
ClusterConfigFileArgs.builder()
.fileName("yarn-site.xml")
.values(Map.of("yarn.webapp.ui2.enable", "false"))
.build())
.build())
.serviceName("yarn-service")
.build())
.sparkProfile(SparkProfileArgs.builder()
.build())
.sshProfile(ClusterPoolSshProfileArgs.builder()
.count(2)
.vmSize("Standard_D3_v2")
.build())
.build())
.clusterType("spark")
.computeProfile(ClusterPoolComputeProfileArgs.builder()
.availabilityZones(
"1",
"2",
"3")
.nodes(NodeProfileArgs.builder()
.count(4)
.type("worker")
.vmSize("Standard_D3_v2")
.build())
.build())
.location("West US 2")
.resourceGroupName("hiloResourcegroup")
.build());
}
}
resources:
clusterPoolCluster:
type: azure-native:hdinsight:ClusterPoolCluster
properties:
clusterName: cluster1
clusterPoolName: clusterpool1
clusterProfile:
authorizationProfile:
userIds:
- testuser1
- testuser2
clusterAccessProfile:
enableInternalIngress: true
clusterVersion: 0.0.1
managedIdentityProfile:
identityList:
- clientId: de91f1d8-767f-460a-ac11-3cf103f74b34
objectId: 40491351-c240-4042-91e0-f644a1d2b441
resourceId: /subscriptions/subid/resourceGroups/hiloResourcegroup/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-msi
type: cluster
ossVersion: 2.2.3
serviceConfigsProfiles:
- configs:
- component: spark-config
files:
- fileName: spark-defaults.conf
values:
spark.eventLog.enabled: 'true'
serviceName: spark-service
- configs:
- component: yarn-config
files:
- fileName: core-site.xml
values:
fs.defaultFS: wasb://testcontainer@teststorage.dfs.core.windows.net/
storage.container: testcontainer
storage.key: test key
storage.name: teststorage
storage.protocol: wasb
- fileName: yarn-site.xml
values:
yarn.webapp.ui2.enable: 'false'
serviceName: yarn-service
sparkProfile: {}
sshProfile:
count: 2
vmSize: Standard_D3_v2
clusterType: spark
computeProfile:
availabilityZones:
- '1'
- '2'
- '3'
nodes:
- count: 4
type: worker
vmSize: Standard_D3_v2
location: West US 2
resourceGroupName: hiloResourcegroup
The clusterAccessProfile with enableInternalIngress set to true restricts cluster endpoints to private network access only. This configuration extends the basic Spark setup with network-level security, ensuring that cluster UIs and APIs are only reachable from within the virtual network where the cluster pool is deployed.
Beyond these examples
These snippets focus on specific cluster-level features: autoscaling (load-based and schedule-based), Apache Ranger integration for governance, service configuration tuning, and network access control. They’re intentionally minimal rather than full cluster deployments.
The examples may reference pre-existing infrastructure such as cluster pools (parent resource), user-assigned managed identities, SQL databases for Ranger metadata, Key Vault secrets for credentials, and storage accounts for data and audit logs. They focus on configuring the cluster rather than provisioning everything around it.
To keep things focused, common cluster patterns are omitted, including:
- SSH profile configuration (sshProfile)
- Availability zone placement (availabilityZones)
- Node type and VM size selection (nodes array)
- Cluster versioning (clusterVersion, ossVersion)
- Authorization profile and user access (authorizationProfile)
These omissions are intentional: the goal is to illustrate how each cluster feature is wired, not provide drop-in production modules. See the ClusterPoolCluster resource reference for all available configuration options.
Let's deploy Azure HDInsight Cluster Pool Clusters
Get started with Pulumi Cloud, then follow our quick setup guide to deploy this infrastructure.
Try Pulumi Cloud for FREEFrequently Asked Questions
Cluster Configuration & Types
What cluster types are available?
HDInsight on AKS supports Trino, Ranger, and Spark cluster types. The cluster type is specified via the
clusterType property.What properties can't be changed after creating a cluster?
The following properties are immutable:
clusterType, clusterName, clusterPoolName, resourceGroupName, and location.How do I configure a managed identity for my cluster?
Use
managedIdentityProfile.identityList with clientId, objectId, resourceId, and type set to cluster. All examples show this pattern for cluster authentication.Autoscaling
What's the difference between load-based and schedule-based autoscaling?
Load-based autoscaling scales nodes based on metrics like CPU usage, using
minNodes, maxNodes, cooldownPeriod, and scalingRules. Schedule-based autoscaling scales to specific node counts at defined times and days using schedules with startTime, endTime, days, and count.How do I configure load-based autoscaling?
Set
autoscaleProfile.autoscaleType to LoadBased and configure loadBasedConfig with minNodes, maxNodes, cooldownPeriod (e.g., 300 seconds), pollInterval (e.g., 60 seconds), and scalingRules that define thresholds and actions.What is gracefulDecommissionTimeout used for?
The
gracefulDecommissionTimeout (in seconds) allows nodes to complete running tasks before being removed during scale-down operations. The example shows 3600 seconds (1 hour).Compute & Nodes
What node types can I configure in my cluster?
Clusters support Head and Worker node types. Configure them in
computeProfile.nodes with type, count, and vmSize properties. Examples show Head nodes with count 2 and Worker nodes with counts ranging from 3-4.How do I deploy my cluster across availability zones?
Specify availability zones in
computeProfile.availabilityZones as an array of zone numbers. Examples show deployment across zones [“1”, “2”, “3”].Access & Security
How do I restrict my cluster to internal-only access?
Set
clusterProfile.clusterAccessProfile.enableInternalIngress to true to enable internal ingress and disable public access.How do I authorize users to access my cluster?
Configure
clusterProfile.authorizationProfile.userIds with a list of user identifiers. All examples show this for user authorization.Service-Specific Configuration
How do I configure Spark service settings?
Use
clusterProfile.serviceConfigsProfiles to define service configurations. Specify serviceName (e.g., “spark-service”), component (e.g., “spark-config”), and files with configuration key-value pairs like spark.eventLog.enabled.What Ranger-specific configuration is required?
For Ranger clusters, configure
rangerProfile with rangerAdmin (database connection, admins), rangerAudit (storage account), and rangerUsersync (mode, users, groups) properties.