The azure-native:hdinsight:Cluster resource, part of the Pulumi Azure Native provider, provisions HDInsight clusters for big data processing: compute nodes, storage integration, networking, and security configuration. This guide focuses on four capabilities: autoscaling worker nodes on schedules, storage integration with Blob and ADLS Gen 2, SSH and Active Directory authentication, and virtual network configuration.
HDInsight clusters require Azure Storage accounts for data persistence and may reference virtual networks, subnets, and Active Directory domains for enterprise security. The examples are intentionally small. Combine them with your own storage accounts, networking infrastructure, and security policies.
Scale worker nodes on a schedule
Analytics workloads often follow predictable patterns, with higher demand during business hours and lower demand overnight or on weekends. Autoscale configuration allows clusters to adjust worker node capacity automatically based on time-of-day schedules.
import * as pulumi from "@pulumi/pulumi";
import * as azure_native from "@pulumi/azure-native";
const cluster = new azure_native.hdinsight.Cluster("cluster", {
clusterName: "cluster1",
properties: {
clusterDefinition: {
componentVersion: {
Hadoop: "2.7",
},
configurations: {
gateway: {
"restAuthCredential.isEnabled": true,
"restAuthCredential.password": "**********",
"restAuthCredential.username": "admin",
},
},
kind: "hadoop",
},
clusterVersion: "3.6",
computeProfile: {
roles: [{
autoscaleConfiguration: {
recurrence: {
schedule: [
{
days: [
azure_native.hdinsight.DaysOfWeek.Monday,
azure_native.hdinsight.DaysOfWeek.Tuesday,
azure_native.hdinsight.DaysOfWeek.Wednesday,
azure_native.hdinsight.DaysOfWeek.Thursday,
azure_native.hdinsight.DaysOfWeek.Friday,
],
timeAndCapacity: {
maxInstanceCount: 3,
minInstanceCount: 3,
time: "09:00",
},
},
{
days: [
azure_native.hdinsight.DaysOfWeek.Monday,
azure_native.hdinsight.DaysOfWeek.Tuesday,
azure_native.hdinsight.DaysOfWeek.Wednesday,
azure_native.hdinsight.DaysOfWeek.Thursday,
azure_native.hdinsight.DaysOfWeek.Friday,
],
timeAndCapacity: {
maxInstanceCount: 6,
minInstanceCount: 6,
time: "18:00",
},
},
{
days: [
azure_native.hdinsight.DaysOfWeek.Saturday,
azure_native.hdinsight.DaysOfWeek.Sunday,
],
timeAndCapacity: {
maxInstanceCount: 2,
minInstanceCount: 2,
time: "09:00",
},
},
{
days: [
azure_native.hdinsight.DaysOfWeek.Saturday,
azure_native.hdinsight.DaysOfWeek.Sunday,
],
timeAndCapacity: {
maxInstanceCount: 4,
minInstanceCount: 4,
time: "18:00",
},
},
],
timeZone: "China Standard Time",
},
},
hardwareProfile: {
vmSize: "Standard_D4_V2",
},
name: "workernode",
osProfile: {
linuxOperatingSystemProfile: {
password: "**********",
username: "sshuser",
},
},
scriptActions: [],
targetInstanceCount: 4,
}],
},
osType: azure_native.hdinsight.OSType.Linux,
storageProfile: {
storageaccounts: [{
container: "hdinsight-autoscale-tes-2019-06-18t05-49-16-591z",
enableSecureChannel: true,
isDefault: true,
key: "storagekey",
name: "mystorage.blob.core.windows.net",
}],
},
tier: azure_native.hdinsight.Tier.Standard,
},
resourceGroupName: "rg1",
});
import pulumi
import pulumi_azure_native as azure_native
cluster = azure_native.hdinsight.Cluster("cluster",
cluster_name="cluster1",
properties={
"cluster_definition": {
"component_version": {
"Hadoop": "2.7",
},
"configurations": {
"gateway": {
"restAuthCredential.isEnabled": True,
"restAuthCredential.password": "**********",
"restAuthCredential.username": "admin",
},
},
"kind": "hadoop",
},
"cluster_version": "3.6",
"compute_profile": {
"roles": [{
"autoscale_configuration": {
"recurrence": {
"schedule": [
{
"days": [
azure_native.hdinsight.DaysOfWeek.MONDAY,
azure_native.hdinsight.DaysOfWeek.TUESDAY,
azure_native.hdinsight.DaysOfWeek.WEDNESDAY,
azure_native.hdinsight.DaysOfWeek.THURSDAY,
azure_native.hdinsight.DaysOfWeek.FRIDAY,
],
"time_and_capacity": {
"max_instance_count": 3,
"min_instance_count": 3,
"time": "09:00",
},
},
{
"days": [
azure_native.hdinsight.DaysOfWeek.MONDAY,
azure_native.hdinsight.DaysOfWeek.TUESDAY,
azure_native.hdinsight.DaysOfWeek.WEDNESDAY,
azure_native.hdinsight.DaysOfWeek.THURSDAY,
azure_native.hdinsight.DaysOfWeek.FRIDAY,
],
"time_and_capacity": {
"max_instance_count": 6,
"min_instance_count": 6,
"time": "18:00",
},
},
{
"days": [
azure_native.hdinsight.DaysOfWeek.SATURDAY,
azure_native.hdinsight.DaysOfWeek.SUNDAY,
],
"time_and_capacity": {
"max_instance_count": 2,
"min_instance_count": 2,
"time": "09:00",
},
},
{
"days": [
azure_native.hdinsight.DaysOfWeek.SATURDAY,
azure_native.hdinsight.DaysOfWeek.SUNDAY,
],
"time_and_capacity": {
"max_instance_count": 4,
"min_instance_count": 4,
"time": "18:00",
},
},
],
"time_zone": "China Standard Time",
},
},
"hardware_profile": {
"vm_size": "Standard_D4_V2",
},
"name": "workernode",
"os_profile": {
"linux_operating_system_profile": {
"password": "**********",
"username": "sshuser",
},
},
"script_actions": [],
"target_instance_count": 4,
}],
},
"os_type": azure_native.hdinsight.OSType.LINUX,
"storage_profile": {
"storageaccounts": [{
"container": "hdinsight-autoscale-tes-2019-06-18t05-49-16-591z",
"enable_secure_channel": True,
"is_default": True,
"key": "storagekey",
"name": "mystorage.blob.core.windows.net",
}],
},
"tier": azure_native.hdinsight.Tier.STANDARD,
},
resource_group_name="rg1")
package main
import (
hdinsight "github.com/pulumi/pulumi-azure-native-sdk/hdinsight/v3"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
_, err := hdinsight.NewCluster(ctx, "cluster", &hdinsight.ClusterArgs{
ClusterName: pulumi.String("cluster1"),
Properties: &hdinsight.ClusterCreatePropertiesArgs{
ClusterDefinition: &hdinsight.ClusterDefinitionArgs{
ComponentVersion: pulumi.StringMap{
"Hadoop": pulumi.String("2.7"),
},
Configurations: pulumi.Any(map[string]interface{}{
"gateway": map[string]interface{}{
"restAuthCredential.isEnabled": true,
"restAuthCredential.password": "**********",
"restAuthCredential.username": "admin",
},
}),
Kind: pulumi.String("hadoop"),
},
ClusterVersion: pulumi.String("3.6"),
ComputeProfile: &hdinsight.ComputeProfileArgs{
Roles: hdinsight.RoleArray{
&hdinsight.RoleArgs{
AutoscaleConfiguration: &hdinsight.AutoscaleArgs{
Recurrence: &hdinsight.AutoscaleRecurrenceArgs{
Schedule: hdinsight.AutoscaleScheduleArray{
&hdinsight.AutoscaleScheduleArgs{
Days: pulumi.StringArray{
pulumi.String(hdinsight.DaysOfWeekMonday),
pulumi.String(hdinsight.DaysOfWeekTuesday),
pulumi.String(hdinsight.DaysOfWeekWednesday),
pulumi.String(hdinsight.DaysOfWeekThursday),
pulumi.String(hdinsight.DaysOfWeekFriday),
},
TimeAndCapacity: &hdinsight.AutoscaleTimeAndCapacityArgs{
MaxInstanceCount: pulumi.Int(3),
MinInstanceCount: pulumi.Int(3),
Time: pulumi.String("09:00"),
},
},
&hdinsight.AutoscaleScheduleArgs{
Days: pulumi.StringArray{
pulumi.String(hdinsight.DaysOfWeekMonday),
pulumi.String(hdinsight.DaysOfWeekTuesday),
pulumi.String(hdinsight.DaysOfWeekWednesday),
pulumi.String(hdinsight.DaysOfWeekThursday),
pulumi.String(hdinsight.DaysOfWeekFriday),
},
TimeAndCapacity: &hdinsight.AutoscaleTimeAndCapacityArgs{
MaxInstanceCount: pulumi.Int(6),
MinInstanceCount: pulumi.Int(6),
Time: pulumi.String("18:00"),
},
},
&hdinsight.AutoscaleScheduleArgs{
Days: pulumi.StringArray{
pulumi.String(hdinsight.DaysOfWeekSaturday),
pulumi.String(hdinsight.DaysOfWeekSunday),
},
TimeAndCapacity: &hdinsight.AutoscaleTimeAndCapacityArgs{
MaxInstanceCount: pulumi.Int(2),
MinInstanceCount: pulumi.Int(2),
Time: pulumi.String("09:00"),
},
},
&hdinsight.AutoscaleScheduleArgs{
Days: pulumi.StringArray{
pulumi.String(hdinsight.DaysOfWeekSaturday),
pulumi.String(hdinsight.DaysOfWeekSunday),
},
TimeAndCapacity: &hdinsight.AutoscaleTimeAndCapacityArgs{
MaxInstanceCount: pulumi.Int(4),
MinInstanceCount: pulumi.Int(4),
Time: pulumi.String("18:00"),
},
},
},
TimeZone: pulumi.String("China Standard Time"),
},
},
HardwareProfile: &hdinsight.HardwareProfileArgs{
VmSize: pulumi.String("Standard_D4_V2"),
},
Name: pulumi.String("workernode"),
OsProfile: &hdinsight.OsProfileArgs{
LinuxOperatingSystemProfile: &hdinsight.LinuxOperatingSystemProfileArgs{
Password: pulumi.String("**********"),
Username: pulumi.String("sshuser"),
},
},
ScriptActions: hdinsight.ScriptActionArray{},
TargetInstanceCount: pulumi.Int(4),
},
},
},
OsType: pulumi.String(hdinsight.OSTypeLinux),
StorageProfile: &hdinsight.StorageProfileArgs{
Storageaccounts: hdinsight.StorageAccountArray{
&hdinsight.StorageAccountArgs{
Container: pulumi.String("hdinsight-autoscale-tes-2019-06-18t05-49-16-591z"),
EnableSecureChannel: pulumi.Bool(true),
IsDefault: pulumi.Bool(true),
Key: pulumi.String("storagekey"),
Name: pulumi.String("mystorage.blob.core.windows.net"),
},
},
},
Tier: pulumi.String(hdinsight.TierStandard),
},
ResourceGroupName: pulumi.String("rg1"),
})
if err != nil {
return err
}
return nil
})
}
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using AzureNative = Pulumi.AzureNative;
return await Deployment.RunAsync(() =>
{
var cluster = new AzureNative.HDInsight.Cluster("cluster", new()
{
ClusterName = "cluster1",
Properties = new AzureNative.HDInsight.Inputs.ClusterCreatePropertiesArgs
{
ClusterDefinition = new AzureNative.HDInsight.Inputs.ClusterDefinitionArgs
{
ComponentVersion =
{
{ "Hadoop", "2.7" },
},
Configurations = new Dictionary<string, object?>
{
["gateway"] = new Dictionary<string, object?>
{
["restAuthCredential.isEnabled"] = true,
["restAuthCredential.password"] = "**********",
["restAuthCredential.username"] = "admin",
},
},
Kind = "hadoop",
},
ClusterVersion = "3.6",
ComputeProfile = new AzureNative.HDInsight.Inputs.ComputeProfileArgs
{
Roles = new[]
{
new AzureNative.HDInsight.Inputs.RoleArgs
{
AutoscaleConfiguration = new AzureNative.HDInsight.Inputs.AutoscaleArgs
{
Recurrence = new AzureNative.HDInsight.Inputs.AutoscaleRecurrenceArgs
{
Schedule = new[]
{
new AzureNative.HDInsight.Inputs.AutoscaleScheduleArgs
{
Days = new[]
{
AzureNative.HDInsight.DaysOfWeek.Monday,
AzureNative.HDInsight.DaysOfWeek.Tuesday,
AzureNative.HDInsight.DaysOfWeek.Wednesday,
AzureNative.HDInsight.DaysOfWeek.Thursday,
AzureNative.HDInsight.DaysOfWeek.Friday,
},
TimeAndCapacity = new AzureNative.HDInsight.Inputs.AutoscaleTimeAndCapacityArgs
{
MaxInstanceCount = 3,
MinInstanceCount = 3,
Time = "09:00",
},
},
new AzureNative.HDInsight.Inputs.AutoscaleScheduleArgs
{
Days = new[]
{
AzureNative.HDInsight.DaysOfWeek.Monday,
AzureNative.HDInsight.DaysOfWeek.Tuesday,
AzureNative.HDInsight.DaysOfWeek.Wednesday,
AzureNative.HDInsight.DaysOfWeek.Thursday,
AzureNative.HDInsight.DaysOfWeek.Friday,
},
TimeAndCapacity = new AzureNative.HDInsight.Inputs.AutoscaleTimeAndCapacityArgs
{
MaxInstanceCount = 6,
MinInstanceCount = 6,
Time = "18:00",
},
},
new AzureNative.HDInsight.Inputs.AutoscaleScheduleArgs
{
Days = new[]
{
AzureNative.HDInsight.DaysOfWeek.Saturday,
AzureNative.HDInsight.DaysOfWeek.Sunday,
},
TimeAndCapacity = new AzureNative.HDInsight.Inputs.AutoscaleTimeAndCapacityArgs
{
MaxInstanceCount = 2,
MinInstanceCount = 2,
Time = "09:00",
},
},
new AzureNative.HDInsight.Inputs.AutoscaleScheduleArgs
{
Days = new[]
{
AzureNative.HDInsight.DaysOfWeek.Saturday,
AzureNative.HDInsight.DaysOfWeek.Sunday,
},
TimeAndCapacity = new AzureNative.HDInsight.Inputs.AutoscaleTimeAndCapacityArgs
{
MaxInstanceCount = 4,
MinInstanceCount = 4,
Time = "18:00",
},
},
},
TimeZone = "China Standard Time",
},
},
HardwareProfile = new AzureNative.HDInsight.Inputs.HardwareProfileArgs
{
VmSize = "Standard_D4_V2",
},
Name = "workernode",
OsProfile = new AzureNative.HDInsight.Inputs.OsProfileArgs
{
LinuxOperatingSystemProfile = new AzureNative.HDInsight.Inputs.LinuxOperatingSystemProfileArgs
{
Password = "**********",
Username = "sshuser",
},
},
ScriptActions = new() { },
TargetInstanceCount = 4,
},
},
},
OsType = AzureNative.HDInsight.OSType.Linux,
StorageProfile = new AzureNative.HDInsight.Inputs.StorageProfileArgs
{
Storageaccounts = new[]
{
new AzureNative.HDInsight.Inputs.StorageAccountArgs
{
Container = "hdinsight-autoscale-tes-2019-06-18t05-49-16-591z",
EnableSecureChannel = true,
IsDefault = true,
Key = "storagekey",
Name = "mystorage.blob.core.windows.net",
},
},
},
Tier = AzureNative.HDInsight.Tier.Standard,
},
ResourceGroupName = "rg1",
});
});
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.azurenative.hdinsight.Cluster;
import com.pulumi.azurenative.hdinsight.ClusterArgs;
import com.pulumi.azurenative.hdinsight.inputs.ClusterCreatePropertiesArgs;
import com.pulumi.azurenative.hdinsight.inputs.ClusterDefinitionArgs;
import com.pulumi.azurenative.hdinsight.inputs.ComputeProfileArgs;
import com.pulumi.azurenative.hdinsight.inputs.StorageProfileArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var cluster = new Cluster("cluster", ClusterArgs.builder()
.clusterName("cluster1")
.properties(ClusterCreatePropertiesArgs.builder()
.clusterDefinition(ClusterDefinitionArgs.builder()
.componentVersion(Map.of("Hadoop", "2.7"))
.configurations(Map.of("gateway", Map.ofEntries(
Map.entry("restAuthCredential.isEnabled", true),
Map.entry("restAuthCredential.password", "**********"),
Map.entry("restAuthCredential.username", "admin")
)))
.kind("hadoop")
.build())
.clusterVersion("3.6")
.computeProfile(ComputeProfileArgs.builder()
.roles(RoleArgs.builder()
.autoscaleConfiguration(AutoscaleArgs.builder()
.recurrence(AutoscaleRecurrenceArgs.builder()
.schedule(
AutoscaleScheduleArgs.builder()
.days(
"Monday",
"Tuesday",
"Wednesday",
"Thursday",
"Friday")
.timeAndCapacity(AutoscaleTimeAndCapacityArgs.builder()
.maxInstanceCount(3)
.minInstanceCount(3)
.time("09:00")
.build())
.build(),
AutoscaleScheduleArgs.builder()
.days(
"Monday",
"Tuesday",
"Wednesday",
"Thursday",
"Friday")
.timeAndCapacity(AutoscaleTimeAndCapacityArgs.builder()
.maxInstanceCount(6)
.minInstanceCount(6)
.time("18:00")
.build())
.build(),
AutoscaleScheduleArgs.builder()
.days(
"Saturday",
"Sunday")
.timeAndCapacity(AutoscaleTimeAndCapacityArgs.builder()
.maxInstanceCount(2)
.minInstanceCount(2)
.time("09:00")
.build())
.build(),
AutoscaleScheduleArgs.builder()
.days(
"Saturday",
"Sunday")
.timeAndCapacity(AutoscaleTimeAndCapacityArgs.builder()
.maxInstanceCount(4)
.minInstanceCount(4)
.time("18:00")
.build())
.build())
.timeZone("China Standard Time")
.build())
.build())
.hardwareProfile(HardwareProfileArgs.builder()
.vmSize("Standard_D4_V2")
.build())
.name("workernode")
.osProfile(OsProfileArgs.builder()
.linuxOperatingSystemProfile(LinuxOperatingSystemProfileArgs.builder()
.password("**********")
.username("sshuser")
.build())
.build())
.scriptActions()
.targetInstanceCount(4)
.build())
.build())
.osType("Linux")
.storageProfile(StorageProfileArgs.builder()
.storageaccounts(StorageAccountArgs.builder()
.container("hdinsight-autoscale-tes-2019-06-18t05-49-16-591z")
.enableSecureChannel(true)
.isDefault(true)
.key("storagekey")
.name("mystorage.blob.core.windows.net")
.build())
.build())
.tier("Standard")
.build())
.resourceGroupName("rg1")
.build());
}
}
resources:
cluster:
type: azure-native:hdinsight:Cluster
properties:
clusterName: cluster1
properties:
clusterDefinition:
componentVersion:
Hadoop: '2.7'
configurations:
gateway:
restAuthCredential.isEnabled: true
restAuthCredential.password: '**********'
restAuthCredential.username: admin
kind: hadoop
clusterVersion: '3.6'
computeProfile:
roles:
- autoscaleConfiguration:
recurrence:
schedule:
- days:
- Monday
- Tuesday
- Wednesday
- Thursday
- Friday
timeAndCapacity:
maxInstanceCount: 3
minInstanceCount: 3
time: 09:00
- days:
- Monday
- Tuesday
- Wednesday
- Thursday
- Friday
timeAndCapacity:
maxInstanceCount: 6
minInstanceCount: 6
time: 18:00
- days:
- Saturday
- Sunday
timeAndCapacity:
maxInstanceCount: 2
minInstanceCount: 2
time: 09:00
- days:
- Saturday
- Sunday
timeAndCapacity:
maxInstanceCount: 4
minInstanceCount: 4
time: 18:00
timeZone: China Standard Time
hardwareProfile:
vmSize: Standard_D4_V2
name: workernode
osProfile:
linuxOperatingSystemProfile:
password: '**********'
username: sshuser
scriptActions: []
targetInstanceCount: 4
osType: Linux
storageProfile:
storageaccounts:
- container: hdinsight-autoscale-tes-2019-06-18t05-49-16-591z
enableSecureChannel: true
isDefault: true
key: storagekey
name: mystorage.blob.core.windows.net
tier: Standard
resourceGroupName: rg1
The autoscaleConfiguration property defines when and how the cluster scales. The recurrence block contains a schedule array where each entry specifies days of the week and a timeAndCapacity block. At the specified time, the cluster adjusts to the configured minInstanceCount and maxInstanceCount. This example scales to 3 nodes at 9 AM and 6 nodes at 6 PM on weekdays, with different capacity on weekends.
Connect to Azure Data Lake Storage Gen 2
Teams using Azure Data Lake Storage Gen 2 for analytics need clusters that can read and write directly to ADLS Gen 2 file systems rather than traditional blob storage.
import * as pulumi from "@pulumi/pulumi";
import * as azure_native from "@pulumi/azure-native";
const cluster = new azure_native.hdinsight.Cluster("cluster", {
clusterName: "cluster1",
properties: {
clusterDefinition: {
configurations: {
gateway: {
"restAuthCredential.isEnabled": "true",
"restAuthCredential.password": "**********",
"restAuthCredential.username": "admin",
},
},
kind: "Hadoop",
},
clusterVersion: "3.6",
computeProfile: {
roles: [
{
hardwareProfile: {
vmSize: "Standard_D3_V2",
},
minInstanceCount: 1,
name: "headnode",
osProfile: {
linuxOperatingSystemProfile: {
password: "**********",
username: "sshuser",
},
},
targetInstanceCount: 2,
},
{
hardwareProfile: {
vmSize: "Standard_D3_V2",
},
minInstanceCount: 1,
name: "workernode",
osProfile: {
linuxOperatingSystemProfile: {
password: "**********",
username: "sshuser",
},
},
targetInstanceCount: 4,
},
{
hardwareProfile: {
vmSize: "Small",
},
minInstanceCount: 1,
name: "zookeepernode",
osProfile: {
linuxOperatingSystemProfile: {
password: "**********",
username: "sshuser",
},
},
targetInstanceCount: 3,
},
],
},
osType: azure_native.hdinsight.OSType.Linux,
storageProfile: {
storageaccounts: [{
enableSecureChannel: true,
fileSystem: "default",
isDefault: true,
key: "storagekey",
name: "mystorage.dfs.core.windows.net",
}],
},
tier: azure_native.hdinsight.Tier.Standard,
},
resourceGroupName: "rg1",
tags: {
key1: "val1",
},
});
import pulumi
import pulumi_azure_native as azure_native
cluster = azure_native.hdinsight.Cluster("cluster",
cluster_name="cluster1",
properties={
"cluster_definition": {
"configurations": {
"gateway": {
"restAuthCredential.isEnabled": "true",
"restAuthCredential.password": "**********",
"restAuthCredential.username": "admin",
},
},
"kind": "Hadoop",
},
"cluster_version": "3.6",
"compute_profile": {
"roles": [
{
"hardware_profile": {
"vm_size": "Standard_D3_V2",
},
"min_instance_count": 1,
"name": "headnode",
"os_profile": {
"linux_operating_system_profile": {
"password": "**********",
"username": "sshuser",
},
},
"target_instance_count": 2,
},
{
"hardware_profile": {
"vm_size": "Standard_D3_V2",
},
"min_instance_count": 1,
"name": "workernode",
"os_profile": {
"linux_operating_system_profile": {
"password": "**********",
"username": "sshuser",
},
},
"target_instance_count": 4,
},
{
"hardware_profile": {
"vm_size": "Small",
},
"min_instance_count": 1,
"name": "zookeepernode",
"os_profile": {
"linux_operating_system_profile": {
"password": "**********",
"username": "sshuser",
},
},
"target_instance_count": 3,
},
],
},
"os_type": azure_native.hdinsight.OSType.LINUX,
"storage_profile": {
"storageaccounts": [{
"enable_secure_channel": True,
"file_system": "default",
"is_default": True,
"key": "storagekey",
"name": "mystorage.dfs.core.windows.net",
}],
},
"tier": azure_native.hdinsight.Tier.STANDARD,
},
resource_group_name="rg1",
tags={
"key1": "val1",
})
package main
import (
hdinsight "github.com/pulumi/pulumi-azure-native-sdk/hdinsight/v3"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
_, err := hdinsight.NewCluster(ctx, "cluster", &hdinsight.ClusterArgs{
ClusterName: pulumi.String("cluster1"),
Properties: &hdinsight.ClusterCreatePropertiesArgs{
ClusterDefinition: &hdinsight.ClusterDefinitionArgs{
Configurations: pulumi.Any(map[string]interface{}{
"gateway": map[string]interface{}{
"restAuthCredential.isEnabled": "true",
"restAuthCredential.password": "**********",
"restAuthCredential.username": "admin",
},
}),
Kind: pulumi.String("Hadoop"),
},
ClusterVersion: pulumi.String("3.6"),
ComputeProfile: &hdinsight.ComputeProfileArgs{
Roles: hdinsight.RoleArray{
&hdinsight.RoleArgs{
HardwareProfile: &hdinsight.HardwareProfileArgs{
VmSize: pulumi.String("Standard_D3_V2"),
},
MinInstanceCount: pulumi.Int(1),
Name: pulumi.String("headnode"),
OsProfile: &hdinsight.OsProfileArgs{
LinuxOperatingSystemProfile: &hdinsight.LinuxOperatingSystemProfileArgs{
Password: pulumi.String("**********"),
Username: pulumi.String("sshuser"),
},
},
TargetInstanceCount: pulumi.Int(2),
},
&hdinsight.RoleArgs{
HardwareProfile: &hdinsight.HardwareProfileArgs{
VmSize: pulumi.String("Standard_D3_V2"),
},
MinInstanceCount: pulumi.Int(1),
Name: pulumi.String("workernode"),
OsProfile: &hdinsight.OsProfileArgs{
LinuxOperatingSystemProfile: &hdinsight.LinuxOperatingSystemProfileArgs{
Password: pulumi.String("**********"),
Username: pulumi.String("sshuser"),
},
},
TargetInstanceCount: pulumi.Int(4),
},
&hdinsight.RoleArgs{
HardwareProfile: &hdinsight.HardwareProfileArgs{
VmSize: pulumi.String("Small"),
},
MinInstanceCount: pulumi.Int(1),
Name: pulumi.String("zookeepernode"),
OsProfile: &hdinsight.OsProfileArgs{
LinuxOperatingSystemProfile: &hdinsight.LinuxOperatingSystemProfileArgs{
Password: pulumi.String("**********"),
Username: pulumi.String("sshuser"),
},
},
TargetInstanceCount: pulumi.Int(3),
},
},
},
OsType: pulumi.String(hdinsight.OSTypeLinux),
StorageProfile: &hdinsight.StorageProfileArgs{
Storageaccounts: hdinsight.StorageAccountArray{
&hdinsight.StorageAccountArgs{
EnableSecureChannel: pulumi.Bool(true),
FileSystem: pulumi.String("default"),
IsDefault: pulumi.Bool(true),
Key: pulumi.String("storagekey"),
Name: pulumi.String("mystorage.dfs.core.windows.net"),
},
},
},
Tier: pulumi.String(hdinsight.TierStandard),
},
ResourceGroupName: pulumi.String("rg1"),
Tags: pulumi.StringMap{
"key1": pulumi.String("val1"),
},
})
if err != nil {
return err
}
return nil
})
}
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using AzureNative = Pulumi.AzureNative;
return await Deployment.RunAsync(() =>
{
var cluster = new AzureNative.HDInsight.Cluster("cluster", new()
{
ClusterName = "cluster1",
Properties = new AzureNative.HDInsight.Inputs.ClusterCreatePropertiesArgs
{
ClusterDefinition = new AzureNative.HDInsight.Inputs.ClusterDefinitionArgs
{
Configurations = new Dictionary<string, object?>
{
["gateway"] = new Dictionary<string, object?>
{
["restAuthCredential.isEnabled"] = "true",
["restAuthCredential.password"] = "**********",
["restAuthCredential.username"] = "admin",
},
},
Kind = "Hadoop",
},
ClusterVersion = "3.6",
ComputeProfile = new AzureNative.HDInsight.Inputs.ComputeProfileArgs
{
Roles = new[]
{
new AzureNative.HDInsight.Inputs.RoleArgs
{
HardwareProfile = new AzureNative.HDInsight.Inputs.HardwareProfileArgs
{
VmSize = "Standard_D3_V2",
},
MinInstanceCount = 1,
Name = "headnode",
OsProfile = new AzureNative.HDInsight.Inputs.OsProfileArgs
{
LinuxOperatingSystemProfile = new AzureNative.HDInsight.Inputs.LinuxOperatingSystemProfileArgs
{
Password = "**********",
Username = "sshuser",
},
},
TargetInstanceCount = 2,
},
new AzureNative.HDInsight.Inputs.RoleArgs
{
HardwareProfile = new AzureNative.HDInsight.Inputs.HardwareProfileArgs
{
VmSize = "Standard_D3_V2",
},
MinInstanceCount = 1,
Name = "workernode",
OsProfile = new AzureNative.HDInsight.Inputs.OsProfileArgs
{
LinuxOperatingSystemProfile = new AzureNative.HDInsight.Inputs.LinuxOperatingSystemProfileArgs
{
Password = "**********",
Username = "sshuser",
},
},
TargetInstanceCount = 4,
},
new AzureNative.HDInsight.Inputs.RoleArgs
{
HardwareProfile = new AzureNative.HDInsight.Inputs.HardwareProfileArgs
{
VmSize = "Small",
},
MinInstanceCount = 1,
Name = "zookeepernode",
OsProfile = new AzureNative.HDInsight.Inputs.OsProfileArgs
{
LinuxOperatingSystemProfile = new AzureNative.HDInsight.Inputs.LinuxOperatingSystemProfileArgs
{
Password = "**********",
Username = "sshuser",
},
},
TargetInstanceCount = 3,
},
},
},
OsType = AzureNative.HDInsight.OSType.Linux,
StorageProfile = new AzureNative.HDInsight.Inputs.StorageProfileArgs
{
Storageaccounts = new[]
{
new AzureNative.HDInsight.Inputs.StorageAccountArgs
{
EnableSecureChannel = true,
FileSystem = "default",
IsDefault = true,
Key = "storagekey",
Name = "mystorage.dfs.core.windows.net",
},
},
},
Tier = AzureNative.HDInsight.Tier.Standard,
},
ResourceGroupName = "rg1",
Tags =
{
{ "key1", "val1" },
},
});
});
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.azurenative.hdinsight.Cluster;
import com.pulumi.azurenative.hdinsight.ClusterArgs;
import com.pulumi.azurenative.hdinsight.inputs.ClusterCreatePropertiesArgs;
import com.pulumi.azurenative.hdinsight.inputs.ClusterDefinitionArgs;
import com.pulumi.azurenative.hdinsight.inputs.ComputeProfileArgs;
import com.pulumi.azurenative.hdinsight.inputs.StorageProfileArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var cluster = new Cluster("cluster", ClusterArgs.builder()
.clusterName("cluster1")
.properties(ClusterCreatePropertiesArgs.builder()
.clusterDefinition(ClusterDefinitionArgs.builder()
.configurations(Map.of("gateway", Map.ofEntries(
Map.entry("restAuthCredential.isEnabled", "true"),
Map.entry("restAuthCredential.password", "**********"),
Map.entry("restAuthCredential.username", "admin")
)))
.kind("Hadoop")
.build())
.clusterVersion("3.6")
.computeProfile(ComputeProfileArgs.builder()
.roles(
RoleArgs.builder()
.hardwareProfile(HardwareProfileArgs.builder()
.vmSize("Standard_D3_V2")
.build())
.minInstanceCount(1)
.name("headnode")
.osProfile(OsProfileArgs.builder()
.linuxOperatingSystemProfile(LinuxOperatingSystemProfileArgs.builder()
.password("**********")
.username("sshuser")
.build())
.build())
.targetInstanceCount(2)
.build(),
RoleArgs.builder()
.hardwareProfile(HardwareProfileArgs.builder()
.vmSize("Standard_D3_V2")
.build())
.minInstanceCount(1)
.name("workernode")
.osProfile(OsProfileArgs.builder()
.linuxOperatingSystemProfile(LinuxOperatingSystemProfileArgs.builder()
.password("**********")
.username("sshuser")
.build())
.build())
.targetInstanceCount(4)
.build(),
RoleArgs.builder()
.hardwareProfile(HardwareProfileArgs.builder()
.vmSize("Small")
.build())
.minInstanceCount(1)
.name("zookeepernode")
.osProfile(OsProfileArgs.builder()
.linuxOperatingSystemProfile(LinuxOperatingSystemProfileArgs.builder()
.password("**********")
.username("sshuser")
.build())
.build())
.targetInstanceCount(3)
.build())
.build())
.osType("Linux")
.storageProfile(StorageProfileArgs.builder()
.storageaccounts(StorageAccountArgs.builder()
.enableSecureChannel(true)
.fileSystem("default")
.isDefault(true)
.key("storagekey")
.name("mystorage.dfs.core.windows.net")
.build())
.build())
.tier("Standard")
.build())
.resourceGroupName("rg1")
.tags(Map.of("key1", "val1"))
.build());
}
}
resources:
cluster:
type: azure-native:hdinsight:Cluster
properties:
clusterName: cluster1
properties:
clusterDefinition:
configurations:
gateway:
restAuthCredential.isEnabled: 'true'
restAuthCredential.password: '**********'
restAuthCredential.username: admin
kind: Hadoop
clusterVersion: '3.6'
computeProfile:
roles:
- hardwareProfile:
vmSize: Standard_D3_V2
minInstanceCount: 1
name: headnode
osProfile:
linuxOperatingSystemProfile:
password: '**********'
username: sshuser
targetInstanceCount: 2
- hardwareProfile:
vmSize: Standard_D3_V2
minInstanceCount: 1
name: workernode
osProfile:
linuxOperatingSystemProfile:
password: '**********'
username: sshuser
targetInstanceCount: 4
- hardwareProfile:
vmSize: Small
minInstanceCount: 1
name: zookeepernode
osProfile:
linuxOperatingSystemProfile:
password: '**********'
username: sshuser
targetInstanceCount: 3
osType: Linux
storageProfile:
storageaccounts:
- enableSecureChannel: true
fileSystem: default
isDefault: true
key: storagekey
name: mystorage.dfs.core.windows.net
tier: Standard
resourceGroupName: rg1
tags:
key1: val1
The storageProfile property defines storage accounts the cluster can access. For ADLS Gen 2, set the fileSystem property to specify the container name and use the .dfs.core.windows.net endpoint in the name property. The cluster uses this storage for both system files and application data.
Configure SSH access with public key authentication
Production clusters typically require SSH access for administration and troubleshooting. Public key authentication provides more secure access than password-based authentication.
import * as pulumi from "@pulumi/pulumi";
import * as azure_native from "@pulumi/azure-native";
const cluster = new azure_native.hdinsight.Cluster("cluster", {
clusterName: "cluster1",
properties: {
clusterDefinition: {
configurations: {
gateway: {
"restAuthCredential.isEnabled": true,
"restAuthCredential.password": "**********",
"restAuthCredential.username": "admin",
},
},
kind: "Hadoop",
},
clusterVersion: "3.5",
computeProfile: {
roles: [
{
hardwareProfile: {
vmSize: "Standard_D3_V2",
},
minInstanceCount: 1,
name: "headnode",
osProfile: {
linuxOperatingSystemProfile: {
sshProfile: {
publicKeys: [{
certificateData: "**********",
}],
},
username: "sshuser",
},
},
targetInstanceCount: 2,
},
{
hardwareProfile: {
vmSize: "Standard_D3_V2",
},
minInstanceCount: 1,
name: "workernode",
osProfile: {
linuxOperatingSystemProfile: {
password: "**********",
username: "sshuser",
},
},
targetInstanceCount: 4,
},
{
hardwareProfile: {
vmSize: "Small",
},
minInstanceCount: 1,
name: "zookeepernode",
osProfile: {
linuxOperatingSystemProfile: {
password: "**********",
username: "sshuser",
},
},
targetInstanceCount: 3,
},
],
},
osType: azure_native.hdinsight.OSType.Linux,
storageProfile: {
storageaccounts: [{
container: "containername",
enableSecureChannel: true,
isDefault: true,
key: "storagekey",
name: "mystorage.blob.core.windows.net",
}],
},
tier: azure_native.hdinsight.Tier.Standard,
},
resourceGroupName: "rg1",
tags: {
key1: "val1",
},
});
import pulumi
import pulumi_azure_native as azure_native
cluster = azure_native.hdinsight.Cluster("cluster",
cluster_name="cluster1",
properties={
"cluster_definition": {
"configurations": {
"gateway": {
"restAuthCredential.isEnabled": True,
"restAuthCredential.password": "**********",
"restAuthCredential.username": "admin",
},
},
"kind": "Hadoop",
},
"cluster_version": "3.5",
"compute_profile": {
"roles": [
{
"hardware_profile": {
"vm_size": "Standard_D3_V2",
},
"min_instance_count": 1,
"name": "headnode",
"os_profile": {
"linux_operating_system_profile": {
"ssh_profile": {
"public_keys": [{
"certificate_data": "**********",
}],
},
"username": "sshuser",
},
},
"target_instance_count": 2,
},
{
"hardware_profile": {
"vm_size": "Standard_D3_V2",
},
"min_instance_count": 1,
"name": "workernode",
"os_profile": {
"linux_operating_system_profile": {
"password": "**********",
"username": "sshuser",
},
},
"target_instance_count": 4,
},
{
"hardware_profile": {
"vm_size": "Small",
},
"min_instance_count": 1,
"name": "zookeepernode",
"os_profile": {
"linux_operating_system_profile": {
"password": "**********",
"username": "sshuser",
},
},
"target_instance_count": 3,
},
],
},
"os_type": azure_native.hdinsight.OSType.LINUX,
"storage_profile": {
"storageaccounts": [{
"container": "containername",
"enable_secure_channel": True,
"is_default": True,
"key": "storagekey",
"name": "mystorage.blob.core.windows.net",
}],
},
"tier": azure_native.hdinsight.Tier.STANDARD,
},
resource_group_name="rg1",
tags={
"key1": "val1",
})
package main
import (
hdinsight "github.com/pulumi/pulumi-azure-native-sdk/hdinsight/v3"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
_, err := hdinsight.NewCluster(ctx, "cluster", &hdinsight.ClusterArgs{
ClusterName: pulumi.String("cluster1"),
Properties: &hdinsight.ClusterCreatePropertiesArgs{
ClusterDefinition: &hdinsight.ClusterDefinitionArgs{
Configurations: pulumi.Any(map[string]interface{}{
"gateway": map[string]interface{}{
"restAuthCredential.isEnabled": true,
"restAuthCredential.password": "**********",
"restAuthCredential.username": "admin",
},
}),
Kind: pulumi.String("Hadoop"),
},
ClusterVersion: pulumi.String("3.5"),
ComputeProfile: &hdinsight.ComputeProfileArgs{
Roles: hdinsight.RoleArray{
&hdinsight.RoleArgs{
HardwareProfile: &hdinsight.HardwareProfileArgs{
VmSize: pulumi.String("Standard_D3_V2"),
},
MinInstanceCount: pulumi.Int(1),
Name: pulumi.String("headnode"),
OsProfile: &hdinsight.OsProfileArgs{
LinuxOperatingSystemProfile: &hdinsight.LinuxOperatingSystemProfileArgs{
SshProfile: &hdinsight.SshProfileArgs{
PublicKeys: hdinsight.SshPublicKeyArray{
&hdinsight.SshPublicKeyArgs{
CertificateData: pulumi.String("**********"),
},
},
},
Username: pulumi.String("sshuser"),
},
},
TargetInstanceCount: pulumi.Int(2),
},
&hdinsight.RoleArgs{
HardwareProfile: &hdinsight.HardwareProfileArgs{
VmSize: pulumi.String("Standard_D3_V2"),
},
MinInstanceCount: pulumi.Int(1),
Name: pulumi.String("workernode"),
OsProfile: &hdinsight.OsProfileArgs{
LinuxOperatingSystemProfile: &hdinsight.LinuxOperatingSystemProfileArgs{
Password: pulumi.String("**********"),
Username: pulumi.String("sshuser"),
},
},
TargetInstanceCount: pulumi.Int(4),
},
&hdinsight.RoleArgs{
HardwareProfile: &hdinsight.HardwareProfileArgs{
VmSize: pulumi.String("Small"),
},
MinInstanceCount: pulumi.Int(1),
Name: pulumi.String("zookeepernode"),
OsProfile: &hdinsight.OsProfileArgs{
LinuxOperatingSystemProfile: &hdinsight.LinuxOperatingSystemProfileArgs{
Password: pulumi.String("**********"),
Username: pulumi.String("sshuser"),
},
},
TargetInstanceCount: pulumi.Int(3),
},
},
},
OsType: pulumi.String(hdinsight.OSTypeLinux),
StorageProfile: &hdinsight.StorageProfileArgs{
Storageaccounts: hdinsight.StorageAccountArray{
&hdinsight.StorageAccountArgs{
Container: pulumi.String("containername"),
EnableSecureChannel: pulumi.Bool(true),
IsDefault: pulumi.Bool(true),
Key: pulumi.String("storagekey"),
Name: pulumi.String("mystorage.blob.core.windows.net"),
},
},
},
Tier: pulumi.String(hdinsight.TierStandard),
},
ResourceGroupName: pulumi.String("rg1"),
Tags: pulumi.StringMap{
"key1": pulumi.String("val1"),
},
})
if err != nil {
return err
}
return nil
})
}
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using AzureNative = Pulumi.AzureNative;
return await Deployment.RunAsync(() =>
{
var cluster = new AzureNative.HDInsight.Cluster("cluster", new()
{
ClusterName = "cluster1",
Properties = new AzureNative.HDInsight.Inputs.ClusterCreatePropertiesArgs
{
ClusterDefinition = new AzureNative.HDInsight.Inputs.ClusterDefinitionArgs
{
Configurations = new Dictionary<string, object?>
{
["gateway"] = new Dictionary<string, object?>
{
["restAuthCredential.isEnabled"] = true,
["restAuthCredential.password"] = "**********",
["restAuthCredential.username"] = "admin",
},
},
Kind = "Hadoop",
},
ClusterVersion = "3.5",
ComputeProfile = new AzureNative.HDInsight.Inputs.ComputeProfileArgs
{
Roles = new[]
{
new AzureNative.HDInsight.Inputs.RoleArgs
{
HardwareProfile = new AzureNative.HDInsight.Inputs.HardwareProfileArgs
{
VmSize = "Standard_D3_V2",
},
MinInstanceCount = 1,
Name = "headnode",
OsProfile = new AzureNative.HDInsight.Inputs.OsProfileArgs
{
LinuxOperatingSystemProfile = new AzureNative.HDInsight.Inputs.LinuxOperatingSystemProfileArgs
{
SshProfile = new AzureNative.HDInsight.Inputs.SshProfileArgs
{
PublicKeys = new[]
{
new AzureNative.HDInsight.Inputs.SshPublicKeyArgs
{
CertificateData = "**********",
},
},
},
Username = "sshuser",
},
},
TargetInstanceCount = 2,
},
new AzureNative.HDInsight.Inputs.RoleArgs
{
HardwareProfile = new AzureNative.HDInsight.Inputs.HardwareProfileArgs
{
VmSize = "Standard_D3_V2",
},
MinInstanceCount = 1,
Name = "workernode",
OsProfile = new AzureNative.HDInsight.Inputs.OsProfileArgs
{
LinuxOperatingSystemProfile = new AzureNative.HDInsight.Inputs.LinuxOperatingSystemProfileArgs
{
Password = "**********",
Username = "sshuser",
},
},
TargetInstanceCount = 4,
},
new AzureNative.HDInsight.Inputs.RoleArgs
{
HardwareProfile = new AzureNative.HDInsight.Inputs.HardwareProfileArgs
{
VmSize = "Small",
},
MinInstanceCount = 1,
Name = "zookeepernode",
OsProfile = new AzureNative.HDInsight.Inputs.OsProfileArgs
{
LinuxOperatingSystemProfile = new AzureNative.HDInsight.Inputs.LinuxOperatingSystemProfileArgs
{
Password = "**********",
Username = "sshuser",
},
},
TargetInstanceCount = 3,
},
},
},
OsType = AzureNative.HDInsight.OSType.Linux,
StorageProfile = new AzureNative.HDInsight.Inputs.StorageProfileArgs
{
Storageaccounts = new[]
{
new AzureNative.HDInsight.Inputs.StorageAccountArgs
{
Container = "containername",
EnableSecureChannel = true,
IsDefault = true,
Key = "storagekey",
Name = "mystorage.blob.core.windows.net",
},
},
},
Tier = AzureNative.HDInsight.Tier.Standard,
},
ResourceGroupName = "rg1",
Tags =
{
{ "key1", "val1" },
},
});
});
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.azurenative.hdinsight.Cluster;
import com.pulumi.azurenative.hdinsight.ClusterArgs;
import com.pulumi.azurenative.hdinsight.inputs.ClusterCreatePropertiesArgs;
import com.pulumi.azurenative.hdinsight.inputs.ClusterDefinitionArgs;
import com.pulumi.azurenative.hdinsight.inputs.ComputeProfileArgs;
import com.pulumi.azurenative.hdinsight.inputs.StorageProfileArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var cluster = new Cluster("cluster", ClusterArgs.builder()
.clusterName("cluster1")
.properties(ClusterCreatePropertiesArgs.builder()
.clusterDefinition(ClusterDefinitionArgs.builder()
.configurations(Map.of("gateway", Map.ofEntries(
Map.entry("restAuthCredential.isEnabled", true),
Map.entry("restAuthCredential.password", "**********"),
Map.entry("restAuthCredential.username", "admin")
)))
.kind("Hadoop")
.build())
.clusterVersion("3.5")
.computeProfile(ComputeProfileArgs.builder()
.roles(
RoleArgs.builder()
.hardwareProfile(HardwareProfileArgs.builder()
.vmSize("Standard_D3_V2")
.build())
.minInstanceCount(1)
.name("headnode")
.osProfile(OsProfileArgs.builder()
.linuxOperatingSystemProfile(LinuxOperatingSystemProfileArgs.builder()
.sshProfile(SshProfileArgs.builder()
.publicKeys(SshPublicKeyArgs.builder()
.certificateData("**********")
.build())
.build())
.username("sshuser")
.build())
.build())
.targetInstanceCount(2)
.build(),
RoleArgs.builder()
.hardwareProfile(HardwareProfileArgs.builder()
.vmSize("Standard_D3_V2")
.build())
.minInstanceCount(1)
.name("workernode")
.osProfile(OsProfileArgs.builder()
.linuxOperatingSystemProfile(LinuxOperatingSystemProfileArgs.builder()
.password("**********")
.username("sshuser")
.build())
.build())
.targetInstanceCount(4)
.build(),
RoleArgs.builder()
.hardwareProfile(HardwareProfileArgs.builder()
.vmSize("Small")
.build())
.minInstanceCount(1)
.name("zookeepernode")
.osProfile(OsProfileArgs.builder()
.linuxOperatingSystemProfile(LinuxOperatingSystemProfileArgs.builder()
.password("**********")
.username("sshuser")
.build())
.build())
.targetInstanceCount(3)
.build())
.build())
.osType("Linux")
.storageProfile(StorageProfileArgs.builder()
.storageaccounts(StorageAccountArgs.builder()
.container("containername")
.enableSecureChannel(true)
.isDefault(true)
.key("storagekey")
.name("mystorage.blob.core.windows.net")
.build())
.build())
.tier("Standard")
.build())
.resourceGroupName("rg1")
.tags(Map.of("key1", "val1"))
.build());
}
}
resources:
cluster:
type: azure-native:hdinsight:Cluster
properties:
clusterName: cluster1
properties:
clusterDefinition:
configurations:
gateway:
restAuthCredential.isEnabled: true
restAuthCredential.password: '**********'
restAuthCredential.username: admin
kind: Hadoop
clusterVersion: '3.5'
computeProfile:
roles:
- hardwareProfile:
vmSize: Standard_D3_V2
minInstanceCount: 1
name: headnode
osProfile:
linuxOperatingSystemProfile:
sshProfile:
publicKeys:
- certificateData: '**********'
username: sshuser
targetInstanceCount: 2
- hardwareProfile:
vmSize: Standard_D3_V2
minInstanceCount: 1
name: workernode
osProfile:
linuxOperatingSystemProfile:
password: '**********'
username: sshuser
targetInstanceCount: 4
- hardwareProfile:
vmSize: Small
minInstanceCount: 1
name: zookeepernode
osProfile:
linuxOperatingSystemProfile:
password: '**********'
username: sshuser
targetInstanceCount: 3
osType: Linux
storageProfile:
storageaccounts:
- container: containername
enableSecureChannel: true
isDefault: true
key: storagekey
name: mystorage.blob.core.windows.net
tier: Standard
resourceGroupName: rg1
tags:
key1: val1
The linuxOperatingSystemProfile contains an sshProfile block where you provide public keys via the publicKeys array. Each entry includes certificateData with your SSH public key. The cluster configures SSH access for the specified username using these keys, allowing secure remote access without passwords.
Join clusters to Active Directory domains
Enterprise deployments often require clusters to integrate with existing Active Directory infrastructure for centralized authentication and authorization.
import * as pulumi from "@pulumi/pulumi";
import * as azure_native from "@pulumi/azure-native";
const cluster = new azure_native.hdinsight.Cluster("cluster", {
clusterName: "cluster1",
properties: {
clusterDefinition: {
configurations: {
gateway: {
"restAuthCredential.isEnabled": true,
"restAuthCredential.password": "**********",
"restAuthCredential.username": "admin",
},
},
kind: "Hadoop",
},
clusterVersion: "3.5",
computeProfile: {
roles: [
{
hardwareProfile: {
vmSize: "Standard_D3_V2",
},
minInstanceCount: 1,
name: "headnode",
osProfile: {
linuxOperatingSystemProfile: {
password: "**********",
sshProfile: {
publicKeys: [{
certificateData: "**********",
}],
},
username: "sshuser",
},
},
scriptActions: [],
targetInstanceCount: 2,
virtualNetworkProfile: {
id: "/subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname",
subnet: "/subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname/subnets/vnetsubnet",
},
},
{
hardwareProfile: {
vmSize: "Standard_D3_V2",
},
minInstanceCount: 1,
name: "workernode",
osProfile: {
linuxOperatingSystemProfile: {
password: "**********",
sshProfile: {
publicKeys: [{
certificateData: "**********",
}],
},
username: "sshuser",
},
},
scriptActions: [],
targetInstanceCount: 4,
virtualNetworkProfile: {
id: "/subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname",
subnet: "/subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname/subnets/vnetsubnet",
},
},
{
hardwareProfile: {
vmSize: "Small",
},
minInstanceCount: 1,
name: "zookeepernode",
osProfile: {
linuxOperatingSystemProfile: {
password: "**********",
sshProfile: {
publicKeys: [{
certificateData: "**********",
}],
},
username: "sshuser",
},
},
scriptActions: [],
targetInstanceCount: 3,
virtualNetworkProfile: {
id: "/subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname",
subnet: "/subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname/subnets/vnetsubnet",
},
},
],
},
osType: azure_native.hdinsight.OSType.Linux,
securityProfile: {
clusterUsersGroupDNs: ["hdiusers"],
directoryType: azure_native.hdinsight.DirectoryType.ActiveDirectory,
domain: "DomainName",
domainUserPassword: "**********",
domainUsername: "DomainUsername",
ldapsUrls: ["ldaps://10.10.0.4:636"],
organizationalUnitDN: "OU=Hadoop,DC=hdinsight,DC=test",
},
storageProfile: {
storageaccounts: [{
container: "containername",
enableSecureChannel: true,
isDefault: true,
key: "storage account key",
name: "mystorage.blob.core.windows.net",
}],
},
tier: azure_native.hdinsight.Tier.Premium,
},
resourceGroupName: "rg1",
tags: {
key1: "val1",
},
});
import pulumi
import pulumi_azure_native as azure_native
cluster = azure_native.hdinsight.Cluster("cluster",
cluster_name="cluster1",
properties={
"cluster_definition": {
"configurations": {
"gateway": {
"restAuthCredential.isEnabled": True,
"restAuthCredential.password": "**********",
"restAuthCredential.username": "admin",
},
},
"kind": "Hadoop",
},
"cluster_version": "3.5",
"compute_profile": {
"roles": [
{
"hardware_profile": {
"vm_size": "Standard_D3_V2",
},
"min_instance_count": 1,
"name": "headnode",
"os_profile": {
"linux_operating_system_profile": {
"password": "**********",
"ssh_profile": {
"public_keys": [{
"certificate_data": "**********",
}],
},
"username": "sshuser",
},
},
"script_actions": [],
"target_instance_count": 2,
"virtual_network_profile": {
"id": "/subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname",
"subnet": "/subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname/subnets/vnetsubnet",
},
},
{
"hardware_profile": {
"vm_size": "Standard_D3_V2",
},
"min_instance_count": 1,
"name": "workernode",
"os_profile": {
"linux_operating_system_profile": {
"password": "**********",
"ssh_profile": {
"public_keys": [{
"certificate_data": "**********",
}],
},
"username": "sshuser",
},
},
"script_actions": [],
"target_instance_count": 4,
"virtual_network_profile": {
"id": "/subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname",
"subnet": "/subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname/subnets/vnetsubnet",
},
},
{
"hardware_profile": {
"vm_size": "Small",
},
"min_instance_count": 1,
"name": "zookeepernode",
"os_profile": {
"linux_operating_system_profile": {
"password": "**********",
"ssh_profile": {
"public_keys": [{
"certificate_data": "**********",
}],
},
"username": "sshuser",
},
},
"script_actions": [],
"target_instance_count": 3,
"virtual_network_profile": {
"id": "/subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname",
"subnet": "/subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname/subnets/vnetsubnet",
},
},
],
},
"os_type": azure_native.hdinsight.OSType.LINUX,
"security_profile": {
"cluster_users_group_dns": ["hdiusers"],
"directory_type": azure_native.hdinsight.DirectoryType.ACTIVE_DIRECTORY,
"domain": "DomainName",
"domain_user_password": "**********",
"domain_username": "DomainUsername",
"ldaps_urls": ["ldaps://10.10.0.4:636"],
"organizational_unit_dn": "OU=Hadoop,DC=hdinsight,DC=test",
},
"storage_profile": {
"storageaccounts": [{
"container": "containername",
"enable_secure_channel": True,
"is_default": True,
"key": "storage account key",
"name": "mystorage.blob.core.windows.net",
}],
},
"tier": azure_native.hdinsight.Tier.PREMIUM,
},
resource_group_name="rg1",
tags={
"key1": "val1",
})
package main
import (
hdinsight "github.com/pulumi/pulumi-azure-native-sdk/hdinsight/v3"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
_, err := hdinsight.NewCluster(ctx, "cluster", &hdinsight.ClusterArgs{
ClusterName: pulumi.String("cluster1"),
Properties: &hdinsight.ClusterCreatePropertiesArgs{
ClusterDefinition: &hdinsight.ClusterDefinitionArgs{
Configurations: pulumi.Any(map[string]interface{}{
"gateway": map[string]interface{}{
"restAuthCredential.isEnabled": true,
"restAuthCredential.password": "**********",
"restAuthCredential.username": "admin",
},
}),
Kind: pulumi.String("Hadoop"),
},
ClusterVersion: pulumi.String("3.5"),
ComputeProfile: &hdinsight.ComputeProfileArgs{
Roles: hdinsight.RoleArray{
&hdinsight.RoleArgs{
HardwareProfile: &hdinsight.HardwareProfileArgs{
VmSize: pulumi.String("Standard_D3_V2"),
},
MinInstanceCount: pulumi.Int(1),
Name: pulumi.String("headnode"),
OsProfile: &hdinsight.OsProfileArgs{
LinuxOperatingSystemProfile: &hdinsight.LinuxOperatingSystemProfileArgs{
Password: pulumi.String("**********"),
SshProfile: &hdinsight.SshProfileArgs{
PublicKeys: hdinsight.SshPublicKeyArray{
&hdinsight.SshPublicKeyArgs{
CertificateData: pulumi.String("**********"),
},
},
},
Username: pulumi.String("sshuser"),
},
},
ScriptActions: hdinsight.ScriptActionArray{},
TargetInstanceCount: pulumi.Int(2),
VirtualNetworkProfile: &hdinsight.VirtualNetworkProfileArgs{
Id: pulumi.String("/subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname"),
Subnet: pulumi.String("/subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname/subnets/vnetsubnet"),
},
},
&hdinsight.RoleArgs{
HardwareProfile: &hdinsight.HardwareProfileArgs{
VmSize: pulumi.String("Standard_D3_V2"),
},
MinInstanceCount: pulumi.Int(1),
Name: pulumi.String("workernode"),
OsProfile: &hdinsight.OsProfileArgs{
LinuxOperatingSystemProfile: &hdinsight.LinuxOperatingSystemProfileArgs{
Password: pulumi.String("**********"),
SshProfile: &hdinsight.SshProfileArgs{
PublicKeys: hdinsight.SshPublicKeyArray{
&hdinsight.SshPublicKeyArgs{
CertificateData: pulumi.String("**********"),
},
},
},
Username: pulumi.String("sshuser"),
},
},
ScriptActions: hdinsight.ScriptActionArray{},
TargetInstanceCount: pulumi.Int(4),
VirtualNetworkProfile: &hdinsight.VirtualNetworkProfileArgs{
Id: pulumi.String("/subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname"),
Subnet: pulumi.String("/subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname/subnets/vnetsubnet"),
},
},
&hdinsight.RoleArgs{
HardwareProfile: &hdinsight.HardwareProfileArgs{
VmSize: pulumi.String("Small"),
},
MinInstanceCount: pulumi.Int(1),
Name: pulumi.String("zookeepernode"),
OsProfile: &hdinsight.OsProfileArgs{
LinuxOperatingSystemProfile: &hdinsight.LinuxOperatingSystemProfileArgs{
Password: pulumi.String("**********"),
SshProfile: &hdinsight.SshProfileArgs{
PublicKeys: hdinsight.SshPublicKeyArray{
&hdinsight.SshPublicKeyArgs{
CertificateData: pulumi.String("**********"),
},
},
},
Username: pulumi.String("sshuser"),
},
},
ScriptActions: hdinsight.ScriptActionArray{},
TargetInstanceCount: pulumi.Int(3),
VirtualNetworkProfile: &hdinsight.VirtualNetworkProfileArgs{
Id: pulumi.String("/subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname"),
Subnet: pulumi.String("/subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname/subnets/vnetsubnet"),
},
},
},
},
OsType: pulumi.String(hdinsight.OSTypeLinux),
SecurityProfile: &hdinsight.SecurityProfileArgs{
ClusterUsersGroupDNs: pulumi.StringArray{
pulumi.String("hdiusers"),
},
DirectoryType: pulumi.String(hdinsight.DirectoryTypeActiveDirectory),
Domain: pulumi.String("DomainName"),
DomainUserPassword: pulumi.String("**********"),
DomainUsername: pulumi.String("DomainUsername"),
LdapsUrls: pulumi.StringArray{
pulumi.String("ldaps://10.10.0.4:636"),
},
OrganizationalUnitDN: pulumi.String("OU=Hadoop,DC=hdinsight,DC=test"),
},
StorageProfile: &hdinsight.StorageProfileArgs{
Storageaccounts: hdinsight.StorageAccountArray{
&hdinsight.StorageAccountArgs{
Container: pulumi.String("containername"),
EnableSecureChannel: pulumi.Bool(true),
IsDefault: pulumi.Bool(true),
Key: pulumi.String("storage account key"),
Name: pulumi.String("mystorage.blob.core.windows.net"),
},
},
},
Tier: pulumi.String(hdinsight.TierPremium),
},
ResourceGroupName: pulumi.String("rg1"),
Tags: pulumi.StringMap{
"key1": pulumi.String("val1"),
},
})
if err != nil {
return err
}
return nil
})
}
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using AzureNative = Pulumi.AzureNative;
return await Deployment.RunAsync(() =>
{
var cluster = new AzureNative.HDInsight.Cluster("cluster", new()
{
ClusterName = "cluster1",
Properties = new AzureNative.HDInsight.Inputs.ClusterCreatePropertiesArgs
{
ClusterDefinition = new AzureNative.HDInsight.Inputs.ClusterDefinitionArgs
{
Configurations = new Dictionary<string, object?>
{
["gateway"] = new Dictionary<string, object?>
{
["restAuthCredential.isEnabled"] = true,
["restAuthCredential.password"] = "**********",
["restAuthCredential.username"] = "admin",
},
},
Kind = "Hadoop",
},
ClusterVersion = "3.5",
ComputeProfile = new AzureNative.HDInsight.Inputs.ComputeProfileArgs
{
Roles = new[]
{
new AzureNative.HDInsight.Inputs.RoleArgs
{
HardwareProfile = new AzureNative.HDInsight.Inputs.HardwareProfileArgs
{
VmSize = "Standard_D3_V2",
},
MinInstanceCount = 1,
Name = "headnode",
OsProfile = new AzureNative.HDInsight.Inputs.OsProfileArgs
{
LinuxOperatingSystemProfile = new AzureNative.HDInsight.Inputs.LinuxOperatingSystemProfileArgs
{
Password = "**********",
SshProfile = new AzureNative.HDInsight.Inputs.SshProfileArgs
{
PublicKeys = new[]
{
new AzureNative.HDInsight.Inputs.SshPublicKeyArgs
{
CertificateData = "**********",
},
},
},
Username = "sshuser",
},
},
ScriptActions = new() { },
TargetInstanceCount = 2,
VirtualNetworkProfile = new AzureNative.HDInsight.Inputs.VirtualNetworkProfileArgs
{
Id = "/subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname",
Subnet = "/subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname/subnets/vnetsubnet",
},
},
new AzureNative.HDInsight.Inputs.RoleArgs
{
HardwareProfile = new AzureNative.HDInsight.Inputs.HardwareProfileArgs
{
VmSize = "Standard_D3_V2",
},
MinInstanceCount = 1,
Name = "workernode",
OsProfile = new AzureNative.HDInsight.Inputs.OsProfileArgs
{
LinuxOperatingSystemProfile = new AzureNative.HDInsight.Inputs.LinuxOperatingSystemProfileArgs
{
Password = "**********",
SshProfile = new AzureNative.HDInsight.Inputs.SshProfileArgs
{
PublicKeys = new[]
{
new AzureNative.HDInsight.Inputs.SshPublicKeyArgs
{
CertificateData = "**********",
},
},
},
Username = "sshuser",
},
},
ScriptActions = new() { },
TargetInstanceCount = 4,
VirtualNetworkProfile = new AzureNative.HDInsight.Inputs.VirtualNetworkProfileArgs
{
Id = "/subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname",
Subnet = "/subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname/subnets/vnetsubnet",
},
},
new AzureNative.HDInsight.Inputs.RoleArgs
{
HardwareProfile = new AzureNative.HDInsight.Inputs.HardwareProfileArgs
{
VmSize = "Small",
},
MinInstanceCount = 1,
Name = "zookeepernode",
OsProfile = new AzureNative.HDInsight.Inputs.OsProfileArgs
{
LinuxOperatingSystemProfile = new AzureNative.HDInsight.Inputs.LinuxOperatingSystemProfileArgs
{
Password = "**********",
SshProfile = new AzureNative.HDInsight.Inputs.SshProfileArgs
{
PublicKeys = new[]
{
new AzureNative.HDInsight.Inputs.SshPublicKeyArgs
{
CertificateData = "**********",
},
},
},
Username = "sshuser",
},
},
ScriptActions = new() { },
TargetInstanceCount = 3,
VirtualNetworkProfile = new AzureNative.HDInsight.Inputs.VirtualNetworkProfileArgs
{
Id = "/subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname",
Subnet = "/subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname/subnets/vnetsubnet",
},
},
},
},
OsType = AzureNative.HDInsight.OSType.Linux,
SecurityProfile = new AzureNative.HDInsight.Inputs.SecurityProfileArgs
{
ClusterUsersGroupDNs = new[]
{
"hdiusers",
},
DirectoryType = AzureNative.HDInsight.DirectoryType.ActiveDirectory,
Domain = "DomainName",
DomainUserPassword = "**********",
DomainUsername = "DomainUsername",
LdapsUrls = new[]
{
"ldaps://10.10.0.4:636",
},
OrganizationalUnitDN = "OU=Hadoop,DC=hdinsight,DC=test",
},
StorageProfile = new AzureNative.HDInsight.Inputs.StorageProfileArgs
{
Storageaccounts = new[]
{
new AzureNative.HDInsight.Inputs.StorageAccountArgs
{
Container = "containername",
EnableSecureChannel = true,
IsDefault = true,
Key = "storage account key",
Name = "mystorage.blob.core.windows.net",
},
},
},
Tier = AzureNative.HDInsight.Tier.Premium,
},
ResourceGroupName = "rg1",
Tags =
{
{ "key1", "val1" },
},
});
});
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.azurenative.hdinsight.Cluster;
import com.pulumi.azurenative.hdinsight.ClusterArgs;
import com.pulumi.azurenative.hdinsight.inputs.ClusterCreatePropertiesArgs;
import com.pulumi.azurenative.hdinsight.inputs.ClusterDefinitionArgs;
import com.pulumi.azurenative.hdinsight.inputs.ComputeProfileArgs;
import com.pulumi.azurenative.hdinsight.inputs.SecurityProfileArgs;
import com.pulumi.azurenative.hdinsight.inputs.StorageProfileArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var cluster = new Cluster("cluster", ClusterArgs.builder()
.clusterName("cluster1")
.properties(ClusterCreatePropertiesArgs.builder()
.clusterDefinition(ClusterDefinitionArgs.builder()
.configurations(Map.of("gateway", Map.ofEntries(
Map.entry("restAuthCredential.isEnabled", true),
Map.entry("restAuthCredential.password", "**********"),
Map.entry("restAuthCredential.username", "admin")
)))
.kind("Hadoop")
.build())
.clusterVersion("3.5")
.computeProfile(ComputeProfileArgs.builder()
.roles(
RoleArgs.builder()
.hardwareProfile(HardwareProfileArgs.builder()
.vmSize("Standard_D3_V2")
.build())
.minInstanceCount(1)
.name("headnode")
.osProfile(OsProfileArgs.builder()
.linuxOperatingSystemProfile(LinuxOperatingSystemProfileArgs.builder()
.password("**********")
.sshProfile(SshProfileArgs.builder()
.publicKeys(SshPublicKeyArgs.builder()
.certificateData("**********")
.build())
.build())
.username("sshuser")
.build())
.build())
.scriptActions()
.targetInstanceCount(2)
.virtualNetworkProfile(VirtualNetworkProfileArgs.builder()
.id("/subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname")
.subnet("/subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname/subnets/vnetsubnet")
.build())
.build(),
RoleArgs.builder()
.hardwareProfile(HardwareProfileArgs.builder()
.vmSize("Standard_D3_V2")
.build())
.minInstanceCount(1)
.name("workernode")
.osProfile(OsProfileArgs.builder()
.linuxOperatingSystemProfile(LinuxOperatingSystemProfileArgs.builder()
.password("**********")
.sshProfile(SshProfileArgs.builder()
.publicKeys(SshPublicKeyArgs.builder()
.certificateData("**********")
.build())
.build())
.username("sshuser")
.build())
.build())
.scriptActions()
.targetInstanceCount(4)
.virtualNetworkProfile(VirtualNetworkProfileArgs.builder()
.id("/subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname")
.subnet("/subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname/subnets/vnetsubnet")
.build())
.build(),
RoleArgs.builder()
.hardwareProfile(HardwareProfileArgs.builder()
.vmSize("Small")
.build())
.minInstanceCount(1)
.name("zookeepernode")
.osProfile(OsProfileArgs.builder()
.linuxOperatingSystemProfile(LinuxOperatingSystemProfileArgs.builder()
.password("**********")
.sshProfile(SshProfileArgs.builder()
.publicKeys(SshPublicKeyArgs.builder()
.certificateData("**********")
.build())
.build())
.username("sshuser")
.build())
.build())
.scriptActions()
.targetInstanceCount(3)
.virtualNetworkProfile(VirtualNetworkProfileArgs.builder()
.id("/subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname")
.subnet("/subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname/subnets/vnetsubnet")
.build())
.build())
.build())
.osType("Linux")
.securityProfile(SecurityProfileArgs.builder()
.clusterUsersGroupDNs("hdiusers")
.directoryType("ActiveDirectory")
.domain("DomainName")
.domainUserPassword("**********")
.domainUsername("DomainUsername")
.ldapsUrls("ldaps://10.10.0.4:636")
.organizationalUnitDN("OU=Hadoop,DC=hdinsight,DC=test")
.build())
.storageProfile(StorageProfileArgs.builder()
.storageaccounts(StorageAccountArgs.builder()
.container("containername")
.enableSecureChannel(true)
.isDefault(true)
.key("storage account key")
.name("mystorage.blob.core.windows.net")
.build())
.build())
.tier("Premium")
.build())
.resourceGroupName("rg1")
.tags(Map.of("key1", "val1"))
.build());
}
}
resources:
cluster:
type: azure-native:hdinsight:Cluster
properties:
clusterName: cluster1
properties:
clusterDefinition:
configurations:
gateway:
restAuthCredential.isEnabled: true
restAuthCredential.password: '**********'
restAuthCredential.username: admin
kind: Hadoop
clusterVersion: '3.5'
computeProfile:
roles:
- hardwareProfile:
vmSize: Standard_D3_V2
minInstanceCount: 1
name: headnode
osProfile:
linuxOperatingSystemProfile:
password: '**********'
sshProfile:
publicKeys:
- certificateData: '**********'
username: sshuser
scriptActions: []
targetInstanceCount: 2
virtualNetworkProfile:
id: /subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname
subnet: /subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname/subnets/vnetsubnet
- hardwareProfile:
vmSize: Standard_D3_V2
minInstanceCount: 1
name: workernode
osProfile:
linuxOperatingSystemProfile:
password: '**********'
sshProfile:
publicKeys:
- certificateData: '**********'
username: sshuser
scriptActions: []
targetInstanceCount: 4
virtualNetworkProfile:
id: /subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname
subnet: /subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname/subnets/vnetsubnet
- hardwareProfile:
vmSize: Small
minInstanceCount: 1
name: zookeepernode
osProfile:
linuxOperatingSystemProfile:
password: '**********'
sshProfile:
publicKeys:
- certificateData: '**********'
username: sshuser
scriptActions: []
targetInstanceCount: 3
virtualNetworkProfile:
id: /subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname
subnet: /subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname/subnets/vnetsubnet
osType: Linux
securityProfile:
clusterUsersGroupDNs:
- hdiusers
directoryType: ActiveDirectory
domain: DomainName
domainUserPassword: '**********'
domainUsername: DomainUsername
ldapsUrls:
- ldaps://10.10.0.4:636
organizationalUnitDN: OU=Hadoop,DC=hdinsight,DC=test
storageProfile:
storageaccounts:
- container: containername
enableSecureChannel: true
isDefault: true
key: storage account key
name: mystorage.blob.core.windows.net
tier: Premium
resourceGroupName: rg1
tags:
key1: val1
The securityProfile property enables domain integration. Set directoryType to ActiveDirectory, provide the domain name, and specify ldapsUrls pointing to your LDAP servers. The organizationalUnitDN defines where cluster computer accounts are created, and clusterUsersGroupDNs lists AD groups that can access the cluster. The cluster joins the domain at creation time and enforces group-based access control.
Configure private link and outbound connectivity
Clusters deployed in virtual networks may need private link endpoints for secure access and specific resource provider connection modes for outbound traffic control.
import * as pulumi from "@pulumi/pulumi";
import * as azure_native from "@pulumi/azure-native";
const cluster = new azure_native.hdinsight.Cluster("cluster", {
clusterName: "cluster1",
properties: {
clusterDefinition: {
configurations: {
gateway: {
"restAuthCredential.isEnabled": true,
"restAuthCredential.password": "**********",
"restAuthCredential.username": "admin",
},
},
kind: "hadoop",
},
clusterVersion: "3.6",
computeProfile: {
roles: [
{
hardwareProfile: {
vmSize: "standard_d3",
},
name: "headnode",
osProfile: {
linuxOperatingSystemProfile: {
password: "**********",
sshProfile: {
publicKeys: [{
certificateData: "**********",
}],
},
username: "sshuser",
},
},
targetInstanceCount: 2,
virtualNetworkProfile: {
id: "/subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname",
subnet: "/subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname/subnets/vnetsubnet",
},
},
{
hardwareProfile: {
vmSize: "standard_d3",
},
name: "workernode",
osProfile: {
linuxOperatingSystemProfile: {
password: "**********",
sshProfile: {
publicKeys: [{
certificateData: "**********",
}],
},
username: "sshuser",
},
},
targetInstanceCount: 2,
virtualNetworkProfile: {
id: "/subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname",
subnet: "/subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname/subnets/vnetsubnet",
},
},
],
},
networkProperties: {
privateLink: azure_native.hdinsight.PrivateLink.Enabled,
publicIpTag: {
ipTagType: "FirstPartyUsage",
tag: "/<TagName>",
},
resourceProviderConnection: azure_native.hdinsight.ResourceProviderConnection.Outbound,
},
osType: azure_native.hdinsight.OSType.Linux,
storageProfile: {
storageaccounts: [{
container: "containername",
enableSecureChannel: true,
isDefault: true,
key: "storage account key",
name: "mystorage",
}],
},
},
resourceGroupName: "rg1",
});
import pulumi
import pulumi_azure_native as azure_native
cluster = azure_native.hdinsight.Cluster("cluster",
cluster_name="cluster1",
properties={
"cluster_definition": {
"configurations": {
"gateway": {
"restAuthCredential.isEnabled": True,
"restAuthCredential.password": "**********",
"restAuthCredential.username": "admin",
},
},
"kind": "hadoop",
},
"cluster_version": "3.6",
"compute_profile": {
"roles": [
{
"hardware_profile": {
"vm_size": "standard_d3",
},
"name": "headnode",
"os_profile": {
"linux_operating_system_profile": {
"password": "**********",
"ssh_profile": {
"public_keys": [{
"certificate_data": "**********",
}],
},
"username": "sshuser",
},
},
"target_instance_count": 2,
"virtual_network_profile": {
"id": "/subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname",
"subnet": "/subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname/subnets/vnetsubnet",
},
},
{
"hardware_profile": {
"vm_size": "standard_d3",
},
"name": "workernode",
"os_profile": {
"linux_operating_system_profile": {
"password": "**********",
"ssh_profile": {
"public_keys": [{
"certificate_data": "**********",
}],
},
"username": "sshuser",
},
},
"target_instance_count": 2,
"virtual_network_profile": {
"id": "/subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname",
"subnet": "/subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname/subnets/vnetsubnet",
},
},
],
},
"network_properties": {
"private_link": azure_native.hdinsight.PrivateLink.ENABLED,
"public_ip_tag": {
"ip_tag_type": "FirstPartyUsage",
"tag": "/<TagName>",
},
"resource_provider_connection": azure_native.hdinsight.ResourceProviderConnection.OUTBOUND,
},
"os_type": azure_native.hdinsight.OSType.LINUX,
"storage_profile": {
"storageaccounts": [{
"container": "containername",
"enable_secure_channel": True,
"is_default": True,
"key": "storage account key",
"name": "mystorage",
}],
},
},
resource_group_name="rg1")
package main
import (
hdinsight "github.com/pulumi/pulumi-azure-native-sdk/hdinsight/v3"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
_, err := hdinsight.NewCluster(ctx, "cluster", &hdinsight.ClusterArgs{
ClusterName: pulumi.String("cluster1"),
Properties: &hdinsight.ClusterCreatePropertiesArgs{
ClusterDefinition: &hdinsight.ClusterDefinitionArgs{
Configurations: pulumi.Any(map[string]interface{}{
"gateway": map[string]interface{}{
"restAuthCredential.isEnabled": true,
"restAuthCredential.password": "**********",
"restAuthCredential.username": "admin",
},
}),
Kind: pulumi.String("hadoop"),
},
ClusterVersion: pulumi.String("3.6"),
ComputeProfile: &hdinsight.ComputeProfileArgs{
Roles: hdinsight.RoleArray{
&hdinsight.RoleArgs{
HardwareProfile: &hdinsight.HardwareProfileArgs{
VmSize: pulumi.String("standard_d3"),
},
Name: pulumi.String("headnode"),
OsProfile: &hdinsight.OsProfileArgs{
LinuxOperatingSystemProfile: &hdinsight.LinuxOperatingSystemProfileArgs{
Password: pulumi.String("**********"),
SshProfile: &hdinsight.SshProfileArgs{
PublicKeys: hdinsight.SshPublicKeyArray{
&hdinsight.SshPublicKeyArgs{
CertificateData: pulumi.String("**********"),
},
},
},
Username: pulumi.String("sshuser"),
},
},
TargetInstanceCount: pulumi.Int(2),
VirtualNetworkProfile: &hdinsight.VirtualNetworkProfileArgs{
Id: pulumi.String("/subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname"),
Subnet: pulumi.String("/subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname/subnets/vnetsubnet"),
},
},
&hdinsight.RoleArgs{
HardwareProfile: &hdinsight.HardwareProfileArgs{
VmSize: pulumi.String("standard_d3"),
},
Name: pulumi.String("workernode"),
OsProfile: &hdinsight.OsProfileArgs{
LinuxOperatingSystemProfile: &hdinsight.LinuxOperatingSystemProfileArgs{
Password: pulumi.String("**********"),
SshProfile: &hdinsight.SshProfileArgs{
PublicKeys: hdinsight.SshPublicKeyArray{
&hdinsight.SshPublicKeyArgs{
CertificateData: pulumi.String("**********"),
},
},
},
Username: pulumi.String("sshuser"),
},
},
TargetInstanceCount: pulumi.Int(2),
VirtualNetworkProfile: &hdinsight.VirtualNetworkProfileArgs{
Id: pulumi.String("/subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname"),
Subnet: pulumi.String("/subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname/subnets/vnetsubnet"),
},
},
},
},
NetworkProperties: &hdinsight.NetworkPropertiesArgs{
PrivateLink: pulumi.String(hdinsight.PrivateLinkEnabled),
PublicIpTag: &hdinsight.IpTagArgs{
IpTagType: pulumi.String("FirstPartyUsage"),
Tag: pulumi.String("/<TagName>"),
},
ResourceProviderConnection: pulumi.String(hdinsight.ResourceProviderConnectionOutbound),
},
OsType: pulumi.String(hdinsight.OSTypeLinux),
StorageProfile: &hdinsight.StorageProfileArgs{
Storageaccounts: hdinsight.StorageAccountArray{
&hdinsight.StorageAccountArgs{
Container: pulumi.String("containername"),
EnableSecureChannel: pulumi.Bool(true),
IsDefault: pulumi.Bool(true),
Key: pulumi.String("storage account key"),
Name: pulumi.String("mystorage"),
},
},
},
},
ResourceGroupName: pulumi.String("rg1"),
})
if err != nil {
return err
}
return nil
})
}
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using AzureNative = Pulumi.AzureNative;
return await Deployment.RunAsync(() =>
{
var cluster = new AzureNative.HDInsight.Cluster("cluster", new()
{
ClusterName = "cluster1",
Properties = new AzureNative.HDInsight.Inputs.ClusterCreatePropertiesArgs
{
ClusterDefinition = new AzureNative.HDInsight.Inputs.ClusterDefinitionArgs
{
Configurations = new Dictionary<string, object?>
{
["gateway"] = new Dictionary<string, object?>
{
["restAuthCredential.isEnabled"] = true,
["restAuthCredential.password"] = "**********",
["restAuthCredential.username"] = "admin",
},
},
Kind = "hadoop",
},
ClusterVersion = "3.6",
ComputeProfile = new AzureNative.HDInsight.Inputs.ComputeProfileArgs
{
Roles = new[]
{
new AzureNative.HDInsight.Inputs.RoleArgs
{
HardwareProfile = new AzureNative.HDInsight.Inputs.HardwareProfileArgs
{
VmSize = "standard_d3",
},
Name = "headnode",
OsProfile = new AzureNative.HDInsight.Inputs.OsProfileArgs
{
LinuxOperatingSystemProfile = new AzureNative.HDInsight.Inputs.LinuxOperatingSystemProfileArgs
{
Password = "**********",
SshProfile = new AzureNative.HDInsight.Inputs.SshProfileArgs
{
PublicKeys = new[]
{
new AzureNative.HDInsight.Inputs.SshPublicKeyArgs
{
CertificateData = "**********",
},
},
},
Username = "sshuser",
},
},
TargetInstanceCount = 2,
VirtualNetworkProfile = new AzureNative.HDInsight.Inputs.VirtualNetworkProfileArgs
{
Id = "/subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname",
Subnet = "/subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname/subnets/vnetsubnet",
},
},
new AzureNative.HDInsight.Inputs.RoleArgs
{
HardwareProfile = new AzureNative.HDInsight.Inputs.HardwareProfileArgs
{
VmSize = "standard_d3",
},
Name = "workernode",
OsProfile = new AzureNative.HDInsight.Inputs.OsProfileArgs
{
LinuxOperatingSystemProfile = new AzureNative.HDInsight.Inputs.LinuxOperatingSystemProfileArgs
{
Password = "**********",
SshProfile = new AzureNative.HDInsight.Inputs.SshProfileArgs
{
PublicKeys = new[]
{
new AzureNative.HDInsight.Inputs.SshPublicKeyArgs
{
CertificateData = "**********",
},
},
},
Username = "sshuser",
},
},
TargetInstanceCount = 2,
VirtualNetworkProfile = new AzureNative.HDInsight.Inputs.VirtualNetworkProfileArgs
{
Id = "/subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname",
Subnet = "/subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname/subnets/vnetsubnet",
},
},
},
},
NetworkProperties = new AzureNative.HDInsight.Inputs.NetworkPropertiesArgs
{
PrivateLink = AzureNative.HDInsight.PrivateLink.Enabled,
PublicIpTag = new AzureNative.HDInsight.Inputs.IpTagArgs
{
IpTagType = "FirstPartyUsage",
Tag = "/<TagName>",
},
ResourceProviderConnection = AzureNative.HDInsight.ResourceProviderConnection.Outbound,
},
OsType = AzureNative.HDInsight.OSType.Linux,
StorageProfile = new AzureNative.HDInsight.Inputs.StorageProfileArgs
{
Storageaccounts = new[]
{
new AzureNative.HDInsight.Inputs.StorageAccountArgs
{
Container = "containername",
EnableSecureChannel = true,
IsDefault = true,
Key = "storage account key",
Name = "mystorage",
},
},
},
},
ResourceGroupName = "rg1",
});
});
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.azurenative.hdinsight.Cluster;
import com.pulumi.azurenative.hdinsight.ClusterArgs;
import com.pulumi.azurenative.hdinsight.inputs.ClusterCreatePropertiesArgs;
import com.pulumi.azurenative.hdinsight.inputs.ClusterDefinitionArgs;
import com.pulumi.azurenative.hdinsight.inputs.ComputeProfileArgs;
import com.pulumi.azurenative.hdinsight.inputs.NetworkPropertiesArgs;
import com.pulumi.azurenative.hdinsight.inputs.IpTagArgs;
import com.pulumi.azurenative.hdinsight.inputs.StorageProfileArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var cluster = new Cluster("cluster", ClusterArgs.builder()
.clusterName("cluster1")
.properties(ClusterCreatePropertiesArgs.builder()
.clusterDefinition(ClusterDefinitionArgs.builder()
.configurations(Map.of("gateway", Map.ofEntries(
Map.entry("restAuthCredential.isEnabled", true),
Map.entry("restAuthCredential.password", "**********"),
Map.entry("restAuthCredential.username", "admin")
)))
.kind("hadoop")
.build())
.clusterVersion("3.6")
.computeProfile(ComputeProfileArgs.builder()
.roles(
RoleArgs.builder()
.hardwareProfile(HardwareProfileArgs.builder()
.vmSize("standard_d3")
.build())
.name("headnode")
.osProfile(OsProfileArgs.builder()
.linuxOperatingSystemProfile(LinuxOperatingSystemProfileArgs.builder()
.password("**********")
.sshProfile(SshProfileArgs.builder()
.publicKeys(SshPublicKeyArgs.builder()
.certificateData("**********")
.build())
.build())
.username("sshuser")
.build())
.build())
.targetInstanceCount(2)
.virtualNetworkProfile(VirtualNetworkProfileArgs.builder()
.id("/subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname")
.subnet("/subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname/subnets/vnetsubnet")
.build())
.build(),
RoleArgs.builder()
.hardwareProfile(HardwareProfileArgs.builder()
.vmSize("standard_d3")
.build())
.name("workernode")
.osProfile(OsProfileArgs.builder()
.linuxOperatingSystemProfile(LinuxOperatingSystemProfileArgs.builder()
.password("**********")
.sshProfile(SshProfileArgs.builder()
.publicKeys(SshPublicKeyArgs.builder()
.certificateData("**********")
.build())
.build())
.username("sshuser")
.build())
.build())
.targetInstanceCount(2)
.virtualNetworkProfile(VirtualNetworkProfileArgs.builder()
.id("/subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname")
.subnet("/subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname/subnets/vnetsubnet")
.build())
.build())
.build())
.networkProperties(NetworkPropertiesArgs.builder()
.privateLink("Enabled")
.publicIpTag(IpTagArgs.builder()
.ipTagType("FirstPartyUsage")
.tag("/<TagName>")
.build())
.resourceProviderConnection("Outbound")
.build())
.osType("Linux")
.storageProfile(StorageProfileArgs.builder()
.storageaccounts(StorageAccountArgs.builder()
.container("containername")
.enableSecureChannel(true)
.isDefault(true)
.key("storage account key")
.name("mystorage")
.build())
.build())
.build())
.resourceGroupName("rg1")
.build());
}
}
resources:
cluster:
type: azure-native:hdinsight:Cluster
properties:
clusterName: cluster1
properties:
clusterDefinition:
configurations:
gateway:
restAuthCredential.isEnabled: true
restAuthCredential.password: '**********'
restAuthCredential.username: admin
kind: hadoop
clusterVersion: '3.6'
computeProfile:
roles:
- hardwareProfile:
vmSize: standard_d3
name: headnode
osProfile:
linuxOperatingSystemProfile:
password: '**********'
sshProfile:
publicKeys:
- certificateData: '**********'
username: sshuser
targetInstanceCount: 2
virtualNetworkProfile:
id: /subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname
subnet: /subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname/subnets/vnetsubnet
- hardwareProfile:
vmSize: standard_d3
name: workernode
osProfile:
linuxOperatingSystemProfile:
password: '**********'
sshProfile:
publicKeys:
- certificateData: '**********'
username: sshuser
targetInstanceCount: 2
virtualNetworkProfile:
id: /subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname
subnet: /subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnetname/subnets/vnetsubnet
networkProperties:
privateLink: Enabled
publicIpTag:
ipTagType: FirstPartyUsage
tag: /<TagName>
resourceProviderConnection: Outbound
osType: Linux
storageProfile:
storageaccounts:
- container: containername
enableSecureChannel: true
isDefault: true
key: storage account key
name: mystorage
resourceGroupName: rg1
The networkProperties block controls network behavior. Setting privateLink to Enabled creates private endpoints for cluster access. The resourceProviderConnection property determines whether the cluster initiates outbound connections to Azure services (Outbound) or expects inbound connections (Inbound). The publicIpTag property adds metadata to public IPs for tracking and routing purposes.
Beyond these examples
These snippets focus on specific cluster-level features: autoscaling and node configuration, storage integration, security (SSH keys, Active Directory, encryption), and networking (VNets, private link, outbound connectivity). They’re intentionally minimal rather than full big data deployments.
The examples may reference pre-existing infrastructure such as Azure Storage accounts (Blob or ADLS Gen 2), virtual networks and subnets, Active Directory domains and LDAP servers for secure clusters, and SSH public keys. They focus on configuring the cluster rather than provisioning the surrounding infrastructure.
To keep things focused, common cluster patterns are omitted, including:
- Cluster types beyond Hadoop, Spark, and Kafka (HBase, Storm, Interactive Query)
- Custom script actions for cluster customization
- Managed identities and service principals
- Disk encryption with customer-managed keys
- Monitoring and logging configuration
- Kafka-specific features (broker configuration, topic management)
These omissions are intentional: the goal is to illustrate how each cluster feature is wired, not provide drop-in analytics platforms. See the HDInsight Cluster resource reference for all available configuration options.
Let's deploy Azure HDInsight Clusters
Get started with Pulumi Cloud, then follow our quick setup guide to deploy this infrastructure.
Try Pulumi Cloud for FREEFrequently Asked Questions
Storage Configuration
What's the difference between blob storage and Azure Data Lake Storage Gen 2?
Blob storage uses
.blob.core.windows.net endpoints with the container property, while ADLS Gen2 uses .dfs.core.windows.net endpoints with the fileSystem property. Both require enableSecureChannel: true and a storage key.Scaling & Performance
How do I configure schedule-based autoscaling for my cluster?
Configure autoscaleConfiguration.recurrence with schedule entries that specify:
- Days (e.g., Monday-Friday, Saturday-Sunday)
- Time (e.g., “09:00”, “18:00”)
- Capacity (
minInstanceCountandmaxInstanceCount)
You can define multiple schedules for different times and days, plus set a timeZone.
What is compute isolation and how do I enable it?
Compute isolation provides dedicated hardware for your cluster. Enable it by setting
computeIsolationProperties.enableComputeIsolation: true in cluster properties.Security & Authentication
Can I use SSH public keys instead of passwords?
Yes, configure
linuxOperatingSystemProfile.sshProfile.publicKeys with certificateData instead of using the password field. You can provide multiple public keys.How do I create a secure cluster with Active Directory integration?
Configure securityProfile with:
directoryType: ActiveDirectorydomain,domainUsername,domainUserPasswordldapsUrls(e.g., “ldaps://10.10.0.4:636”)organizationalUnitDN(e.g., “OU=Hadoop,DC=hdinsight,DC=test”)clusterUsersGroupDNsfor user access
Also set tier: Premium for secure clusters.
How do I set the minimum TLS version for my cluster?
Set
minSupportedTlsVersion to “1.2” (or your desired version) in the cluster properties.What encryption options are available for HDInsight clusters?
Two encryption options:
- Encryption at host: Set
diskEncryptionProperties.encryptionAtHost: true(requires Standard_DS14_v2 or compatible VM sizes) - Encryption in transit: Set
encryptionInTransitProperties.isEncryptionInTransitEnabled: true
Networking & Connectivity
How do I enable private link for my cluster?
Configure networkProperties with:
privateLink: EnabledresourceProviderConnection: Outbound- Optionally set
publicIpTagwithipTagTypeandtag
Clusters must be deployed in a virtual network with virtualNetworkProfile configured on roles.
Can I deploy clusters across availability zones?
Yes, specify the
zones array at the cluster level (e.g., ["1"]). Availability zones provide high availability and fault tolerance.Cluster Types & Configuration
How do I enable Kafka Rest Proxy?
Add a
kafkamanagementnode role to your compute profile and configure kafkaRestProperties with clientGroupInfo containing groupId and groupName for your Azure AD security group.What node roles are required for different cluster types?
Required roles vary by cluster type:
- Hadoop/Spark:
headnode,workernode,zookeepernode - Kafka: Same as Hadoop, plus optional
kafkamanagementnodefor Rest Proxy
Each role requires hardwareProfile (VM size), osProfile (credentials), and targetInstanceCount.