The azure-native:hdinsight:ClusterPool resource, part of the Pulumi Azure Native provider, provisions the shared infrastructure foundation for HDInsight on AKS clusters: compute resources, availability zones, and network configuration. This guide focuses on three capabilities: compute profile and availability zone placement, private API server access, and user-defined routing for egress traffic.
Cluster pools require an Azure resource group and location. Network-isolated configurations reference existing virtual networks and subnets. The examples are intentionally small. Combine them with your own networking infrastructure and HDInsight cluster definitions.
Create a cluster pool with compute and availability zones
Most HDInsight on AKS deployments begin by provisioning a cluster pool that defines the compute infrastructure and availability zone placement for workloads.
import * as pulumi from "@pulumi/pulumi";
import * as azure_native from "@pulumi/azure-native";
const clusterPool = new azure_native.hdinsight.ClusterPool("clusterPool", {
clusterPoolName: "clusterpool1",
clusterPoolProfile: {
clusterPoolVersion: "1.2",
},
computeProfile: {
availabilityZones: [
"1",
"2",
"3",
],
vmSize: "Standard_D3_v2",
},
location: "West US 2",
resourceGroupName: "hiloResourcegroup",
});
import pulumi
import pulumi_azure_native as azure_native
cluster_pool = azure_native.hdinsight.ClusterPool("clusterPool",
cluster_pool_name="clusterpool1",
cluster_pool_profile={
"cluster_pool_version": "1.2",
},
compute_profile={
"availability_zones": [
"1",
"2",
"3",
],
"vm_size": "Standard_D3_v2",
},
location="West US 2",
resource_group_name="hiloResourcegroup")
package main
import (
hdinsight "github.com/pulumi/pulumi-azure-native-sdk/hdinsight/v3"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
_, err := hdinsight.NewClusterPool(ctx, "clusterPool", &hdinsight.ClusterPoolArgs{
ClusterPoolName: pulumi.String("clusterpool1"),
ClusterPoolProfile: &hdinsight.ClusterPoolResourcePropertiesClusterPoolProfileArgs{
ClusterPoolVersion: pulumi.String("1.2"),
},
ComputeProfile: &hdinsight.ClusterPoolResourcePropertiesComputeProfileArgs{
AvailabilityZones: pulumi.StringArray{
pulumi.String("1"),
pulumi.String("2"),
pulumi.String("3"),
},
VmSize: pulumi.String("Standard_D3_v2"),
},
Location: pulumi.String("West US 2"),
ResourceGroupName: pulumi.String("hiloResourcegroup"),
})
if err != nil {
return err
}
return nil
})
}
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using AzureNative = Pulumi.AzureNative;
return await Deployment.RunAsync(() =>
{
var clusterPool = new AzureNative.HDInsight.ClusterPool("clusterPool", new()
{
ClusterPoolName = "clusterpool1",
ClusterPoolProfile = new AzureNative.HDInsight.Inputs.ClusterPoolResourcePropertiesClusterPoolProfileArgs
{
ClusterPoolVersion = "1.2",
},
ComputeProfile = new AzureNative.HDInsight.Inputs.ClusterPoolResourcePropertiesComputeProfileArgs
{
AvailabilityZones = new[]
{
"1",
"2",
"3",
},
VmSize = "Standard_D3_v2",
},
Location = "West US 2",
ResourceGroupName = "hiloResourcegroup",
});
});
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.azurenative.hdinsight.ClusterPool;
import com.pulumi.azurenative.hdinsight.ClusterPoolArgs;
import com.pulumi.azurenative.hdinsight.inputs.ClusterPoolResourcePropertiesClusterPoolProfileArgs;
import com.pulumi.azurenative.hdinsight.inputs.ClusterPoolResourcePropertiesComputeProfileArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var clusterPool = new ClusterPool("clusterPool", ClusterPoolArgs.builder()
.clusterPoolName("clusterpool1")
.clusterPoolProfile(ClusterPoolResourcePropertiesClusterPoolProfileArgs.builder()
.clusterPoolVersion("1.2")
.build())
.computeProfile(ClusterPoolResourcePropertiesComputeProfileArgs.builder()
.availabilityZones(
"1",
"2",
"3")
.vmSize("Standard_D3_v2")
.build())
.location("West US 2")
.resourceGroupName("hiloResourcegroup")
.build());
}
}
resources:
clusterPool:
type: azure-native:hdinsight:ClusterPool
properties:
clusterPoolName: clusterpool1
clusterPoolProfile:
clusterPoolVersion: '1.2'
computeProfile:
availabilityZones:
- '1'
- '2'
- '3'
vmSize: Standard_D3_v2
location: West US 2
resourceGroupName: hiloResourcegroup
The clusterPoolProfile sets the HDInsight version (clusterPoolVersion), while computeProfile defines the VM size and availability zones where nodes will run. The cluster pool creates an underlying AKS cluster that hosts HDInsight workloads. Multiple HDInsight clusters can share a single cluster pool, reducing infrastructure overhead.
Deploy with private API server access
Organizations with strict network security requirements often need to restrict Kubernetes API server access to private networks only.
import * as pulumi from "@pulumi/pulumi";
import * as azure_native from "@pulumi/azure-native";
const clusterPool = new azure_native.hdinsight.ClusterPool("clusterPool", {
clusterPoolName: "clusterpool1",
clusterPoolProfile: {
clusterPoolVersion: "1.2",
},
computeProfile: {
availabilityZones: [
"1",
"2",
"3",
],
vmSize: "Standard_D3_v2",
},
location: "West US 2",
networkProfile: {
enablePrivateApiServer: true,
subnetId: "/subscriptions/subid/resourceGroups/hiloResourcegroup/providers/Microsoft.Network/virtualNetworks/vnet1/subnets/subnet1",
},
resourceGroupName: "hiloResourcegroup",
});
import pulumi
import pulumi_azure_native as azure_native
cluster_pool = azure_native.hdinsight.ClusterPool("clusterPool",
cluster_pool_name="clusterpool1",
cluster_pool_profile={
"cluster_pool_version": "1.2",
},
compute_profile={
"availability_zones": [
"1",
"2",
"3",
],
"vm_size": "Standard_D3_v2",
},
location="West US 2",
network_profile={
"enable_private_api_server": True,
"subnet_id": "/subscriptions/subid/resourceGroups/hiloResourcegroup/providers/Microsoft.Network/virtualNetworks/vnet1/subnets/subnet1",
},
resource_group_name="hiloResourcegroup")
package main
import (
hdinsight "github.com/pulumi/pulumi-azure-native-sdk/hdinsight/v3"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
_, err := hdinsight.NewClusterPool(ctx, "clusterPool", &hdinsight.ClusterPoolArgs{
ClusterPoolName: pulumi.String("clusterpool1"),
ClusterPoolProfile: &hdinsight.ClusterPoolResourcePropertiesClusterPoolProfileArgs{
ClusterPoolVersion: pulumi.String("1.2"),
},
ComputeProfile: &hdinsight.ClusterPoolResourcePropertiesComputeProfileArgs{
AvailabilityZones: pulumi.StringArray{
pulumi.String("1"),
pulumi.String("2"),
pulumi.String("3"),
},
VmSize: pulumi.String("Standard_D3_v2"),
},
Location: pulumi.String("West US 2"),
NetworkProfile: &hdinsight.ClusterPoolResourcePropertiesNetworkProfileArgs{
EnablePrivateApiServer: pulumi.Bool(true),
SubnetId: pulumi.String("/subscriptions/subid/resourceGroups/hiloResourcegroup/providers/Microsoft.Network/virtualNetworks/vnet1/subnets/subnet1"),
},
ResourceGroupName: pulumi.String("hiloResourcegroup"),
})
if err != nil {
return err
}
return nil
})
}
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using AzureNative = Pulumi.AzureNative;
return await Deployment.RunAsync(() =>
{
var clusterPool = new AzureNative.HDInsight.ClusterPool("clusterPool", new()
{
ClusterPoolName = "clusterpool1",
ClusterPoolProfile = new AzureNative.HDInsight.Inputs.ClusterPoolResourcePropertiesClusterPoolProfileArgs
{
ClusterPoolVersion = "1.2",
},
ComputeProfile = new AzureNative.HDInsight.Inputs.ClusterPoolResourcePropertiesComputeProfileArgs
{
AvailabilityZones = new[]
{
"1",
"2",
"3",
},
VmSize = "Standard_D3_v2",
},
Location = "West US 2",
NetworkProfile = new AzureNative.HDInsight.Inputs.ClusterPoolResourcePropertiesNetworkProfileArgs
{
EnablePrivateApiServer = true,
SubnetId = "/subscriptions/subid/resourceGroups/hiloResourcegroup/providers/Microsoft.Network/virtualNetworks/vnet1/subnets/subnet1",
},
ResourceGroupName = "hiloResourcegroup",
});
});
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.azurenative.hdinsight.ClusterPool;
import com.pulumi.azurenative.hdinsight.ClusterPoolArgs;
import com.pulumi.azurenative.hdinsight.inputs.ClusterPoolResourcePropertiesClusterPoolProfileArgs;
import com.pulumi.azurenative.hdinsight.inputs.ClusterPoolResourcePropertiesComputeProfileArgs;
import com.pulumi.azurenative.hdinsight.inputs.ClusterPoolResourcePropertiesNetworkProfileArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var clusterPool = new ClusterPool("clusterPool", ClusterPoolArgs.builder()
.clusterPoolName("clusterpool1")
.clusterPoolProfile(ClusterPoolResourcePropertiesClusterPoolProfileArgs.builder()
.clusterPoolVersion("1.2")
.build())
.computeProfile(ClusterPoolResourcePropertiesComputeProfileArgs.builder()
.availabilityZones(
"1",
"2",
"3")
.vmSize("Standard_D3_v2")
.build())
.location("West US 2")
.networkProfile(ClusterPoolResourcePropertiesNetworkProfileArgs.builder()
.enablePrivateApiServer(true)
.subnetId("/subscriptions/subid/resourceGroups/hiloResourcegroup/providers/Microsoft.Network/virtualNetworks/vnet1/subnets/subnet1")
.build())
.resourceGroupName("hiloResourcegroup")
.build());
}
}
resources:
clusterPool:
type: azure-native:hdinsight:ClusterPool
properties:
clusterPoolName: clusterpool1
clusterPoolProfile:
clusterPoolVersion: '1.2'
computeProfile:
availabilityZones:
- '1'
- '2'
- '3'
vmSize: Standard_D3_v2
location: West US 2
networkProfile:
enablePrivateApiServer: true
subnetId: /subscriptions/subid/resourceGroups/hiloResourcegroup/providers/Microsoft.Network/virtualNetworks/vnet1/subnets/subnet1
resourceGroupName: hiloResourcegroup
The networkProfile enables private API server access by setting enablePrivateApiServer to true and specifying a subnetId. This places the Kubernetes control plane on a private endpoint within your virtual network, blocking public internet access. The cluster pool still requires the same compute profile configuration from the basic example.
Control outbound traffic with user-defined routing
Environments that route all egress traffic through firewalls or network virtual appliances require user-defined routing instead of default Azure load balancer egress.
import * as pulumi from "@pulumi/pulumi";
import * as azure_native from "@pulumi/azure-native";
const clusterPool = new azure_native.hdinsight.ClusterPool("clusterPool", {
clusterPoolName: "clusterpool1",
clusterPoolProfile: {
clusterPoolVersion: "1.2",
},
computeProfile: {
availabilityZones: [
"1",
"2",
"3",
],
vmSize: "Standard_D3_v2",
},
location: "West US 2",
networkProfile: {
outboundType: azure_native.hdinsight.OutboundType.UserDefinedRouting,
subnetId: "/subscriptions/subid/resourceGroups/hiloResourcegroup/providers/Microsoft.Network/virtualNetworks/vnet1/subnets/subnet1",
},
resourceGroupName: "hiloResourcegroup",
});
import pulumi
import pulumi_azure_native as azure_native
cluster_pool = azure_native.hdinsight.ClusterPool("clusterPool",
cluster_pool_name="clusterpool1",
cluster_pool_profile={
"cluster_pool_version": "1.2",
},
compute_profile={
"availability_zones": [
"1",
"2",
"3",
],
"vm_size": "Standard_D3_v2",
},
location="West US 2",
network_profile={
"outbound_type": azure_native.hdinsight.OutboundType.USER_DEFINED_ROUTING,
"subnet_id": "/subscriptions/subid/resourceGroups/hiloResourcegroup/providers/Microsoft.Network/virtualNetworks/vnet1/subnets/subnet1",
},
resource_group_name="hiloResourcegroup")
package main
import (
hdinsight "github.com/pulumi/pulumi-azure-native-sdk/hdinsight/v3"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
_, err := hdinsight.NewClusterPool(ctx, "clusterPool", &hdinsight.ClusterPoolArgs{
ClusterPoolName: pulumi.String("clusterpool1"),
ClusterPoolProfile: &hdinsight.ClusterPoolResourcePropertiesClusterPoolProfileArgs{
ClusterPoolVersion: pulumi.String("1.2"),
},
ComputeProfile: &hdinsight.ClusterPoolResourcePropertiesComputeProfileArgs{
AvailabilityZones: pulumi.StringArray{
pulumi.String("1"),
pulumi.String("2"),
pulumi.String("3"),
},
VmSize: pulumi.String("Standard_D3_v2"),
},
Location: pulumi.String("West US 2"),
NetworkProfile: &hdinsight.ClusterPoolResourcePropertiesNetworkProfileArgs{
OutboundType: pulumi.String(hdinsight.OutboundTypeUserDefinedRouting),
SubnetId: pulumi.String("/subscriptions/subid/resourceGroups/hiloResourcegroup/providers/Microsoft.Network/virtualNetworks/vnet1/subnets/subnet1"),
},
ResourceGroupName: pulumi.String("hiloResourcegroup"),
})
if err != nil {
return err
}
return nil
})
}
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using AzureNative = Pulumi.AzureNative;
return await Deployment.RunAsync(() =>
{
var clusterPool = new AzureNative.HDInsight.ClusterPool("clusterPool", new()
{
ClusterPoolName = "clusterpool1",
ClusterPoolProfile = new AzureNative.HDInsight.Inputs.ClusterPoolResourcePropertiesClusterPoolProfileArgs
{
ClusterPoolVersion = "1.2",
},
ComputeProfile = new AzureNative.HDInsight.Inputs.ClusterPoolResourcePropertiesComputeProfileArgs
{
AvailabilityZones = new[]
{
"1",
"2",
"3",
},
VmSize = "Standard_D3_v2",
},
Location = "West US 2",
NetworkProfile = new AzureNative.HDInsight.Inputs.ClusterPoolResourcePropertiesNetworkProfileArgs
{
OutboundType = AzureNative.HDInsight.OutboundType.UserDefinedRouting,
SubnetId = "/subscriptions/subid/resourceGroups/hiloResourcegroup/providers/Microsoft.Network/virtualNetworks/vnet1/subnets/subnet1",
},
ResourceGroupName = "hiloResourcegroup",
});
});
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.azurenative.hdinsight.ClusterPool;
import com.pulumi.azurenative.hdinsight.ClusterPoolArgs;
import com.pulumi.azurenative.hdinsight.inputs.ClusterPoolResourcePropertiesClusterPoolProfileArgs;
import com.pulumi.azurenative.hdinsight.inputs.ClusterPoolResourcePropertiesComputeProfileArgs;
import com.pulumi.azurenative.hdinsight.inputs.ClusterPoolResourcePropertiesNetworkProfileArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var clusterPool = new ClusterPool("clusterPool", ClusterPoolArgs.builder()
.clusterPoolName("clusterpool1")
.clusterPoolProfile(ClusterPoolResourcePropertiesClusterPoolProfileArgs.builder()
.clusterPoolVersion("1.2")
.build())
.computeProfile(ClusterPoolResourcePropertiesComputeProfileArgs.builder()
.availabilityZones(
"1",
"2",
"3")
.vmSize("Standard_D3_v2")
.build())
.location("West US 2")
.networkProfile(ClusterPoolResourcePropertiesNetworkProfileArgs.builder()
.outboundType("userDefinedRouting")
.subnetId("/subscriptions/subid/resourceGroups/hiloResourcegroup/providers/Microsoft.Network/virtualNetworks/vnet1/subnets/subnet1")
.build())
.resourceGroupName("hiloResourcegroup")
.build());
}
}
resources:
clusterPool:
type: azure-native:hdinsight:ClusterPool
properties:
clusterPoolName: clusterpool1
clusterPoolProfile:
clusterPoolVersion: '1.2'
computeProfile:
availabilityZones:
- '1'
- '2'
- '3'
vmSize: Standard_D3_v2
location: West US 2
networkProfile:
outboundType: userDefinedRouting
subnetId: /subscriptions/subid/resourceGroups/hiloResourcegroup/providers/Microsoft.Network/virtualNetworks/vnet1/subnets/subnet1
resourceGroupName: hiloResourcegroup
The outboundType property set to UserDefinedRouting tells the cluster pool to use routes defined in the subnet’s route table rather than Azure’s default load balancer. Your subnet must have a route table configured to direct traffic to your firewall or network virtual appliance. This provides an alternative to private API server access for controlling network traffic.
Beyond these examples
These snippets focus on specific cluster pool features: compute profile and availability zone placement, private API server access, and user-defined routing for egress traffic. They’re intentionally minimal rather than full HDInsight deployments.
The examples may reference pre-existing infrastructure such as Azure resource groups, virtual networks and subnets (for network-isolated configurations), and route tables (for user-defined routing). They focus on configuring the cluster pool rather than provisioning the surrounding network infrastructure.
To keep things focused, common cluster pool patterns are omitted, including:
- Log Analytics integration (logAnalyticsProfile)
- Custom managed resource group naming (managedResourceGroupName)
- Tags for resource organization
- AKS cluster profile customization
These omissions are intentional: the goal is to illustrate how each cluster pool feature is wired, not provide drop-in HDInsight modules. See the ClusterPool resource reference for all available configuration options.
Let's create Azure HDInsight Cluster Pools
Get started with Pulumi Cloud, then follow our quick setup guide to deploy this infrastructure.
Try Pulumi Cloud for FREEFrequently Asked Questions
Networking & Connectivity
How do I enable private API server access for my cluster pool?
Set
networkProfile.enablePrivateApiServer to true and provide a subnetId pointing to your virtual network subnet.How do I configure user-defined routing for outbound traffic?
Set
networkProfile.outboundType to userDefinedRouting and specify a subnetId. This allows you to control outbound connectivity through custom routes.Do I need a subnet for all cluster pool configurations?
No, a subnet is only required when using
networkProfile features like private API server or user-defined routing. Basic cluster pools don’t require a subnetId.Can I use both private API server and user-defined routing together?
Yes, you can configure both
enablePrivateApiServer and outboundType in the networkProfile with the same subnetId.Resource Configuration
What are the required properties for creating a cluster pool?
You must specify
computeProfile (with vmSize and optionally availabilityZones) and location. The clusterPoolName and resourceGroupName are also required.How do I configure availability zones for high availability?
Specify
availabilityZones in the computeProfile as an array of zone numbers (e.g., ["1", "2", "3"]).What cluster pool versions are available?
The examples show version
1.2 in clusterPoolProfile.clusterPoolVersion. Check Azure documentation for the latest supported versions.Resource Management
What properties can't I change after creating a cluster pool?
The
location and clusterPoolName properties are immutable and require recreating the resource to change.What's the naming pattern for the AKS managed resource group?
Azure automatically generates it using the pattern
MC_{managedResourceGroupName}_{clusterPoolName}_{region} based on your managedResourceGroupName input.Can I specify a custom name for the AKS managed resource group?
No, the
aksManagedResourceGroupName is computed automatically. You can only influence it by setting managedResourceGroupName, which is used in the generation pattern.API Versions & Compatibility
Which API version does the ClusterPool resource use?
It uses Azure REST API version
2024-05-01-preview by default. Version 2.x of the Azure Native provider used 2023-06-01-preview.How do I use a different API version for ClusterPool?
Generate a local SDK package using the CLI command
pulumi package add azure-native hdinsight [ApiVersion]. Available versions include 2023-06-01-preview and 2023-11-01-preview.