The azure-native:kusto:CosmosDbDataConnection resource, part of the Pulumi Azure Native provider, defines a data connection that continuously ingests documents from Cosmos DB into Azure Data Explorer tables. This guide focuses on three capabilities: connecting Cosmos DB containers to Kusto tables, configuring managed identity authentication, and setting retrieval windows and mapping rules.
Data connections bridge Cosmos DB and Azure Data Explorer. They require existing Cosmos DB accounts, Kusto clusters and databases, target tables, managed identities with permissions, and mapping rules. The examples are intentionally small. Combine them with your own infrastructure and IAM configuration.
Stream Cosmos DB changes into Azure Data Explorer
Analytics pipelines often ingest Cosmos DB documents into Azure Data Explorer for querying and analysis.
import * as pulumi from "@pulumi/pulumi";
import * as azure_native from "@pulumi/azure-native";
const cosmosDbDataConnection = new azure_native.kusto.CosmosDbDataConnection("cosmosDbDataConnection", {
clusterName: "kustoCluster",
cosmosDbAccountResourceId: "/subscriptions/12345678-1234-1234-1234-123456789098/resourceGroups/kustorptest/providers/Microsoft.DocumentDb/databaseAccounts/cosmosDbAccountTest1",
cosmosDbContainer: "cosmosDbContainerTest",
cosmosDbDatabase: "cosmosDbDatabaseTest",
dataConnectionName: "dataConnectionTest",
databaseName: "KustoDatabase1",
kind: "CosmosDb",
location: "westus",
managedIdentityResourceId: "/subscriptions/12345678-1234-1234-1234-123456789098/resourceGroups/kustorptest/providers/Microsoft.ManagedIdentity/userAssignedIdentities/managedidentityTest1",
mappingRuleName: "TestMapping",
resourceGroupName: "kustorptest",
retrievalStartDate: "2022-07-29T12:00:00.6554616Z",
tableName: "TestTable",
});
import pulumi
import pulumi_azure_native as azure_native
cosmos_db_data_connection = azure_native.kusto.CosmosDbDataConnection("cosmosDbDataConnection",
cluster_name="kustoCluster",
cosmos_db_account_resource_id="/subscriptions/12345678-1234-1234-1234-123456789098/resourceGroups/kustorptest/providers/Microsoft.DocumentDb/databaseAccounts/cosmosDbAccountTest1",
cosmos_db_container="cosmosDbContainerTest",
cosmos_db_database="cosmosDbDatabaseTest",
data_connection_name="dataConnectionTest",
database_name="KustoDatabase1",
kind="CosmosDb",
location="westus",
managed_identity_resource_id="/subscriptions/12345678-1234-1234-1234-123456789098/resourceGroups/kustorptest/providers/Microsoft.ManagedIdentity/userAssignedIdentities/managedidentityTest1",
mapping_rule_name="TestMapping",
resource_group_name="kustorptest",
retrieval_start_date="2022-07-29T12:00:00.6554616Z",
table_name="TestTable")
package main
import (
kusto "github.com/pulumi/pulumi-azure-native-sdk/kusto/v3"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
_, err := kusto.NewCosmosDbDataConnection(ctx, "cosmosDbDataConnection", &kusto.CosmosDbDataConnectionArgs{
ClusterName: pulumi.String("kustoCluster"),
CosmosDbAccountResourceId: pulumi.String("/subscriptions/12345678-1234-1234-1234-123456789098/resourceGroups/kustorptest/providers/Microsoft.DocumentDb/databaseAccounts/cosmosDbAccountTest1"),
CosmosDbContainer: pulumi.String("cosmosDbContainerTest"),
CosmosDbDatabase: pulumi.String("cosmosDbDatabaseTest"),
DataConnectionName: pulumi.String("dataConnectionTest"),
DatabaseName: pulumi.String("KustoDatabase1"),
Kind: pulumi.String("CosmosDb"),
Location: pulumi.String("westus"),
ManagedIdentityResourceId: pulumi.String("/subscriptions/12345678-1234-1234-1234-123456789098/resourceGroups/kustorptest/providers/Microsoft.ManagedIdentity/userAssignedIdentities/managedidentityTest1"),
MappingRuleName: pulumi.String("TestMapping"),
ResourceGroupName: pulumi.String("kustorptest"),
RetrievalStartDate: pulumi.String("2022-07-29T12:00:00.6554616Z"),
TableName: pulumi.String("TestTable"),
})
if err != nil {
return err
}
return nil
})
}
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using AzureNative = Pulumi.AzureNative;
return await Deployment.RunAsync(() =>
{
var cosmosDbDataConnection = new AzureNative.Kusto.CosmosDbDataConnection("cosmosDbDataConnection", new()
{
ClusterName = "kustoCluster",
CosmosDbAccountResourceId = "/subscriptions/12345678-1234-1234-1234-123456789098/resourceGroups/kustorptest/providers/Microsoft.DocumentDb/databaseAccounts/cosmosDbAccountTest1",
CosmosDbContainer = "cosmosDbContainerTest",
CosmosDbDatabase = "cosmosDbDatabaseTest",
DataConnectionName = "dataConnectionTest",
DatabaseName = "KustoDatabase1",
Kind = "CosmosDb",
Location = "westus",
ManagedIdentityResourceId = "/subscriptions/12345678-1234-1234-1234-123456789098/resourceGroups/kustorptest/providers/Microsoft.ManagedIdentity/userAssignedIdentities/managedidentityTest1",
MappingRuleName = "TestMapping",
ResourceGroupName = "kustorptest",
RetrievalStartDate = "2022-07-29T12:00:00.6554616Z",
TableName = "TestTable",
});
});
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.azurenative.kusto.CosmosDbDataConnection;
import com.pulumi.azurenative.kusto.CosmosDbDataConnectionArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var cosmosDbDataConnection = new CosmosDbDataConnection("cosmosDbDataConnection", CosmosDbDataConnectionArgs.builder()
.clusterName("kustoCluster")
.cosmosDbAccountResourceId("/subscriptions/12345678-1234-1234-1234-123456789098/resourceGroups/kustorptest/providers/Microsoft.DocumentDb/databaseAccounts/cosmosDbAccountTest1")
.cosmosDbContainer("cosmosDbContainerTest")
.cosmosDbDatabase("cosmosDbDatabaseTest")
.dataConnectionName("dataConnectionTest")
.databaseName("KustoDatabase1")
.kind("CosmosDb")
.location("westus")
.managedIdentityResourceId("/subscriptions/12345678-1234-1234-1234-123456789098/resourceGroups/kustorptest/providers/Microsoft.ManagedIdentity/userAssignedIdentities/managedidentityTest1")
.mappingRuleName("TestMapping")
.resourceGroupName("kustorptest")
.retrievalStartDate("2022-07-29T12:00:00.6554616Z")
.tableName("TestTable")
.build());
}
}
resources:
cosmosDbDataConnection:
type: azure-native:kusto:CosmosDbDataConnection
properties:
clusterName: kustoCluster
cosmosDbAccountResourceId: /subscriptions/12345678-1234-1234-1234-123456789098/resourceGroups/kustorptest/providers/Microsoft.DocumentDb/databaseAccounts/cosmosDbAccountTest1
cosmosDbContainer: cosmosDbContainerTest
cosmosDbDatabase: cosmosDbDatabaseTest
dataConnectionName: dataConnectionTest
databaseName: KustoDatabase1
kind: CosmosDb
location: westus
managedIdentityResourceId: /subscriptions/12345678-1234-1234-1234-123456789098/resourceGroups/kustorptest/providers/Microsoft.ManagedIdentity/userAssignedIdentities/managedidentityTest1
mappingRuleName: TestMapping
resourceGroupName: kustorptest
retrievalStartDate: 2022-07-29T12:00:00.6554616Z
tableName: TestTable
The data connection continuously retrieves documents from the specified Cosmos DB container and writes them to the Kusto table. The cosmosDbAccountResourceId, cosmosDbDatabase, and cosmosDbContainer properties identify the source; tableName specifies the destination. The managedIdentityResourceId provides authentication credentials for accessing Cosmos DB. The retrievalStartDate property sets a time boundary, ingesting only documents created or updated after that timestamp. The mappingRuleName references a transformation rule that maps Cosmos DB document structure to the Kusto table schema.
Create a minimal data connection placeholder
Some deployments establish the data connection resource first, then update it with full configuration later.
import * as pulumi from "@pulumi/pulumi";
import * as azure_native from "@pulumi/azure-native";
const cosmosDbDataConnection = new azure_native.kusto.CosmosDbDataConnection("cosmosDbDataConnection", {
clusterName: "kustoCluster",
dataConnectionName: "dataConnectionTest",
databaseName: "KustoDatabase8",
resourceGroupName: "kustorptest",
});
import pulumi
import pulumi_azure_native as azure_native
cosmos_db_data_connection = azure_native.kusto.CosmosDbDataConnection("cosmosDbDataConnection",
cluster_name="kustoCluster",
data_connection_name="dataConnectionTest",
database_name="KustoDatabase8",
resource_group_name="kustorptest")
package main
import (
kusto "github.com/pulumi/pulumi-azure-native-sdk/kusto/v3"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
_, err := kusto.NewCosmosDbDataConnection(ctx, "cosmosDbDataConnection", &kusto.CosmosDbDataConnectionArgs{
ClusterName: pulumi.String("kustoCluster"),
DataConnectionName: pulumi.String("dataConnectionTest"),
DatabaseName: pulumi.String("KustoDatabase8"),
ResourceGroupName: pulumi.String("kustorptest"),
})
if err != nil {
return err
}
return nil
})
}
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using AzureNative = Pulumi.AzureNative;
return await Deployment.RunAsync(() =>
{
var cosmosDbDataConnection = new AzureNative.Kusto.CosmosDbDataConnection("cosmosDbDataConnection", new()
{
ClusterName = "kustoCluster",
DataConnectionName = "dataConnectionTest",
DatabaseName = "KustoDatabase8",
ResourceGroupName = "kustorptest",
});
});
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.azurenative.kusto.CosmosDbDataConnection;
import com.pulumi.azurenative.kusto.CosmosDbDataConnectionArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var cosmosDbDataConnection = new CosmosDbDataConnection("cosmosDbDataConnection", CosmosDbDataConnectionArgs.builder()
.clusterName("kustoCluster")
.dataConnectionName("dataConnectionTest")
.databaseName("KustoDatabase8")
.resourceGroupName("kustorptest")
.build());
}
}
resources:
cosmosDbDataConnection:
type: azure-native:kusto:CosmosDbDataConnection
properties:
clusterName: kustoCluster
dataConnectionName: dataConnectionTest
databaseName: KustoDatabase8
resourceGroupName: kustorptest
This configuration creates the data connection resource with only the required identifiers: clusterName, databaseName, dataConnectionName, and resourceGroupName. Without Cosmos DB source properties (cosmosDbAccountResourceId, cosmosDbContainer, cosmosDbDatabase), managed identity configuration, or table mapping, this connection won’t ingest data. You’d typically update it later with the full configuration shown in the first example.
Beyond these examples
These snippets focus on specific data connection features: Cosmos DB to Kusto ingestion, managed identity authentication, and mapping rules and retrieval windows. They’re intentionally minimal rather than full data pipeline deployments.
The examples rely on pre-existing infrastructure such as Cosmos DB accounts, databases, and containers; Azure Data Explorer clusters and databases; target tables with schemas; managed identities with appropriate permissions; and mapping rules for data transformation. They focus on configuring the data connection rather than provisioning the surrounding infrastructure.
To keep things focused, common data connection patterns are omitted, including:
- IAM role assignments for managed identity access
- Cosmos DB change feed configuration
- Error handling and retry policies
- Monitoring and diagnostic settings
These omissions are intentional: the goal is to illustrate how the data connection is wired, not provide drop-in ingestion modules. See the CosmosDbDataConnection resource reference for all available configuration options.
Let's configure Azure Kusto CosmosDB Data Connections
Get started with Pulumi Cloud, then follow our quick setup guide to deploy this infrastructure.
Try Pulumi Cloud for FREEFrequently Asked Questions
Configuration & Setup
cosmosDbAccountResourceId, cosmosDbContainer, cosmosDbDatabase, kind (must be “CosmosDb”), managedIdentityResourceId, and tableName.kind property must be set to “CosmosDb” for Cosmos DB data connections.managedIdentityResourceId. This identity authenticates with Cosmos DB to retrieve data.Data Ingestion & Filtering
retrievalStartDate property to retrieve only documents created or updated after the specified date.mappingRuleName to use an existing mapping rule that transforms data during ingestion.tableName is case-sensitive and must exactly match an existing table in your Kusto cluster.Immutability & Limitations
clusterName, dataConnectionName, databaseName, and resourceGroupName. Changing these requires recreating the resource.