The azure-native:monitor:PipelineGroup resource, part of the Pulumi Azure Native provider, defines a telemetry pipeline that receives logs via UDP or syslog, optionally processes them, and exports to Azure Monitor Workspace Logs. This guide focuses on three capabilities: UDP and syslog receiver configuration, JSON array mapping and batch processing, and local caching and load balancer networking.
Pipeline groups run in Azure Arc custom locations and export to Data Collection Rules and Azure Monitor Workspace Logs tables that must exist separately. The examples are intentionally small. Combine them with your own Data Collection Rules, custom locations, and workspace configuration.
Receive UDP logs and export to Azure Monitor Workspace
Many log collection pipelines start by listening for UDP traffic on a specific port, then routing those logs to Azure Monitor Workspace for storage and analysis.
import * as pulumi from "@pulumi/pulumi";
import * as azure_native from "@pulumi/azure-native";
const pipelineGroup = new azure_native.monitor.PipelineGroup("pipelineGroup", {
extendedLocation: {
name: "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/myResourceGroup/providers/microsoft.extendedlocation/customlocations/myTestCustomLocation",
type: azure_native.monitor.ExtendedLocationType.CustomLocation,
},
location: "eastus2",
pipelineGroupName: "plGroup1",
properties: {
exporters: [{
azureMonitorWorkspaceLogs: {
api: {
dataCollectionEndpointUrl: "https://logs-myingestion-eb0s.eastus-1.ingest.monitor.azure.com",
dataCollectionRule: "dcr-00000000000000000000000000000000",
schema: {
recordMap: [
{
from: "body",
to: "Body",
},
{
from: "severity_text",
to: "SeverityText",
},
{
from: "time_unix_nano",
to: "TimeGenerated",
},
],
},
stream: "Custom-MyTableRawData_CL",
},
concurrency: {
batchQueueSize: 100,
workerCount: 4,
},
},
name: "my-workspace-logs-exporter1",
type: azure_native.monitor.ExporterType.AzureMonitorWorkspaceLogs,
}],
processors: [],
receivers: [{
name: "udp-receiver1",
type: azure_native.monitor.ReceiverType.UDP,
udp: {
encoding: azure_native.monitor.StreamEncodingType.Utf_8,
endpoint: "0.0.0.0:518",
},
}],
service: {
pipelines: [{
exporters: ["my-workspace-logs-exporter1"],
name: "MyPipelineForLogs1",
processors: [],
receivers: ["udp-receiver1"],
type: azure_native.monitor.PipelineType.Logs,
}],
},
},
resourceGroupName: "myResourceGroup",
tags: {
tag1: "A",
tag2: "B",
},
});
import pulumi
import pulumi_azure_native as azure_native
pipeline_group = azure_native.monitor.PipelineGroup("pipelineGroup",
extended_location={
"name": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/myResourceGroup/providers/microsoft.extendedlocation/customlocations/myTestCustomLocation",
"type": azure_native.monitor.ExtendedLocationType.CUSTOM_LOCATION,
},
location="eastus2",
pipeline_group_name="plGroup1",
properties={
"exporters": [{
"azure_monitor_workspace_logs": {
"api": {
"data_collection_endpoint_url": "https://logs-myingestion-eb0s.eastus-1.ingest.monitor.azure.com",
"data_collection_rule": "dcr-00000000000000000000000000000000",
"schema": {
"record_map": [
{
"from_": "body",
"to": "Body",
},
{
"from_": "severity_text",
"to": "SeverityText",
},
{
"from_": "time_unix_nano",
"to": "TimeGenerated",
},
],
},
"stream": "Custom-MyTableRawData_CL",
},
"concurrency": {
"batch_queue_size": 100,
"worker_count": 4,
},
},
"name": "my-workspace-logs-exporter1",
"type": azure_native.monitor.ExporterType.AZURE_MONITOR_WORKSPACE_LOGS,
}],
"processors": [],
"receivers": [{
"name": "udp-receiver1",
"type": azure_native.monitor.ReceiverType.UDP,
"udp": {
"encoding": azure_native.monitor.StreamEncodingType.UTF_8,
"endpoint": "0.0.0.0:518",
},
}],
"service": {
"pipelines": [{
"exporters": ["my-workspace-logs-exporter1"],
"name": "MyPipelineForLogs1",
"processors": [],
"receivers": ["udp-receiver1"],
"type": azure_native.monitor.PipelineType.LOGS,
}],
},
},
resource_group_name="myResourceGroup",
tags={
"tag1": "A",
"tag2": "B",
})
package main
import (
monitor "github.com/pulumi/pulumi-azure-native-sdk/monitor/v3"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
_, err := monitor.NewPipelineGroup(ctx, "pipelineGroup", &monitor.PipelineGroupArgs{
ExtendedLocation: &monitor.AzureResourceManagerCommonTypesExtendedLocationArgs{
Name: pulumi.String("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/myResourceGroup/providers/microsoft.extendedlocation/customlocations/myTestCustomLocation"),
Type: pulumi.String(monitor.ExtendedLocationTypeCustomLocation),
},
Location: pulumi.String("eastus2"),
PipelineGroupName: pulumi.String("plGroup1"),
Properties: &monitor.PipelineGroupPropertiesArgs{
Exporters: monitor.ExporterArray{
&monitor.ExporterArgs{
AzureMonitorWorkspaceLogs: &monitor.AzureMonitorWorkspaceLogsExporterArgs{
Api: &monitor.AzureMonitorWorkspaceLogsApiConfigArgs{
DataCollectionEndpointUrl: pulumi.String("https://logs-myingestion-eb0s.eastus-1.ingest.monitor.azure.com"),
DataCollectionRule: pulumi.String("dcr-00000000000000000000000000000000"),
Schema: &monitor.SchemaMapArgs{
RecordMap: monitor.RecordMapArray{
&monitor.RecordMapArgs{
From: pulumi.String("body"),
To: pulumi.String("Body"),
},
&monitor.RecordMapArgs{
From: pulumi.String("severity_text"),
To: pulumi.String("SeverityText"),
},
&monitor.RecordMapArgs{
From: pulumi.String("time_unix_nano"),
To: pulumi.String("TimeGenerated"),
},
},
},
Stream: pulumi.String("Custom-MyTableRawData_CL"),
},
Concurrency: &monitor.ConcurrencyConfigurationArgs{
BatchQueueSize: pulumi.Int(100),
WorkerCount: pulumi.Int(4),
},
},
Name: pulumi.String("my-workspace-logs-exporter1"),
Type: pulumi.String(monitor.ExporterTypeAzureMonitorWorkspaceLogs),
},
},
Processors: monitor.ProcessorArray{},
Receivers: monitor.ReceiverArray{
&monitor.ReceiverArgs{
Name: pulumi.String("udp-receiver1"),
Type: pulumi.String(monitor.ReceiverTypeUDP),
Udp: &monitor.UdpReceiverArgs{
Encoding: pulumi.String(monitor.StreamEncodingType_Utf_8),
Endpoint: pulumi.String("0.0.0.0:518"),
},
},
},
Service: &monitor.ServiceArgs{
Pipelines: monitor.PipelineArray{
&monitor.PipelineArgs{
Exporters: pulumi.StringArray{
pulumi.String("my-workspace-logs-exporter1"),
},
Name: pulumi.String("MyPipelineForLogs1"),
Processors: pulumi.StringArray{},
Receivers: pulumi.StringArray{
pulumi.String("udp-receiver1"),
},
Type: pulumi.String(monitor.PipelineTypeLogs),
},
},
},
},
ResourceGroupName: pulumi.String("myResourceGroup"),
Tags: pulumi.StringMap{
"tag1": pulumi.String("A"),
"tag2": pulumi.String("B"),
},
})
if err != nil {
return err
}
return nil
})
}
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using AzureNative = Pulumi.AzureNative;
return await Deployment.RunAsync(() =>
{
var pipelineGroup = new AzureNative.Monitor.PipelineGroup("pipelineGroup", new()
{
ExtendedLocation = new AzureNative.Monitor.Inputs.AzureResourceManagerCommonTypesExtendedLocationArgs
{
Name = "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/myResourceGroup/providers/microsoft.extendedlocation/customlocations/myTestCustomLocation",
Type = AzureNative.Monitor.ExtendedLocationType.CustomLocation,
},
Location = "eastus2",
PipelineGroupName = "plGroup1",
Properties = new AzureNative.Monitor.Inputs.PipelineGroupPropertiesArgs
{
Exporters = new[]
{
new AzureNative.Monitor.Inputs.ExporterArgs
{
AzureMonitorWorkspaceLogs = new AzureNative.Monitor.Inputs.AzureMonitorWorkspaceLogsExporterArgs
{
Api = new AzureNative.Monitor.Inputs.AzureMonitorWorkspaceLogsApiConfigArgs
{
DataCollectionEndpointUrl = "https://logs-myingestion-eb0s.eastus-1.ingest.monitor.azure.com",
DataCollectionRule = "dcr-00000000000000000000000000000000",
Schema = new AzureNative.Monitor.Inputs.SchemaMapArgs
{
RecordMap = new[]
{
new AzureNative.Monitor.Inputs.RecordMapArgs
{
From = "body",
To = "Body",
},
new AzureNative.Monitor.Inputs.RecordMapArgs
{
From = "severity_text",
To = "SeverityText",
},
new AzureNative.Monitor.Inputs.RecordMapArgs
{
From = "time_unix_nano",
To = "TimeGenerated",
},
},
},
Stream = "Custom-MyTableRawData_CL",
},
Concurrency = new AzureNative.Monitor.Inputs.ConcurrencyConfigurationArgs
{
BatchQueueSize = 100,
WorkerCount = 4,
},
},
Name = "my-workspace-logs-exporter1",
Type = AzureNative.Monitor.ExporterType.AzureMonitorWorkspaceLogs,
},
},
Processors = new() { },
Receivers = new[]
{
new AzureNative.Monitor.Inputs.ReceiverArgs
{
Name = "udp-receiver1",
Type = AzureNative.Monitor.ReceiverType.UDP,
Udp = new AzureNative.Monitor.Inputs.UdpReceiverArgs
{
Encoding = AzureNative.Monitor.StreamEncodingType.Utf_8,
Endpoint = "0.0.0.0:518",
},
},
},
Service = new AzureNative.Monitor.Inputs.ServiceArgs
{
Pipelines = new[]
{
new AzureNative.Monitor.Inputs.PipelineArgs
{
Exporters = new[]
{
"my-workspace-logs-exporter1",
},
Name = "MyPipelineForLogs1",
Processors = new() { },
Receivers = new[]
{
"udp-receiver1",
},
Type = AzureNative.Monitor.PipelineType.Logs,
},
},
},
},
ResourceGroupName = "myResourceGroup",
Tags =
{
{ "tag1", "A" },
{ "tag2", "B" },
},
});
});
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.azurenative.monitor.PipelineGroup;
import com.pulumi.azurenative.monitor.PipelineGroupArgs;
import com.pulumi.azurenative.monitor.inputs.AzureResourceManagerCommonTypesExtendedLocationArgs;
import com.pulumi.azurenative.monitor.inputs.PipelineGroupPropertiesArgs;
import com.pulumi.azurenative.monitor.inputs.ServiceArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var pipelineGroup = new PipelineGroup("pipelineGroup", PipelineGroupArgs.builder()
.extendedLocation(AzureResourceManagerCommonTypesExtendedLocationArgs.builder()
.name("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/myResourceGroup/providers/microsoft.extendedlocation/customlocations/myTestCustomLocation")
.type("CustomLocation")
.build())
.location("eastus2")
.pipelineGroupName("plGroup1")
.properties(PipelineGroupPropertiesArgs.builder()
.exporters(ExporterArgs.builder()
.azureMonitorWorkspaceLogs(AzureMonitorWorkspaceLogsExporterArgs.builder()
.api(AzureMonitorWorkspaceLogsApiConfigArgs.builder()
.dataCollectionEndpointUrl("https://logs-myingestion-eb0s.eastus-1.ingest.monitor.azure.com")
.dataCollectionRule("dcr-00000000000000000000000000000000")
.schema(SchemaMapArgs.builder()
.recordMap(
RecordMapArgs.builder()
.from("body")
.to("Body")
.build(),
RecordMapArgs.builder()
.from("severity_text")
.to("SeverityText")
.build(),
RecordMapArgs.builder()
.from("time_unix_nano")
.to("TimeGenerated")
.build())
.build())
.stream("Custom-MyTableRawData_CL")
.build())
.concurrency(ConcurrencyConfigurationArgs.builder()
.batchQueueSize(100)
.workerCount(4)
.build())
.build())
.name("my-workspace-logs-exporter1")
.type("AzureMonitorWorkspaceLogs")
.build())
.processors()
.receivers(ReceiverArgs.builder()
.name("udp-receiver1")
.type("UDP")
.udp(UdpReceiverArgs.builder()
.encoding("utf-8")
.endpoint("0.0.0.0:518")
.build())
.build())
.service(ServiceArgs.builder()
.pipelines(PipelineArgs.builder()
.exporters("my-workspace-logs-exporter1")
.name("MyPipelineForLogs1")
.processors()
.receivers("udp-receiver1")
.type("Logs")
.build())
.build())
.build())
.resourceGroupName("myResourceGroup")
.tags(Map.ofEntries(
Map.entry("tag1", "A"),
Map.entry("tag2", "B")
))
.build());
}
}
resources:
pipelineGroup:
type: azure-native:monitor:PipelineGroup
properties:
extendedLocation:
name: /subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/myResourceGroup/providers/microsoft.extendedlocation/customlocations/myTestCustomLocation
type: CustomLocation
location: eastus2
pipelineGroupName: plGroup1
properties:
exporters:
- azureMonitorWorkspaceLogs:
api:
dataCollectionEndpointUrl: https://logs-myingestion-eb0s.eastus-1.ingest.monitor.azure.com
dataCollectionRule: dcr-00000000000000000000000000000000
schema:
recordMap:
- from: body
to: Body
- from: severity_text
to: SeverityText
- from: time_unix_nano
to: TimeGenerated
stream: Custom-MyTableRawData_CL
concurrency:
batchQueueSize: 100
workerCount: 4
name: my-workspace-logs-exporter1
type: AzureMonitorWorkspaceLogs
processors: []
receivers:
- name: udp-receiver1
type: UDP
udp:
encoding: utf-8
endpoint: 0.0.0.0:518
service:
pipelines:
- exporters:
- my-workspace-logs-exporter1
name: MyPipelineForLogs1
processors: []
receivers:
- udp-receiver1
type: Logs
resourceGroupName: myResourceGroup
tags:
tag1: A
tag2: B
The receiver listens on the specified UDP endpoint and forwards logs to the exporter. The azureMonitorWorkspaceLogs exporter sends data to the dataCollectionEndpointUrl using the specified dataCollectionRule. The schema.recordMap transforms incoming fields (like “body”, “severity_text”) to match the target table schema. The service.pipelines section connects the receiver to the exporter by name.
Transform JSON arrays during UDP log ingestion
When UDP logs contain JSON arrays that need field extraction or restructuring, the jsonArrayMapper transforms nested data before export.
import * as pulumi from "@pulumi/pulumi";
import * as azure_native from "@pulumi/azure-native";
const pipelineGroup = new azure_native.monitor.PipelineGroup("pipelineGroup", {
extendedLocation: {
name: "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/myResourceGroup/providers/microsoft.extendedlocation/customlocations/myTestCustomLocation",
type: azure_native.monitor.ExtendedLocationType.CustomLocation,
},
location: "eastus2",
pipelineGroupName: "plGroup1",
properties: {
exporters: [{
azureMonitorWorkspaceLogs: {
api: {
dataCollectionEndpointUrl: "https://logs-myingestion-eb0s.eastus-1.ingest.monitor.azure.com",
dataCollectionRule: "dcr-00000000000000000000000000000000",
schema: {
recordMap: [
{
from: "body",
to: "Body",
},
{
from: "severity_text",
to: "SeverityText",
},
{
from: "time_unix_nano",
to: "TimeGenerated",
},
],
},
stream: "Custom-MyTableRawData_CL",
},
concurrency: {
batchQueueSize: 100,
workerCount: 4,
},
},
name: "my-workspace-logs-exporter1",
type: azure_native.monitor.ExporterType.AzureMonitorWorkspaceLogs,
}],
processors: [],
receivers: [{
name: "udp-receiver1",
type: azure_native.monitor.ReceiverType.UDP,
udp: {
encoding: azure_native.monitor.StreamEncodingType.Utf_8,
endpoint: "0.0.0.0:518",
jsonArrayMapper: {
destinationField: {
destination: azure_native.monitor.JsonMapperElement.Attributes,
},
keys: [
"key1",
"key2",
"key3",
],
sourceField: {
fieldName: "field1",
},
},
},
}],
service: {
pipelines: [{
exporters: ["my-workspace-logs-exporter1"],
name: "MyPipelineForLogs1",
processors: [],
receivers: ["udp-receiver1"],
type: azure_native.monitor.PipelineType.Logs,
}],
},
},
resourceGroupName: "myResourceGroup",
tags: {
tag1: "A",
tag2: "B",
},
});
import pulumi
import pulumi_azure_native as azure_native
pipeline_group = azure_native.monitor.PipelineGroup("pipelineGroup",
extended_location={
"name": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/myResourceGroup/providers/microsoft.extendedlocation/customlocations/myTestCustomLocation",
"type": azure_native.monitor.ExtendedLocationType.CUSTOM_LOCATION,
},
location="eastus2",
pipeline_group_name="plGroup1",
properties={
"exporters": [{
"azure_monitor_workspace_logs": {
"api": {
"data_collection_endpoint_url": "https://logs-myingestion-eb0s.eastus-1.ingest.monitor.azure.com",
"data_collection_rule": "dcr-00000000000000000000000000000000",
"schema": {
"record_map": [
{
"from_": "body",
"to": "Body",
},
{
"from_": "severity_text",
"to": "SeverityText",
},
{
"from_": "time_unix_nano",
"to": "TimeGenerated",
},
],
},
"stream": "Custom-MyTableRawData_CL",
},
"concurrency": {
"batch_queue_size": 100,
"worker_count": 4,
},
},
"name": "my-workspace-logs-exporter1",
"type": azure_native.monitor.ExporterType.AZURE_MONITOR_WORKSPACE_LOGS,
}],
"processors": [],
"receivers": [{
"name": "udp-receiver1",
"type": azure_native.monitor.ReceiverType.UDP,
"udp": {
"encoding": azure_native.monitor.StreamEncodingType.UTF_8,
"endpoint": "0.0.0.0:518",
"json_array_mapper": {
"destination_field": {
"destination": azure_native.monitor.JsonMapperElement.ATTRIBUTES,
},
"keys": [
"key1",
"key2",
"key3",
],
"source_field": {
"field_name": "field1",
},
},
},
}],
"service": {
"pipelines": [{
"exporters": ["my-workspace-logs-exporter1"],
"name": "MyPipelineForLogs1",
"processors": [],
"receivers": ["udp-receiver1"],
"type": azure_native.monitor.PipelineType.LOGS,
}],
},
},
resource_group_name="myResourceGroup",
tags={
"tag1": "A",
"tag2": "B",
})
package main
import (
monitor "github.com/pulumi/pulumi-azure-native-sdk/monitor/v3"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
_, err := monitor.NewPipelineGroup(ctx, "pipelineGroup", &monitor.PipelineGroupArgs{
ExtendedLocation: &monitor.AzureResourceManagerCommonTypesExtendedLocationArgs{
Name: pulumi.String("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/myResourceGroup/providers/microsoft.extendedlocation/customlocations/myTestCustomLocation"),
Type: pulumi.String(monitor.ExtendedLocationTypeCustomLocation),
},
Location: pulumi.String("eastus2"),
PipelineGroupName: pulumi.String("plGroup1"),
Properties: &monitor.PipelineGroupPropertiesArgs{
Exporters: monitor.ExporterArray{
&monitor.ExporterArgs{
AzureMonitorWorkspaceLogs: &monitor.AzureMonitorWorkspaceLogsExporterArgs{
Api: &monitor.AzureMonitorWorkspaceLogsApiConfigArgs{
DataCollectionEndpointUrl: pulumi.String("https://logs-myingestion-eb0s.eastus-1.ingest.monitor.azure.com"),
DataCollectionRule: pulumi.String("dcr-00000000000000000000000000000000"),
Schema: &monitor.SchemaMapArgs{
RecordMap: monitor.RecordMapArray{
&monitor.RecordMapArgs{
From: pulumi.String("body"),
To: pulumi.String("Body"),
},
&monitor.RecordMapArgs{
From: pulumi.String("severity_text"),
To: pulumi.String("SeverityText"),
},
&monitor.RecordMapArgs{
From: pulumi.String("time_unix_nano"),
To: pulumi.String("TimeGenerated"),
},
},
},
Stream: pulumi.String("Custom-MyTableRawData_CL"),
},
Concurrency: &monitor.ConcurrencyConfigurationArgs{
BatchQueueSize: pulumi.Int(100),
WorkerCount: pulumi.Int(4),
},
},
Name: pulumi.String("my-workspace-logs-exporter1"),
Type: pulumi.String(monitor.ExporterTypeAzureMonitorWorkspaceLogs),
},
},
Processors: monitor.ProcessorArray{},
Receivers: monitor.ReceiverArray{
&monitor.ReceiverArgs{
Name: pulumi.String("udp-receiver1"),
Type: pulumi.String(monitor.ReceiverTypeUDP),
Udp: &monitor.UdpReceiverArgs{
Encoding: pulumi.String(monitor.StreamEncodingType_Utf_8),
Endpoint: pulumi.String("0.0.0.0:518"),
JsonArrayMapper: &monitor.JsonArrayMapperArgs{
DestinationField: &monitor.JsonMapperDestinationFieldArgs{
Destination: pulumi.String(monitor.JsonMapperElementAttributes),
},
Keys: pulumi.StringArray{
pulumi.String("key1"),
pulumi.String("key2"),
pulumi.String("key3"),
},
SourceField: &monitor.JsonMapperSourceFieldArgs{
FieldName: pulumi.String("field1"),
},
},
},
},
},
Service: &monitor.ServiceArgs{
Pipelines: monitor.PipelineArray{
&monitor.PipelineArgs{
Exporters: pulumi.StringArray{
pulumi.String("my-workspace-logs-exporter1"),
},
Name: pulumi.String("MyPipelineForLogs1"),
Processors: pulumi.StringArray{},
Receivers: pulumi.StringArray{
pulumi.String("udp-receiver1"),
},
Type: pulumi.String(monitor.PipelineTypeLogs),
},
},
},
},
ResourceGroupName: pulumi.String("myResourceGroup"),
Tags: pulumi.StringMap{
"tag1": pulumi.String("A"),
"tag2": pulumi.String("B"),
},
})
if err != nil {
return err
}
return nil
})
}
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using AzureNative = Pulumi.AzureNative;
return await Deployment.RunAsync(() =>
{
var pipelineGroup = new AzureNative.Monitor.PipelineGroup("pipelineGroup", new()
{
ExtendedLocation = new AzureNative.Monitor.Inputs.AzureResourceManagerCommonTypesExtendedLocationArgs
{
Name = "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/myResourceGroup/providers/microsoft.extendedlocation/customlocations/myTestCustomLocation",
Type = AzureNative.Monitor.ExtendedLocationType.CustomLocation,
},
Location = "eastus2",
PipelineGroupName = "plGroup1",
Properties = new AzureNative.Monitor.Inputs.PipelineGroupPropertiesArgs
{
Exporters = new[]
{
new AzureNative.Monitor.Inputs.ExporterArgs
{
AzureMonitorWorkspaceLogs = new AzureNative.Monitor.Inputs.AzureMonitorWorkspaceLogsExporterArgs
{
Api = new AzureNative.Monitor.Inputs.AzureMonitorWorkspaceLogsApiConfigArgs
{
DataCollectionEndpointUrl = "https://logs-myingestion-eb0s.eastus-1.ingest.monitor.azure.com",
DataCollectionRule = "dcr-00000000000000000000000000000000",
Schema = new AzureNative.Monitor.Inputs.SchemaMapArgs
{
RecordMap = new[]
{
new AzureNative.Monitor.Inputs.RecordMapArgs
{
From = "body",
To = "Body",
},
new AzureNative.Monitor.Inputs.RecordMapArgs
{
From = "severity_text",
To = "SeverityText",
},
new AzureNative.Monitor.Inputs.RecordMapArgs
{
From = "time_unix_nano",
To = "TimeGenerated",
},
},
},
Stream = "Custom-MyTableRawData_CL",
},
Concurrency = new AzureNative.Monitor.Inputs.ConcurrencyConfigurationArgs
{
BatchQueueSize = 100,
WorkerCount = 4,
},
},
Name = "my-workspace-logs-exporter1",
Type = AzureNative.Monitor.ExporterType.AzureMonitorWorkspaceLogs,
},
},
Processors = new() { },
Receivers = new[]
{
new AzureNative.Monitor.Inputs.ReceiverArgs
{
Name = "udp-receiver1",
Type = AzureNative.Monitor.ReceiverType.UDP,
Udp = new AzureNative.Monitor.Inputs.UdpReceiverArgs
{
Encoding = AzureNative.Monitor.StreamEncodingType.Utf_8,
Endpoint = "0.0.0.0:518",
JsonArrayMapper = new AzureNative.Monitor.Inputs.JsonArrayMapperArgs
{
DestinationField = new AzureNative.Monitor.Inputs.JsonMapperDestinationFieldArgs
{
Destination = AzureNative.Monitor.JsonMapperElement.Attributes,
},
Keys = new[]
{
"key1",
"key2",
"key3",
},
SourceField = new AzureNative.Monitor.Inputs.JsonMapperSourceFieldArgs
{
FieldName = "field1",
},
},
},
},
},
Service = new AzureNative.Monitor.Inputs.ServiceArgs
{
Pipelines = new[]
{
new AzureNative.Monitor.Inputs.PipelineArgs
{
Exporters = new[]
{
"my-workspace-logs-exporter1",
},
Name = "MyPipelineForLogs1",
Processors = new() { },
Receivers = new[]
{
"udp-receiver1",
},
Type = AzureNative.Monitor.PipelineType.Logs,
},
},
},
},
ResourceGroupName = "myResourceGroup",
Tags =
{
{ "tag1", "A" },
{ "tag2", "B" },
},
});
});
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.azurenative.monitor.PipelineGroup;
import com.pulumi.azurenative.monitor.PipelineGroupArgs;
import com.pulumi.azurenative.monitor.inputs.AzureResourceManagerCommonTypesExtendedLocationArgs;
import com.pulumi.azurenative.monitor.inputs.PipelineGroupPropertiesArgs;
import com.pulumi.azurenative.monitor.inputs.ServiceArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var pipelineGroup = new PipelineGroup("pipelineGroup", PipelineGroupArgs.builder()
.extendedLocation(AzureResourceManagerCommonTypesExtendedLocationArgs.builder()
.name("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/myResourceGroup/providers/microsoft.extendedlocation/customlocations/myTestCustomLocation")
.type("CustomLocation")
.build())
.location("eastus2")
.pipelineGroupName("plGroup1")
.properties(PipelineGroupPropertiesArgs.builder()
.exporters(ExporterArgs.builder()
.azureMonitorWorkspaceLogs(AzureMonitorWorkspaceLogsExporterArgs.builder()
.api(AzureMonitorWorkspaceLogsApiConfigArgs.builder()
.dataCollectionEndpointUrl("https://logs-myingestion-eb0s.eastus-1.ingest.monitor.azure.com")
.dataCollectionRule("dcr-00000000000000000000000000000000")
.schema(SchemaMapArgs.builder()
.recordMap(
RecordMapArgs.builder()
.from("body")
.to("Body")
.build(),
RecordMapArgs.builder()
.from("severity_text")
.to("SeverityText")
.build(),
RecordMapArgs.builder()
.from("time_unix_nano")
.to("TimeGenerated")
.build())
.build())
.stream("Custom-MyTableRawData_CL")
.build())
.concurrency(ConcurrencyConfigurationArgs.builder()
.batchQueueSize(100)
.workerCount(4)
.build())
.build())
.name("my-workspace-logs-exporter1")
.type("AzureMonitorWorkspaceLogs")
.build())
.processors()
.receivers(ReceiverArgs.builder()
.name("udp-receiver1")
.type("UDP")
.udp(UdpReceiverArgs.builder()
.encoding("utf-8")
.endpoint("0.0.0.0:518")
.jsonArrayMapper(JsonArrayMapperArgs.builder()
.destinationField(JsonMapperDestinationFieldArgs.builder()
.destination("attributes")
.build())
.keys(
"key1",
"key2",
"key3")
.sourceField(JsonMapperSourceFieldArgs.builder()
.fieldName("field1")
.build())
.build())
.build())
.build())
.service(ServiceArgs.builder()
.pipelines(PipelineArgs.builder()
.exporters("my-workspace-logs-exporter1")
.name("MyPipelineForLogs1")
.processors()
.receivers("udp-receiver1")
.type("Logs")
.build())
.build())
.build())
.resourceGroupName("myResourceGroup")
.tags(Map.ofEntries(
Map.entry("tag1", "A"),
Map.entry("tag2", "B")
))
.build());
}
}
resources:
pipelineGroup:
type: azure-native:monitor:PipelineGroup
properties:
extendedLocation:
name: /subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/myResourceGroup/providers/microsoft.extendedlocation/customlocations/myTestCustomLocation
type: CustomLocation
location: eastus2
pipelineGroupName: plGroup1
properties:
exporters:
- azureMonitorWorkspaceLogs:
api:
dataCollectionEndpointUrl: https://logs-myingestion-eb0s.eastus-1.ingest.monitor.azure.com
dataCollectionRule: dcr-00000000000000000000000000000000
schema:
recordMap:
- from: body
to: Body
- from: severity_text
to: SeverityText
- from: time_unix_nano
to: TimeGenerated
stream: Custom-MyTableRawData_CL
concurrency:
batchQueueSize: 100
workerCount: 4
name: my-workspace-logs-exporter1
type: AzureMonitorWorkspaceLogs
processors: []
receivers:
- name: udp-receiver1
type: UDP
udp:
encoding: utf-8
endpoint: 0.0.0.0:518
jsonArrayMapper:
destinationField:
destination: attributes
keys:
- key1
- key2
- key3
sourceField:
fieldName: field1
service:
pipelines:
- exporters:
- my-workspace-logs-exporter1
name: MyPipelineForLogs1
processors: []
receivers:
- udp-receiver1
type: Logs
resourceGroupName: myResourceGroup
tags:
tag1: A
tag2: B
The jsonArrayMapper extracts the sourceField (“field1”) from incoming JSON, splits it into the specified keys, and writes them to the destinationField (attributes). This transformation happens before the logs reach the exporter, allowing you to normalize complex JSON structures into flat fields that match your workspace schema.
Collect syslog messages with batch processing
Syslog receivers listen on standard syslog ports and can apply batch processing to group messages before export, reducing API calls and improving throughput.
import * as pulumi from "@pulumi/pulumi";
import * as azure_native from "@pulumi/azure-native";
const pipelineGroup = new azure_native.monitor.PipelineGroup("pipelineGroup", {
extendedLocation: {
name: "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/myResourceGroup/providers/microsoft.extendedlocation/customlocations/myTestCustomLocation",
type: azure_native.monitor.ExtendedLocationType.CustomLocation,
},
location: "eastus2",
pipelineGroupName: "plGroup1",
properties: {
exporters: [{
azureMonitorWorkspaceLogs: {
api: {
dataCollectionEndpointUrl: "https://logs-myingestion-eb0s.eastus-1.ingest.monitor.azure.com",
dataCollectionRule: "dcr-00000000000000000000000000000000",
schema: {
recordMap: [
{
from: "body",
to: "Body",
},
{
from: "severity_text",
to: "SeverityText",
},
{
from: "time_unix_nano",
to: "TimeGenerated",
},
],
},
stream: "Custom-MyTableRawData_CL",
},
concurrency: {
batchQueueSize: 100,
workerCount: 4,
},
},
name: "my-workspace-logs-exporter1",
type: azure_native.monitor.ExporterType.AzureMonitorWorkspaceLogs,
}],
processors: [{
name: "batch-processor1",
type: azure_native.monitor.ProcessorType.Batch,
}],
receivers: [{
name: "syslog-receiver1",
syslog: {
endpoint: "0.0.0.0:514",
},
type: azure_native.monitor.ReceiverType.Syslog,
}],
service: {
pipelines: [{
exporters: ["my-workspace-logs-exporter1"],
name: "MyPipelineForLogs1",
processors: ["batch-processor1"],
receivers: ["syslog-receiver1"],
type: azure_native.monitor.PipelineType.Logs,
}],
},
},
resourceGroupName: "myResourceGroup",
tags: {
tag1: "A",
tag2: "B",
},
});
import pulumi
import pulumi_azure_native as azure_native
pipeline_group = azure_native.monitor.PipelineGroup("pipelineGroup",
extended_location={
"name": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/myResourceGroup/providers/microsoft.extendedlocation/customlocations/myTestCustomLocation",
"type": azure_native.monitor.ExtendedLocationType.CUSTOM_LOCATION,
},
location="eastus2",
pipeline_group_name="plGroup1",
properties={
"exporters": [{
"azure_monitor_workspace_logs": {
"api": {
"data_collection_endpoint_url": "https://logs-myingestion-eb0s.eastus-1.ingest.monitor.azure.com",
"data_collection_rule": "dcr-00000000000000000000000000000000",
"schema": {
"record_map": [
{
"from_": "body",
"to": "Body",
},
{
"from_": "severity_text",
"to": "SeverityText",
},
{
"from_": "time_unix_nano",
"to": "TimeGenerated",
},
],
},
"stream": "Custom-MyTableRawData_CL",
},
"concurrency": {
"batch_queue_size": 100,
"worker_count": 4,
},
},
"name": "my-workspace-logs-exporter1",
"type": azure_native.monitor.ExporterType.AZURE_MONITOR_WORKSPACE_LOGS,
}],
"processors": [{
"name": "batch-processor1",
"type": azure_native.monitor.ProcessorType.BATCH,
}],
"receivers": [{
"name": "syslog-receiver1",
"syslog": {
"endpoint": "0.0.0.0:514",
},
"type": azure_native.monitor.ReceiverType.SYSLOG,
}],
"service": {
"pipelines": [{
"exporters": ["my-workspace-logs-exporter1"],
"name": "MyPipelineForLogs1",
"processors": ["batch-processor1"],
"receivers": ["syslog-receiver1"],
"type": azure_native.monitor.PipelineType.LOGS,
}],
},
},
resource_group_name="myResourceGroup",
tags={
"tag1": "A",
"tag2": "B",
})
package main
import (
monitor "github.com/pulumi/pulumi-azure-native-sdk/monitor/v3"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
_, err := monitor.NewPipelineGroup(ctx, "pipelineGroup", &monitor.PipelineGroupArgs{
ExtendedLocation: &monitor.AzureResourceManagerCommonTypesExtendedLocationArgs{
Name: pulumi.String("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/myResourceGroup/providers/microsoft.extendedlocation/customlocations/myTestCustomLocation"),
Type: pulumi.String(monitor.ExtendedLocationTypeCustomLocation),
},
Location: pulumi.String("eastus2"),
PipelineGroupName: pulumi.String("plGroup1"),
Properties: &monitor.PipelineGroupPropertiesArgs{
Exporters: monitor.ExporterArray{
&monitor.ExporterArgs{
AzureMonitorWorkspaceLogs: &monitor.AzureMonitorWorkspaceLogsExporterArgs{
Api: &monitor.AzureMonitorWorkspaceLogsApiConfigArgs{
DataCollectionEndpointUrl: pulumi.String("https://logs-myingestion-eb0s.eastus-1.ingest.monitor.azure.com"),
DataCollectionRule: pulumi.String("dcr-00000000000000000000000000000000"),
Schema: &monitor.SchemaMapArgs{
RecordMap: monitor.RecordMapArray{
&monitor.RecordMapArgs{
From: pulumi.String("body"),
To: pulumi.String("Body"),
},
&monitor.RecordMapArgs{
From: pulumi.String("severity_text"),
To: pulumi.String("SeverityText"),
},
&monitor.RecordMapArgs{
From: pulumi.String("time_unix_nano"),
To: pulumi.String("TimeGenerated"),
},
},
},
Stream: pulumi.String("Custom-MyTableRawData_CL"),
},
Concurrency: &monitor.ConcurrencyConfigurationArgs{
BatchQueueSize: pulumi.Int(100),
WorkerCount: pulumi.Int(4),
},
},
Name: pulumi.String("my-workspace-logs-exporter1"),
Type: pulumi.String(monitor.ExporterTypeAzureMonitorWorkspaceLogs),
},
},
Processors: monitor.ProcessorArray{
&monitor.ProcessorArgs{
Name: pulumi.String("batch-processor1"),
Type: pulumi.String(monitor.ProcessorTypeBatch),
},
},
Receivers: monitor.ReceiverArray{
&monitor.ReceiverArgs{
Name: pulumi.String("syslog-receiver1"),
Syslog: &monitor.SyslogReceiverArgs{
Endpoint: pulumi.String("0.0.0.0:514"),
},
Type: pulumi.String(monitor.ReceiverTypeSyslog),
},
},
Service: &monitor.ServiceArgs{
Pipelines: monitor.PipelineArray{
&monitor.PipelineArgs{
Exporters: pulumi.StringArray{
pulumi.String("my-workspace-logs-exporter1"),
},
Name: pulumi.String("MyPipelineForLogs1"),
Processors: pulumi.StringArray{
pulumi.String("batch-processor1"),
},
Receivers: pulumi.StringArray{
pulumi.String("syslog-receiver1"),
},
Type: pulumi.String(monitor.PipelineTypeLogs),
},
},
},
},
ResourceGroupName: pulumi.String("myResourceGroup"),
Tags: pulumi.StringMap{
"tag1": pulumi.String("A"),
"tag2": pulumi.String("B"),
},
})
if err != nil {
return err
}
return nil
})
}
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using AzureNative = Pulumi.AzureNative;
return await Deployment.RunAsync(() =>
{
var pipelineGroup = new AzureNative.Monitor.PipelineGroup("pipelineGroup", new()
{
ExtendedLocation = new AzureNative.Monitor.Inputs.AzureResourceManagerCommonTypesExtendedLocationArgs
{
Name = "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/myResourceGroup/providers/microsoft.extendedlocation/customlocations/myTestCustomLocation",
Type = AzureNative.Monitor.ExtendedLocationType.CustomLocation,
},
Location = "eastus2",
PipelineGroupName = "plGroup1",
Properties = new AzureNative.Monitor.Inputs.PipelineGroupPropertiesArgs
{
Exporters = new[]
{
new AzureNative.Monitor.Inputs.ExporterArgs
{
AzureMonitorWorkspaceLogs = new AzureNative.Monitor.Inputs.AzureMonitorWorkspaceLogsExporterArgs
{
Api = new AzureNative.Monitor.Inputs.AzureMonitorWorkspaceLogsApiConfigArgs
{
DataCollectionEndpointUrl = "https://logs-myingestion-eb0s.eastus-1.ingest.monitor.azure.com",
DataCollectionRule = "dcr-00000000000000000000000000000000",
Schema = new AzureNative.Monitor.Inputs.SchemaMapArgs
{
RecordMap = new[]
{
new AzureNative.Monitor.Inputs.RecordMapArgs
{
From = "body",
To = "Body",
},
new AzureNative.Monitor.Inputs.RecordMapArgs
{
From = "severity_text",
To = "SeverityText",
},
new AzureNative.Monitor.Inputs.RecordMapArgs
{
From = "time_unix_nano",
To = "TimeGenerated",
},
},
},
Stream = "Custom-MyTableRawData_CL",
},
Concurrency = new AzureNative.Monitor.Inputs.ConcurrencyConfigurationArgs
{
BatchQueueSize = 100,
WorkerCount = 4,
},
},
Name = "my-workspace-logs-exporter1",
Type = AzureNative.Monitor.ExporterType.AzureMonitorWorkspaceLogs,
},
},
Processors = new[]
{
new AzureNative.Monitor.Inputs.ProcessorArgs
{
Name = "batch-processor1",
Type = AzureNative.Monitor.ProcessorType.Batch,
},
},
Receivers = new[]
{
new AzureNative.Monitor.Inputs.ReceiverArgs
{
Name = "syslog-receiver1",
Syslog = new AzureNative.Monitor.Inputs.SyslogReceiverArgs
{
Endpoint = "0.0.0.0:514",
},
Type = AzureNative.Monitor.ReceiverType.Syslog,
},
},
Service = new AzureNative.Monitor.Inputs.ServiceArgs
{
Pipelines = new[]
{
new AzureNative.Monitor.Inputs.PipelineArgs
{
Exporters = new[]
{
"my-workspace-logs-exporter1",
},
Name = "MyPipelineForLogs1",
Processors = new[]
{
"batch-processor1",
},
Receivers = new[]
{
"syslog-receiver1",
},
Type = AzureNative.Monitor.PipelineType.Logs,
},
},
},
},
ResourceGroupName = "myResourceGroup",
Tags =
{
{ "tag1", "A" },
{ "tag2", "B" },
},
});
});
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.azurenative.monitor.PipelineGroup;
import com.pulumi.azurenative.monitor.PipelineGroupArgs;
import com.pulumi.azurenative.monitor.inputs.AzureResourceManagerCommonTypesExtendedLocationArgs;
import com.pulumi.azurenative.monitor.inputs.PipelineGroupPropertiesArgs;
import com.pulumi.azurenative.monitor.inputs.ServiceArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var pipelineGroup = new PipelineGroup("pipelineGroup", PipelineGroupArgs.builder()
.extendedLocation(AzureResourceManagerCommonTypesExtendedLocationArgs.builder()
.name("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/myResourceGroup/providers/microsoft.extendedlocation/customlocations/myTestCustomLocation")
.type("CustomLocation")
.build())
.location("eastus2")
.pipelineGroupName("plGroup1")
.properties(PipelineGroupPropertiesArgs.builder()
.exporters(ExporterArgs.builder()
.azureMonitorWorkspaceLogs(AzureMonitorWorkspaceLogsExporterArgs.builder()
.api(AzureMonitorWorkspaceLogsApiConfigArgs.builder()
.dataCollectionEndpointUrl("https://logs-myingestion-eb0s.eastus-1.ingest.monitor.azure.com")
.dataCollectionRule("dcr-00000000000000000000000000000000")
.schema(SchemaMapArgs.builder()
.recordMap(
RecordMapArgs.builder()
.from("body")
.to("Body")
.build(),
RecordMapArgs.builder()
.from("severity_text")
.to("SeverityText")
.build(),
RecordMapArgs.builder()
.from("time_unix_nano")
.to("TimeGenerated")
.build())
.build())
.stream("Custom-MyTableRawData_CL")
.build())
.concurrency(ConcurrencyConfigurationArgs.builder()
.batchQueueSize(100)
.workerCount(4)
.build())
.build())
.name("my-workspace-logs-exporter1")
.type("AzureMonitorWorkspaceLogs")
.build())
.processors(ProcessorArgs.builder()
.name("batch-processor1")
.type("Batch")
.build())
.receivers(ReceiverArgs.builder()
.name("syslog-receiver1")
.syslog(SyslogReceiverArgs.builder()
.endpoint("0.0.0.0:514")
.build())
.type("Syslog")
.build())
.service(ServiceArgs.builder()
.pipelines(PipelineArgs.builder()
.exporters("my-workspace-logs-exporter1")
.name("MyPipelineForLogs1")
.processors("batch-processor1")
.receivers("syslog-receiver1")
.type("Logs")
.build())
.build())
.build())
.resourceGroupName("myResourceGroup")
.tags(Map.ofEntries(
Map.entry("tag1", "A"),
Map.entry("tag2", "B")
))
.build());
}
}
resources:
pipelineGroup:
type: azure-native:monitor:PipelineGroup
properties:
extendedLocation:
name: /subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/myResourceGroup/providers/microsoft.extendedlocation/customlocations/myTestCustomLocation
type: CustomLocation
location: eastus2
pipelineGroupName: plGroup1
properties:
exporters:
- azureMonitorWorkspaceLogs:
api:
dataCollectionEndpointUrl: https://logs-myingestion-eb0s.eastus-1.ingest.monitor.azure.com
dataCollectionRule: dcr-00000000000000000000000000000000
schema:
recordMap:
- from: body
to: Body
- from: severity_text
to: SeverityText
- from: time_unix_nano
to: TimeGenerated
stream: Custom-MyTableRawData_CL
concurrency:
batchQueueSize: 100
workerCount: 4
name: my-workspace-logs-exporter1
type: AzureMonitorWorkspaceLogs
processors:
- name: batch-processor1
type: Batch
receivers:
- name: syslog-receiver1
syslog:
endpoint: 0.0.0.0:514
type: Syslog
service:
pipelines:
- exporters:
- my-workspace-logs-exporter1
name: MyPipelineForLogs1
processors:
- batch-processor1
receivers:
- syslog-receiver1
type: Logs
resourceGroupName: myResourceGroup
tags:
tag1: A
tag2: B
The syslog receiver listens on port 514 (the standard syslog port). The batch processor groups messages together before sending them to the exporter, reducing the number of API calls to Azure Monitor. The processors array in the pipeline configuration specifies which processors to apply between the receiver and exporter.
Enable local caching for syslog export resilience
When network connectivity to Azure Monitor is unreliable, local caching stores logs temporarily and retries export, preventing data loss during outages.
import * as pulumi from "@pulumi/pulumi";
import * as azure_native from "@pulumi/azure-native";
const pipelineGroup = new azure_native.monitor.PipelineGroup("pipelineGroup", {
extendedLocation: {
name: "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/myResourceGroup/providers/microsoft.extendedlocation/customlocations/myTestCustomLocation",
type: azure_native.monitor.ExtendedLocationType.CustomLocation,
},
location: "eastus2",
pipelineGroupName: "plGroup1",
properties: {
exporters: [{
azureMonitorWorkspaceLogs: {
api: {
dataCollectionEndpointUrl: "https://logs-myingestion-eb0s.eastus-1.ingest.monitor.azure.com",
dataCollectionRule: "dcr-00000000000000000000000000000000",
schema: {
recordMap: [
{
from: "body",
to: "Body",
},
{
from: "severity_text",
to: "SeverityText",
},
{
from: "time_unix_nano",
to: "TimeGenerated",
},
],
},
stream: "Custom-MyTableRawData_CL",
},
cache: {
maxStorageUsage: 100,
retentionPeriod: 10,
},
concurrency: {
batchQueueSize: 100,
workerCount: 4,
},
},
name: "my-workspace-logs-exporter1",
type: azure_native.monitor.ExporterType.AzureMonitorWorkspaceLogs,
}],
processors: [{
name: "batch-processor1",
type: azure_native.monitor.ProcessorType.Batch,
}],
receivers: [{
name: "syslog-receiver1",
syslog: {
endpoint: "0.0.0.0:514",
},
type: azure_native.monitor.ReceiverType.Syslog,
}],
service: {
pipelines: [{
exporters: ["my-workspace-logs-exporter1"],
name: "MyPipelineForLogs1",
processors: ["batch-processor1"],
receivers: ["syslog-receiver1"],
type: azure_native.monitor.PipelineType.Logs,
}],
},
},
resourceGroupName: "myResourceGroup",
tags: {
tag1: "A",
tag2: "B",
},
});
import pulumi
import pulumi_azure_native as azure_native
pipeline_group = azure_native.monitor.PipelineGroup("pipelineGroup",
extended_location={
"name": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/myResourceGroup/providers/microsoft.extendedlocation/customlocations/myTestCustomLocation",
"type": azure_native.monitor.ExtendedLocationType.CUSTOM_LOCATION,
},
location="eastus2",
pipeline_group_name="plGroup1",
properties={
"exporters": [{
"azure_monitor_workspace_logs": {
"api": {
"data_collection_endpoint_url": "https://logs-myingestion-eb0s.eastus-1.ingest.monitor.azure.com",
"data_collection_rule": "dcr-00000000000000000000000000000000",
"schema": {
"record_map": [
{
"from_": "body",
"to": "Body",
},
{
"from_": "severity_text",
"to": "SeverityText",
},
{
"from_": "time_unix_nano",
"to": "TimeGenerated",
},
],
},
"stream": "Custom-MyTableRawData_CL",
},
"cache": {
"max_storage_usage": 100,
"retention_period": 10,
},
"concurrency": {
"batch_queue_size": 100,
"worker_count": 4,
},
},
"name": "my-workspace-logs-exporter1",
"type": azure_native.monitor.ExporterType.AZURE_MONITOR_WORKSPACE_LOGS,
}],
"processors": [{
"name": "batch-processor1",
"type": azure_native.monitor.ProcessorType.BATCH,
}],
"receivers": [{
"name": "syslog-receiver1",
"syslog": {
"endpoint": "0.0.0.0:514",
},
"type": azure_native.monitor.ReceiverType.SYSLOG,
}],
"service": {
"pipelines": [{
"exporters": ["my-workspace-logs-exporter1"],
"name": "MyPipelineForLogs1",
"processors": ["batch-processor1"],
"receivers": ["syslog-receiver1"],
"type": azure_native.monitor.PipelineType.LOGS,
}],
},
},
resource_group_name="myResourceGroup",
tags={
"tag1": "A",
"tag2": "B",
})
package main
import (
monitor "github.com/pulumi/pulumi-azure-native-sdk/monitor/v3"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
_, err := monitor.NewPipelineGroup(ctx, "pipelineGroup", &monitor.PipelineGroupArgs{
ExtendedLocation: &monitor.AzureResourceManagerCommonTypesExtendedLocationArgs{
Name: pulumi.String("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/myResourceGroup/providers/microsoft.extendedlocation/customlocations/myTestCustomLocation"),
Type: pulumi.String(monitor.ExtendedLocationTypeCustomLocation),
},
Location: pulumi.String("eastus2"),
PipelineGroupName: pulumi.String("plGroup1"),
Properties: &monitor.PipelineGroupPropertiesArgs{
Exporters: monitor.ExporterArray{
&monitor.ExporterArgs{
AzureMonitorWorkspaceLogs: &monitor.AzureMonitorWorkspaceLogsExporterArgs{
Api: &monitor.AzureMonitorWorkspaceLogsApiConfigArgs{
DataCollectionEndpointUrl: pulumi.String("https://logs-myingestion-eb0s.eastus-1.ingest.monitor.azure.com"),
DataCollectionRule: pulumi.String("dcr-00000000000000000000000000000000"),
Schema: &monitor.SchemaMapArgs{
RecordMap: monitor.RecordMapArray{
&monitor.RecordMapArgs{
From: pulumi.String("body"),
To: pulumi.String("Body"),
},
&monitor.RecordMapArgs{
From: pulumi.String("severity_text"),
To: pulumi.String("SeverityText"),
},
&monitor.RecordMapArgs{
From: pulumi.String("time_unix_nano"),
To: pulumi.String("TimeGenerated"),
},
},
},
Stream: pulumi.String("Custom-MyTableRawData_CL"),
},
Cache: &monitor.CacheConfigurationArgs{
MaxStorageUsage: pulumi.Int(100),
RetentionPeriod: pulumi.Int(10),
},
Concurrency: &monitor.ConcurrencyConfigurationArgs{
BatchQueueSize: pulumi.Int(100),
WorkerCount: pulumi.Int(4),
},
},
Name: pulumi.String("my-workspace-logs-exporter1"),
Type: pulumi.String(monitor.ExporterTypeAzureMonitorWorkspaceLogs),
},
},
Processors: monitor.ProcessorArray{
&monitor.ProcessorArgs{
Name: pulumi.String("batch-processor1"),
Type: pulumi.String(monitor.ProcessorTypeBatch),
},
},
Receivers: monitor.ReceiverArray{
&monitor.ReceiverArgs{
Name: pulumi.String("syslog-receiver1"),
Syslog: &monitor.SyslogReceiverArgs{
Endpoint: pulumi.String("0.0.0.0:514"),
},
Type: pulumi.String(monitor.ReceiverTypeSyslog),
},
},
Service: &monitor.ServiceArgs{
Pipelines: monitor.PipelineArray{
&monitor.PipelineArgs{
Exporters: pulumi.StringArray{
pulumi.String("my-workspace-logs-exporter1"),
},
Name: pulumi.String("MyPipelineForLogs1"),
Processors: pulumi.StringArray{
pulumi.String("batch-processor1"),
},
Receivers: pulumi.StringArray{
pulumi.String("syslog-receiver1"),
},
Type: pulumi.String(monitor.PipelineTypeLogs),
},
},
},
},
ResourceGroupName: pulumi.String("myResourceGroup"),
Tags: pulumi.StringMap{
"tag1": pulumi.String("A"),
"tag2": pulumi.String("B"),
},
})
if err != nil {
return err
}
return nil
})
}
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using AzureNative = Pulumi.AzureNative;
return await Deployment.RunAsync(() =>
{
var pipelineGroup = new AzureNative.Monitor.PipelineGroup("pipelineGroup", new()
{
ExtendedLocation = new AzureNative.Monitor.Inputs.AzureResourceManagerCommonTypesExtendedLocationArgs
{
Name = "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/myResourceGroup/providers/microsoft.extendedlocation/customlocations/myTestCustomLocation",
Type = AzureNative.Monitor.ExtendedLocationType.CustomLocation,
},
Location = "eastus2",
PipelineGroupName = "plGroup1",
Properties = new AzureNative.Monitor.Inputs.PipelineGroupPropertiesArgs
{
Exporters = new[]
{
new AzureNative.Monitor.Inputs.ExporterArgs
{
AzureMonitorWorkspaceLogs = new AzureNative.Monitor.Inputs.AzureMonitorWorkspaceLogsExporterArgs
{
Api = new AzureNative.Monitor.Inputs.AzureMonitorWorkspaceLogsApiConfigArgs
{
DataCollectionEndpointUrl = "https://logs-myingestion-eb0s.eastus-1.ingest.monitor.azure.com",
DataCollectionRule = "dcr-00000000000000000000000000000000",
Schema = new AzureNative.Monitor.Inputs.SchemaMapArgs
{
RecordMap = new[]
{
new AzureNative.Monitor.Inputs.RecordMapArgs
{
From = "body",
To = "Body",
},
new AzureNative.Monitor.Inputs.RecordMapArgs
{
From = "severity_text",
To = "SeverityText",
},
new AzureNative.Monitor.Inputs.RecordMapArgs
{
From = "time_unix_nano",
To = "TimeGenerated",
},
},
},
Stream = "Custom-MyTableRawData_CL",
},
Cache = new AzureNative.Monitor.Inputs.CacheConfigurationArgs
{
MaxStorageUsage = 100,
RetentionPeriod = 10,
},
Concurrency = new AzureNative.Monitor.Inputs.ConcurrencyConfigurationArgs
{
BatchQueueSize = 100,
WorkerCount = 4,
},
},
Name = "my-workspace-logs-exporter1",
Type = AzureNative.Monitor.ExporterType.AzureMonitorWorkspaceLogs,
},
},
Processors = new[]
{
new AzureNative.Monitor.Inputs.ProcessorArgs
{
Name = "batch-processor1",
Type = AzureNative.Monitor.ProcessorType.Batch,
},
},
Receivers = new[]
{
new AzureNative.Monitor.Inputs.ReceiverArgs
{
Name = "syslog-receiver1",
Syslog = new AzureNative.Monitor.Inputs.SyslogReceiverArgs
{
Endpoint = "0.0.0.0:514",
},
Type = AzureNative.Monitor.ReceiverType.Syslog,
},
},
Service = new AzureNative.Monitor.Inputs.ServiceArgs
{
Pipelines = new[]
{
new AzureNative.Monitor.Inputs.PipelineArgs
{
Exporters = new[]
{
"my-workspace-logs-exporter1",
},
Name = "MyPipelineForLogs1",
Processors = new[]
{
"batch-processor1",
},
Receivers = new[]
{
"syslog-receiver1",
},
Type = AzureNative.Monitor.PipelineType.Logs,
},
},
},
},
ResourceGroupName = "myResourceGroup",
Tags =
{
{ "tag1", "A" },
{ "tag2", "B" },
},
});
});
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.azurenative.monitor.PipelineGroup;
import com.pulumi.azurenative.monitor.PipelineGroupArgs;
import com.pulumi.azurenative.monitor.inputs.AzureResourceManagerCommonTypesExtendedLocationArgs;
import com.pulumi.azurenative.monitor.inputs.PipelineGroupPropertiesArgs;
import com.pulumi.azurenative.monitor.inputs.ServiceArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var pipelineGroup = new PipelineGroup("pipelineGroup", PipelineGroupArgs.builder()
.extendedLocation(AzureResourceManagerCommonTypesExtendedLocationArgs.builder()
.name("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/myResourceGroup/providers/microsoft.extendedlocation/customlocations/myTestCustomLocation")
.type("CustomLocation")
.build())
.location("eastus2")
.pipelineGroupName("plGroup1")
.properties(PipelineGroupPropertiesArgs.builder()
.exporters(ExporterArgs.builder()
.azureMonitorWorkspaceLogs(AzureMonitorWorkspaceLogsExporterArgs.builder()
.api(AzureMonitorWorkspaceLogsApiConfigArgs.builder()
.dataCollectionEndpointUrl("https://logs-myingestion-eb0s.eastus-1.ingest.monitor.azure.com")
.dataCollectionRule("dcr-00000000000000000000000000000000")
.schema(SchemaMapArgs.builder()
.recordMap(
RecordMapArgs.builder()
.from("body")
.to("Body")
.build(),
RecordMapArgs.builder()
.from("severity_text")
.to("SeverityText")
.build(),
RecordMapArgs.builder()
.from("time_unix_nano")
.to("TimeGenerated")
.build())
.build())
.stream("Custom-MyTableRawData_CL")
.build())
.cache(CacheConfigurationArgs.builder()
.maxStorageUsage(100)
.retentionPeriod(10)
.build())
.concurrency(ConcurrencyConfigurationArgs.builder()
.batchQueueSize(100)
.workerCount(4)
.build())
.build())
.name("my-workspace-logs-exporter1")
.type("AzureMonitorWorkspaceLogs")
.build())
.processors(ProcessorArgs.builder()
.name("batch-processor1")
.type("Batch")
.build())
.receivers(ReceiverArgs.builder()
.name("syslog-receiver1")
.syslog(SyslogReceiverArgs.builder()
.endpoint("0.0.0.0:514")
.build())
.type("Syslog")
.build())
.service(ServiceArgs.builder()
.pipelines(PipelineArgs.builder()
.exporters("my-workspace-logs-exporter1")
.name("MyPipelineForLogs1")
.processors("batch-processor1")
.receivers("syslog-receiver1")
.type("Logs")
.build())
.build())
.build())
.resourceGroupName("myResourceGroup")
.tags(Map.ofEntries(
Map.entry("tag1", "A"),
Map.entry("tag2", "B")
))
.build());
}
}
resources:
pipelineGroup:
type: azure-native:monitor:PipelineGroup
properties:
extendedLocation:
name: /subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/myResourceGroup/providers/microsoft.extendedlocation/customlocations/myTestCustomLocation
type: CustomLocation
location: eastus2
pipelineGroupName: plGroup1
properties:
exporters:
- azureMonitorWorkspaceLogs:
api:
dataCollectionEndpointUrl: https://logs-myingestion-eb0s.eastus-1.ingest.monitor.azure.com
dataCollectionRule: dcr-00000000000000000000000000000000
schema:
recordMap:
- from: body
to: Body
- from: severity_text
to: SeverityText
- from: time_unix_nano
to: TimeGenerated
stream: Custom-MyTableRawData_CL
cache:
maxStorageUsage: 100
retentionPeriod: 10
concurrency:
batchQueueSize: 100
workerCount: 4
name: my-workspace-logs-exporter1
type: AzureMonitorWorkspaceLogs
processors:
- name: batch-processor1
type: Batch
receivers:
- name: syslog-receiver1
syslog:
endpoint: 0.0.0.0:514
type: Syslog
service:
pipelines:
- exporters:
- my-workspace-logs-exporter1
name: MyPipelineForLogs1
processors:
- batch-processor1
receivers:
- syslog-receiver1
type: Logs
resourceGroupName: myResourceGroup
tags:
tag1: A
tag2: B
The cache configuration sets maxStorageUsage (in MB) and retentionPeriod (in minutes) for local storage. When the exporter cannot reach Azure Monitor, logs accumulate in the cache up to the storage limit. Once connectivity returns, cached logs are sent to the workspace. This prevents data loss during network interruptions.
Expose syslog receiver through load balancer
Production deployments often need external access to receivers through a load balancer with custom DNS, allowing centralized log collection from distributed sources.
import * as pulumi from "@pulumi/pulumi";
import * as azure_native from "@pulumi/azure-native";
const pipelineGroup = new azure_native.monitor.PipelineGroup("pipelineGroup", {
extendedLocation: {
name: "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/myResourceGroup/providers/microsoft.extendedlocation/customlocations/myTestCustomLocation",
type: azure_native.monitor.ExtendedLocationType.CustomLocation,
},
location: "eastus2",
pipelineGroupName: "plGroup1",
properties: {
exporters: [{
azureMonitorWorkspaceLogs: {
api: {
dataCollectionEndpointUrl: "https://logs-myingestion-eb0s.eastus-1.ingest.monitor.azure.com",
dataCollectionRule: "dcr-00000000000000000000000000000000",
schema: {
recordMap: [
{
from: "body",
to: "Body",
},
{
from: "severity_text",
to: "SeverityText",
},
{
from: "time_unix_nano",
to: "TimeGenerated",
},
],
},
stream: "Custom-MyTableRawData_CL",
},
concurrency: {
batchQueueSize: 100,
workerCount: 4,
},
},
name: "my-workspace-logs-exporter1",
type: azure_native.monitor.ExporterType.AzureMonitorWorkspaceLogs,
}],
networkingConfigurations: [{
externalNetworkingMode: azure_native.monitor.ExternalNetworkingMode.LoadBalancerOnly,
host: "azuremonitorpipeline.contoso.com",
routes: [{
receiver: "syslog-receiver1",
}],
}],
processors: [],
receivers: [{
name: "syslog-receiver1",
syslog: {
endpoint: "0.0.0.0:514",
},
type: azure_native.monitor.ReceiverType.Syslog,
}],
service: {
pipelines: [{
exporters: ["my-workspace-logs-exporter1"],
name: "MyPipelineForLogs1",
processors: [],
receivers: ["syslog-receiver1"],
type: azure_native.monitor.PipelineType.Logs,
}],
},
},
resourceGroupName: "myResourceGroup",
tags: {
tag1: "A",
tag2: "B",
},
});
import pulumi
import pulumi_azure_native as azure_native
pipeline_group = azure_native.monitor.PipelineGroup("pipelineGroup",
extended_location={
"name": "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/myResourceGroup/providers/microsoft.extendedlocation/customlocations/myTestCustomLocation",
"type": azure_native.monitor.ExtendedLocationType.CUSTOM_LOCATION,
},
location="eastus2",
pipeline_group_name="plGroup1",
properties={
"exporters": [{
"azure_monitor_workspace_logs": {
"api": {
"data_collection_endpoint_url": "https://logs-myingestion-eb0s.eastus-1.ingest.monitor.azure.com",
"data_collection_rule": "dcr-00000000000000000000000000000000",
"schema": {
"record_map": [
{
"from_": "body",
"to": "Body",
},
{
"from_": "severity_text",
"to": "SeverityText",
},
{
"from_": "time_unix_nano",
"to": "TimeGenerated",
},
],
},
"stream": "Custom-MyTableRawData_CL",
},
"concurrency": {
"batch_queue_size": 100,
"worker_count": 4,
},
},
"name": "my-workspace-logs-exporter1",
"type": azure_native.monitor.ExporterType.AZURE_MONITOR_WORKSPACE_LOGS,
}],
"networking_configurations": [{
"external_networking_mode": azure_native.monitor.ExternalNetworkingMode.LOAD_BALANCER_ONLY,
"host": "azuremonitorpipeline.contoso.com",
"routes": [{
"receiver": "syslog-receiver1",
}],
}],
"processors": [],
"receivers": [{
"name": "syslog-receiver1",
"syslog": {
"endpoint": "0.0.0.0:514",
},
"type": azure_native.monitor.ReceiverType.SYSLOG,
}],
"service": {
"pipelines": [{
"exporters": ["my-workspace-logs-exporter1"],
"name": "MyPipelineForLogs1",
"processors": [],
"receivers": ["syslog-receiver1"],
"type": azure_native.monitor.PipelineType.LOGS,
}],
},
},
resource_group_name="myResourceGroup",
tags={
"tag1": "A",
"tag2": "B",
})
package main
import (
monitor "github.com/pulumi/pulumi-azure-native-sdk/monitor/v3"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
_, err := monitor.NewPipelineGroup(ctx, "pipelineGroup", &monitor.PipelineGroupArgs{
ExtendedLocation: &monitor.AzureResourceManagerCommonTypesExtendedLocationArgs{
Name: pulumi.String("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/myResourceGroup/providers/microsoft.extendedlocation/customlocations/myTestCustomLocation"),
Type: pulumi.String(monitor.ExtendedLocationTypeCustomLocation),
},
Location: pulumi.String("eastus2"),
PipelineGroupName: pulumi.String("plGroup1"),
Properties: &monitor.PipelineGroupPropertiesArgs{
Exporters: monitor.ExporterArray{
&monitor.ExporterArgs{
AzureMonitorWorkspaceLogs: &monitor.AzureMonitorWorkspaceLogsExporterArgs{
Api: &monitor.AzureMonitorWorkspaceLogsApiConfigArgs{
DataCollectionEndpointUrl: pulumi.String("https://logs-myingestion-eb0s.eastus-1.ingest.monitor.azure.com"),
DataCollectionRule: pulumi.String("dcr-00000000000000000000000000000000"),
Schema: &monitor.SchemaMapArgs{
RecordMap: monitor.RecordMapArray{
&monitor.RecordMapArgs{
From: pulumi.String("body"),
To: pulumi.String("Body"),
},
&monitor.RecordMapArgs{
From: pulumi.String("severity_text"),
To: pulumi.String("SeverityText"),
},
&monitor.RecordMapArgs{
From: pulumi.String("time_unix_nano"),
To: pulumi.String("TimeGenerated"),
},
},
},
Stream: pulumi.String("Custom-MyTableRawData_CL"),
},
Concurrency: &monitor.ConcurrencyConfigurationArgs{
BatchQueueSize: pulumi.Int(100),
WorkerCount: pulumi.Int(4),
},
},
Name: pulumi.String("my-workspace-logs-exporter1"),
Type: pulumi.String(monitor.ExporterTypeAzureMonitorWorkspaceLogs),
},
},
NetworkingConfigurations: monitor.NetworkingConfigurationArray{
&monitor.NetworkingConfigurationArgs{
ExternalNetworkingMode: pulumi.String(monitor.ExternalNetworkingModeLoadBalancerOnly),
Host: pulumi.String("azuremonitorpipeline.contoso.com"),
Routes: monitor.NetworkingRouteArray{
&monitor.NetworkingRouteArgs{
Receiver: pulumi.String("syslog-receiver1"),
},
},
},
},
Processors: monitor.ProcessorArray{},
Receivers: monitor.ReceiverArray{
&monitor.ReceiverArgs{
Name: pulumi.String("syslog-receiver1"),
Syslog: &monitor.SyslogReceiverArgs{
Endpoint: pulumi.String("0.0.0.0:514"),
},
Type: pulumi.String(monitor.ReceiverTypeSyslog),
},
},
Service: &monitor.ServiceArgs{
Pipelines: monitor.PipelineArray{
&monitor.PipelineArgs{
Exporters: pulumi.StringArray{
pulumi.String("my-workspace-logs-exporter1"),
},
Name: pulumi.String("MyPipelineForLogs1"),
Processors: pulumi.StringArray{},
Receivers: pulumi.StringArray{
pulumi.String("syslog-receiver1"),
},
Type: pulumi.String(monitor.PipelineTypeLogs),
},
},
},
},
ResourceGroupName: pulumi.String("myResourceGroup"),
Tags: pulumi.StringMap{
"tag1": pulumi.String("A"),
"tag2": pulumi.String("B"),
},
})
if err != nil {
return err
}
return nil
})
}
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using AzureNative = Pulumi.AzureNative;
return await Deployment.RunAsync(() =>
{
var pipelineGroup = new AzureNative.Monitor.PipelineGroup("pipelineGroup", new()
{
ExtendedLocation = new AzureNative.Monitor.Inputs.AzureResourceManagerCommonTypesExtendedLocationArgs
{
Name = "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/myResourceGroup/providers/microsoft.extendedlocation/customlocations/myTestCustomLocation",
Type = AzureNative.Monitor.ExtendedLocationType.CustomLocation,
},
Location = "eastus2",
PipelineGroupName = "plGroup1",
Properties = new AzureNative.Monitor.Inputs.PipelineGroupPropertiesArgs
{
Exporters = new[]
{
new AzureNative.Monitor.Inputs.ExporterArgs
{
AzureMonitorWorkspaceLogs = new AzureNative.Monitor.Inputs.AzureMonitorWorkspaceLogsExporterArgs
{
Api = new AzureNative.Monitor.Inputs.AzureMonitorWorkspaceLogsApiConfigArgs
{
DataCollectionEndpointUrl = "https://logs-myingestion-eb0s.eastus-1.ingest.monitor.azure.com",
DataCollectionRule = "dcr-00000000000000000000000000000000",
Schema = new AzureNative.Monitor.Inputs.SchemaMapArgs
{
RecordMap = new[]
{
new AzureNative.Monitor.Inputs.RecordMapArgs
{
From = "body",
To = "Body",
},
new AzureNative.Monitor.Inputs.RecordMapArgs
{
From = "severity_text",
To = "SeverityText",
},
new AzureNative.Monitor.Inputs.RecordMapArgs
{
From = "time_unix_nano",
To = "TimeGenerated",
},
},
},
Stream = "Custom-MyTableRawData_CL",
},
Concurrency = new AzureNative.Monitor.Inputs.ConcurrencyConfigurationArgs
{
BatchQueueSize = 100,
WorkerCount = 4,
},
},
Name = "my-workspace-logs-exporter1",
Type = AzureNative.Monitor.ExporterType.AzureMonitorWorkspaceLogs,
},
},
NetworkingConfigurations = new[]
{
new AzureNative.Monitor.Inputs.NetworkingConfigurationArgs
{
ExternalNetworkingMode = AzureNative.Monitor.ExternalNetworkingMode.LoadBalancerOnly,
Host = "azuremonitorpipeline.contoso.com",
Routes = new[]
{
new AzureNative.Monitor.Inputs.NetworkingRouteArgs
{
Receiver = "syslog-receiver1",
},
},
},
},
Processors = new() { },
Receivers = new[]
{
new AzureNative.Monitor.Inputs.ReceiverArgs
{
Name = "syslog-receiver1",
Syslog = new AzureNative.Monitor.Inputs.SyslogReceiverArgs
{
Endpoint = "0.0.0.0:514",
},
Type = AzureNative.Monitor.ReceiverType.Syslog,
},
},
Service = new AzureNative.Monitor.Inputs.ServiceArgs
{
Pipelines = new[]
{
new AzureNative.Monitor.Inputs.PipelineArgs
{
Exporters = new[]
{
"my-workspace-logs-exporter1",
},
Name = "MyPipelineForLogs1",
Processors = new() { },
Receivers = new[]
{
"syslog-receiver1",
},
Type = AzureNative.Monitor.PipelineType.Logs,
},
},
},
},
ResourceGroupName = "myResourceGroup",
Tags =
{
{ "tag1", "A" },
{ "tag2", "B" },
},
});
});
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.azurenative.monitor.PipelineGroup;
import com.pulumi.azurenative.monitor.PipelineGroupArgs;
import com.pulumi.azurenative.monitor.inputs.AzureResourceManagerCommonTypesExtendedLocationArgs;
import com.pulumi.azurenative.monitor.inputs.PipelineGroupPropertiesArgs;
import com.pulumi.azurenative.monitor.inputs.ServiceArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var pipelineGroup = new PipelineGroup("pipelineGroup", PipelineGroupArgs.builder()
.extendedLocation(AzureResourceManagerCommonTypesExtendedLocationArgs.builder()
.name("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/myResourceGroup/providers/microsoft.extendedlocation/customlocations/myTestCustomLocation")
.type("CustomLocation")
.build())
.location("eastus2")
.pipelineGroupName("plGroup1")
.properties(PipelineGroupPropertiesArgs.builder()
.exporters(ExporterArgs.builder()
.azureMonitorWorkspaceLogs(AzureMonitorWorkspaceLogsExporterArgs.builder()
.api(AzureMonitorWorkspaceLogsApiConfigArgs.builder()
.dataCollectionEndpointUrl("https://logs-myingestion-eb0s.eastus-1.ingest.monitor.azure.com")
.dataCollectionRule("dcr-00000000000000000000000000000000")
.schema(SchemaMapArgs.builder()
.recordMap(
RecordMapArgs.builder()
.from("body")
.to("Body")
.build(),
RecordMapArgs.builder()
.from("severity_text")
.to("SeverityText")
.build(),
RecordMapArgs.builder()
.from("time_unix_nano")
.to("TimeGenerated")
.build())
.build())
.stream("Custom-MyTableRawData_CL")
.build())
.concurrency(ConcurrencyConfigurationArgs.builder()
.batchQueueSize(100)
.workerCount(4)
.build())
.build())
.name("my-workspace-logs-exporter1")
.type("AzureMonitorWorkspaceLogs")
.build())
.networkingConfigurations(NetworkingConfigurationArgs.builder()
.externalNetworkingMode("LoadBalancerOnly")
.host("azuremonitorpipeline.contoso.com")
.routes(NetworkingRouteArgs.builder()
.receiver("syslog-receiver1")
.build())
.build())
.processors()
.receivers(ReceiverArgs.builder()
.name("syslog-receiver1")
.syslog(SyslogReceiverArgs.builder()
.endpoint("0.0.0.0:514")
.build())
.type("Syslog")
.build())
.service(ServiceArgs.builder()
.pipelines(PipelineArgs.builder()
.exporters("my-workspace-logs-exporter1")
.name("MyPipelineForLogs1")
.processors()
.receivers("syslog-receiver1")
.type("Logs")
.build())
.build())
.build())
.resourceGroupName("myResourceGroup")
.tags(Map.ofEntries(
Map.entry("tag1", "A"),
Map.entry("tag2", "B")
))
.build());
}
}
resources:
pipelineGroup:
type: azure-native:monitor:PipelineGroup
properties:
extendedLocation:
name: /subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/myResourceGroup/providers/microsoft.extendedlocation/customlocations/myTestCustomLocation
type: CustomLocation
location: eastus2
pipelineGroupName: plGroup1
properties:
exporters:
- azureMonitorWorkspaceLogs:
api:
dataCollectionEndpointUrl: https://logs-myingestion-eb0s.eastus-1.ingest.monitor.azure.com
dataCollectionRule: dcr-00000000000000000000000000000000
schema:
recordMap:
- from: body
to: Body
- from: severity_text
to: SeverityText
- from: time_unix_nano
to: TimeGenerated
stream: Custom-MyTableRawData_CL
concurrency:
batchQueueSize: 100
workerCount: 4
name: my-workspace-logs-exporter1
type: AzureMonitorWorkspaceLogs
networkingConfigurations:
- externalNetworkingMode: LoadBalancerOnly
host: azuremonitorpipeline.contoso.com
routes:
- receiver: syslog-receiver1
processors: []
receivers:
- name: syslog-receiver1
syslog:
endpoint: 0.0.0.0:514
type: Syslog
service:
pipelines:
- exporters:
- my-workspace-logs-exporter1
name: MyPipelineForLogs1
processors: []
receivers:
- syslog-receiver1
type: Logs
resourceGroupName: myResourceGroup
tags:
tag1: A
tag2: B
The networkingConfigurations section sets externalNetworkingMode to LoadBalancerOnly, exposing the receiver through a load balancer. The host property defines the DNS name clients use to reach the receiver. The routes array maps the receiver to the load balancer endpoint, enabling external syslog sources to send logs to the pipeline.
Beyond these examples
These snippets focus on specific pipeline group features: UDP and syslog receivers, JSON array mapping and batch processing, and local caching and load balancer networking. They’re intentionally minimal rather than full observability solutions.
The examples reference pre-existing infrastructure such as Azure Arc custom locations, Data Collection Rules and endpoints, and Azure Monitor Workspace Logs tables. They focus on configuring the pipeline rather than provisioning the surrounding infrastructure.
To keep things focused, common pipeline patterns are omitted, including:
- OTLP and other receiver types
- Advanced processor configurations (filtering, transformation)
- Multiple exporters or multi-pipeline routing
- Authentication and TLS configuration
- Metrics and traces pipelines (only logs shown)
These omissions are intentional: the goal is to illustrate how each pipeline feature is wired, not provide drop-in monitoring modules. See the PipelineGroup resource reference for all available configuration options.
Let's configure Azure Monitor Pipeline Groups
Get started with Pulumi Cloud, then follow our quick setup guide to deploy this infrastructure.
Try Pulumi Cloud for FREEFrequently Asked Questions
Configuration & Setup
What is extendedLocation and why is it required?
extendedLocation specifies the Azure custom location where the pipeline group runs. It requires a name (full resource path to the custom location) and type set to CustomLocation.What properties can't I change after creating a pipeline group?
Four properties are immutable:
location, extendedLocation, pipelineGroupName, and resourceGroupName. Changing any of these requires recreating the resource.Is this resource production-ready?
No, all available API versions (2024-10-01-preview, 2023-10-01-preview, 2025-03-01-preview) are in preview and may have breaking changes.
Receivers & Data Ingestion
What receiver types are available?
Two receiver types are shown in the examples:
- UDP - Requires
endpoint(e.g., “0.0.0.0:518”) andencoding(e.g., “utf-8”), with optionaljsonArrayMapper - Syslog - Requires
endpoint(e.g., “0.0.0.0:514”)
What does jsonArrayMapper do in UDP receivers?
It maps JSON array fields to OpenTelemetry attributes. You specify a
sourceField (field name), keys (array of key names), and destinationField (where to map them, like “attributes”).How do I connect receivers to exporters?
Use
service.pipelines to reference component names. Each pipeline specifies receivers (array of receiver names), optional processors (array of processor names), and exporters (array of exporter names).Data Processing & Export
How do I map incoming fields to Azure Monitor schema?
Configure
schema.recordMap in the exporter’s api section. Each entry maps a source field (from, like “body”) to a destination field (to, like “Body”).What's required to export to Azure Monitor Workspace Logs?
The
azureMonitorWorkspaceLogs exporter requires four api settings: dataCollectionEndpointUrl, dataCollectionRule (DCR ID), schema.recordMap, and stream (custom table name like “Custom-MyTableRawData_CL”).Are processors required in a pipeline?
No, processors are optional. Examples 1-2 use empty processor arrays, while examples 3-4 include a batch processor for performance optimization.
Performance & Optimization
How do concurrency settings affect performance?
The
concurrency configuration in exporters controls throughput. batchQueueSize sets the queue size (100 in examples) and workerCount sets parallel workers (4 in examples).What is the cache configuration for?
The optional
cache configuration in exporters provides persistence during outages. maxStorageUsage sets storage limit and retentionPeriod sets how long to retain cached data.Networking
How do I configure external networking for my pipeline group?
Use
networkingConfigurations to set externalNetworkingMode (like “LoadBalancerOnly”), host (custom domain), and routes (array mapping receivers to network paths).