azure.datafactory.DataFlow

using Pulumi;
using Azure = Pulumi.Azure;

class MyStack : Stack
{
    public MyStack()
    {
        var exampleResourceGroup = new Azure.Core.ResourceGroup("exampleResourceGroup", new Azure.Core.ResourceGroupArgs
        {
            Location = "West Europe",
        });
        var exampleAccount = new Azure.Storage.Account("exampleAccount", new Azure.Storage.AccountArgs
        {
            Location = exampleResourceGroup.Location,
            ResourceGroupName = exampleResourceGroup.Name,
            AccountTier = "Standard",
            AccountReplicationType = "LRS",
        });
        var exampleFactory = new Azure.DataFactory.Factory("exampleFactory", new Azure.DataFactory.FactoryArgs
        {
            Location = exampleResourceGroup.Location,
            ResourceGroupName = exampleResourceGroup.Name,
        });
        var exampleLinkedCustomService = new Azure.DataFactory.LinkedCustomService("exampleLinkedCustomService", new Azure.DataFactory.LinkedCustomServiceArgs
        {
            DataFactoryId = exampleFactory.Id,
            Type = "AzureBlobStorage",
            TypePropertiesJson = exampleAccount.PrimaryConnectionString.Apply(primaryConnectionString => @$"{{
  ""connectionString"": ""{primaryConnectionString}""
}}
"),
        });
        var example1 = new Azure.DataFactory.DatasetJson("example1", new Azure.DataFactory.DatasetJsonArgs
        {
            ResourceGroupName = exampleResourceGroup.Name,
            DataFactoryName = exampleFactory.Name,
            LinkedServiceName = exampleLinkedCustomService.Name,
            AzureBlobStorageLocation = new Azure.DataFactory.Inputs.DatasetJsonAzureBlobStorageLocationArgs
            {
                Container = "container",
                Path = "foo/bar/",
                Filename = "foo.txt",
            },
            Encoding = "UTF-8",
        });
        var example2 = new Azure.DataFactory.DatasetJson("example2", new Azure.DataFactory.DatasetJsonArgs
        {
            ResourceGroupName = exampleResourceGroup.Name,
            DataFactoryName = exampleFactory.Name,
            LinkedServiceName = exampleLinkedCustomService.Name,
            AzureBlobStorageLocation = new Azure.DataFactory.Inputs.DatasetJsonAzureBlobStorageLocationArgs
            {
                Container = "container",
                Path = "foo/bar/",
                Filename = "bar.txt",
            },
            Encoding = "UTF-8",
        });
        var exampleDataFlow = new Azure.DataFactory.DataFlow("exampleDataFlow", new Azure.DataFactory.DataFlowArgs
        {
            DataFactoryId = exampleFactory.Id,
            Sources = 
            {
                new Azure.DataFactory.Inputs.DataFlowSourceArgs
                {
                    Name = "source1",
                    Dataset = new Azure.DataFactory.Inputs.DataFlowSourceDatasetArgs
                    {
                        Name = example1.Name,
                    },
                },
            },
            Sinks = 
            {
                new Azure.DataFactory.Inputs.DataFlowSinkArgs
                {
                    Name = "sink1",
                    Dataset = new Azure.DataFactory.Inputs.DataFlowSinkDatasetArgs
                    {
                        Name = example2.Name,
                    },
                },
            },
            Script = @"source(
  allowSchemaDrift: true, 
  validateSchema: false, 
  limit: 100, 
  ignoreNoFilesFound: false, 
  documentForm: 'documentPerLine') ~> source1 
source1 sink(
  allowSchemaDrift: true, 
  validateSchema: false, 
  skipDuplicateMapInputs: true, 
  skipDuplicateMapOutputs: true) ~> sink1
",
        });
    }

}

package main

import (
	"fmt"

	"github.com/pulumi/pulumi-azure/sdk/v4/go/azure/core"
	"github.com/pulumi/pulumi-azure/sdk/v4/go/azure/datafactory"
	"github.com/pulumi/pulumi-azure/sdk/v4/go/azure/storage"
	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)

func main() {
	pulumi.Run(func(ctx *pulumi.Context) error {
		exampleResourceGroup, err := core.NewResourceGroup(ctx, "exampleResourceGroup", &core.ResourceGroupArgs{
			Location: pulumi.String("West Europe"),
		})
		if err != nil {
			return err
		}
		exampleAccount, err := storage.NewAccount(ctx, "exampleAccount", &storage.AccountArgs{
			Location:               exampleResourceGroup.Location,
			ResourceGroupName:      exampleResourceGroup.Name,
			AccountTier:            pulumi.String("Standard"),
			AccountReplicationType: pulumi.String("LRS"),
		})
		if err != nil {
			return err
		}
		exampleFactory, err := datafactory.NewFactory(ctx, "exampleFactory", &datafactory.FactoryArgs{
			Location:          exampleResourceGroup.Location,
			ResourceGroupName: exampleResourceGroup.Name,
		})
		if err != nil {
			return err
		}
		exampleLinkedCustomService, err := datafactory.NewLinkedCustomService(ctx, "exampleLinkedCustomService", &datafactory.LinkedCustomServiceArgs{
			DataFactoryId: exampleFactory.ID(),
			Type:          pulumi.String("AzureBlobStorage"),
			TypePropertiesJson: exampleAccount.PrimaryConnectionString.ApplyT(func(primaryConnectionString string) (string, error) {
				return fmt.Sprintf("%v%v%v%v%v", "{\n", "  \"connectionString\": \"", primaryConnectionString, "\"\n", "}\n"), nil
			}).(pulumi.StringOutput),
		})
		if err != nil {
			return err
		}
		example1, err := datafactory.NewDatasetJson(ctx, "example1", &datafactory.DatasetJsonArgs{
			ResourceGroupName: exampleResourceGroup.Name,
			DataFactoryName:   exampleFactory.Name,
			LinkedServiceName: exampleLinkedCustomService.Name,
			AzureBlobStorageLocation: &datafactory.DatasetJsonAzureBlobStorageLocationArgs{
				Container: pulumi.String("container"),
				Path:      pulumi.String("foo/bar/"),
				Filename:  pulumi.String("foo.txt"),
			},
			Encoding: pulumi.String("UTF-8"),
		})
		if err != nil {
			return err
		}
		example2, err := datafactory.NewDatasetJson(ctx, "example2", &datafactory.DatasetJsonArgs{
			ResourceGroupName: exampleResourceGroup.Name,
			DataFactoryName:   exampleFactory.Name,
			LinkedServiceName: exampleLinkedCustomService.Name,
			AzureBlobStorageLocation: &datafactory.DatasetJsonAzureBlobStorageLocationArgs{
				Container: pulumi.String("container"),
				Path:      pulumi.String("foo/bar/"),
				Filename:  pulumi.String("bar.txt"),
			},
			Encoding: pulumi.String("UTF-8"),
		})
		if err != nil {
			return err
		}
		_, err = datafactory.NewDataFlow(ctx, "exampleDataFlow", &datafactory.DataFlowArgs{
			DataFactoryId: exampleFactory.ID(),
			Sources: datafactory.DataFlowSourceArray{
				&datafactory.DataFlowSourceArgs{
					Name: pulumi.String("source1"),
					Dataset: &datafactory.DataFlowSourceDatasetArgs{
						Name: example1.Name,
					},
				},
			},
			Sinks: datafactory.DataFlowSinkArray{
				&datafactory.DataFlowSinkArgs{
					Name: pulumi.String("sink1"),
					Dataset: &datafactory.DataFlowSinkDatasetArgs{
						Name: example2.Name,
					},
				},
			},
			Script: pulumi.String(fmt.Sprintf("%v%v%v%v%v%v%v%v%v%v%v", "source(\n", "  allowSchemaDrift: true, \n", "  validateSchema: false, \n", "  limit: 100, \n", "  ignoreNoFilesFound: false, \n", "  documentForm: 'documentPerLine') ~> source1 \n", "source1 sink(\n", "  allowSchemaDrift: true, \n", "  validateSchema: false, \n", "  skipDuplicateMapInputs: true, \n", "  skipDuplicateMapOutputs: true) ~> sink1\n")),
		})
		if err != nil {
			return err
		}
		return nil
	})
}

import * as pulumi from "@pulumi/pulumi";
import * as azure from "@pulumi/azure";

const exampleResourceGroup = new azure.core.ResourceGroup("exampleResourceGroup", {location: "West Europe"});
const exampleAccount = new azure.storage.Account("exampleAccount", {
    location: exampleResourceGroup.location,
    resourceGroupName: exampleResourceGroup.name,
    accountTier: "Standard",
    accountReplicationType: "LRS",
});
const exampleFactory = new azure.datafactory.Factory("exampleFactory", {
    location: exampleResourceGroup.location,
    resourceGroupName: exampleResourceGroup.name,
});
const exampleLinkedCustomService = new azure.datafactory.LinkedCustomService("exampleLinkedCustomService", {
    dataFactoryId: exampleFactory.id,
    type: "AzureBlobStorage",
    typePropertiesJson: pulumi.interpolate`{
  "connectionString": "${exampleAccount.primaryConnectionString}"
}
`,
});
const example1 = new azure.datafactory.DatasetJson("example1", {
    resourceGroupName: exampleResourceGroup.name,
    dataFactoryName: exampleFactory.name,
    linkedServiceName: exampleLinkedCustomService.name,
    azureBlobStorageLocation: {
        container: "container",
        path: "foo/bar/",
        filename: "foo.txt",
    },
    encoding: "UTF-8",
});
const example2 = new azure.datafactory.DatasetJson("example2", {
    resourceGroupName: exampleResourceGroup.name,
    dataFactoryName: exampleFactory.name,
    linkedServiceName: exampleLinkedCustomService.name,
    azureBlobStorageLocation: {
        container: "container",
        path: "foo/bar/",
        filename: "bar.txt",
    },
    encoding: "UTF-8",
});
const exampleDataFlow = new azure.datafactory.DataFlow("exampleDataFlow", {
    dataFactoryId: exampleFactory.id,
    sources: [{
        name: "source1",
        dataset: {
            name: example1.name,
        },
    }],
    sinks: [{
        name: "sink1",
        dataset: {
            name: example2.name,
        },
    }],
    script: `source(
  allowSchemaDrift: true, 
  validateSchema: false, 
  limit: 100, 
  ignoreNoFilesFound: false, 
  documentForm: 'documentPerLine') ~> source1 
source1 sink(
  allowSchemaDrift: true, 
  validateSchema: false, 
  skipDuplicateMapInputs: true, 
  skipDuplicateMapOutputs: true) ~> sink1
`,
});

import pulumi
import pulumi_azure as azure

example_resource_group = azure.core.ResourceGroup("exampleResourceGroup", location="West Europe")
example_account = azure.storage.Account("exampleAccount",
    location=example_resource_group.location,
    resource_group_name=example_resource_group.name,
    account_tier="Standard",
    account_replication_type="LRS")
example_factory = azure.datafactory.Factory("exampleFactory",
    location=example_resource_group.location,
    resource_group_name=example_resource_group.name)
example_linked_custom_service = azure.datafactory.LinkedCustomService("exampleLinkedCustomService",
    data_factory_id=example_factory.id,
    type="AzureBlobStorage",
    type_properties_json=example_account.primary_connection_string.apply(lambda primary_connection_string: f"""{{
  "connectionString": "{primary_connection_string}"
}}
"""))
example1 = azure.datafactory.DatasetJson("example1",
    resource_group_name=example_resource_group.name,
    data_factory_name=example_factory.name,
    linked_service_name=example_linked_custom_service.name,
    azure_blob_storage_location=azure.datafactory.DatasetJsonAzureBlobStorageLocationArgs(
        container="container",
        path="foo/bar/",
        filename="foo.txt",
    ),
    encoding="UTF-8")
example2 = azure.datafactory.DatasetJson("example2",
    resource_group_name=example_resource_group.name,
    data_factory_name=example_factory.name,
    linked_service_name=example_linked_custom_service.name,
    azure_blob_storage_location=azure.datafactory.DatasetJsonAzureBlobStorageLocationArgs(
        container="container",
        path="foo/bar/",
        filename="bar.txt",
    ),
    encoding="UTF-8")
example_data_flow = azure.datafactory.DataFlow("exampleDataFlow",
    data_factory_id=example_factory.id,
    sources=[azure.datafactory.DataFlowSourceArgs(
        name="source1",
        dataset=azure.datafactory.DataFlowSourceDatasetArgs(
            name=example1.name,
        ),
    )],
    sinks=[azure.datafactory.DataFlowSinkArgs(
        name="sink1",
        dataset=azure.datafactory.DataFlowSinkDatasetArgs(
            name=example2.name,
        ),
    )],
    script="""source(
  allowSchemaDrift: true, 
  validateSchema: false, 
  limit: 100, 
  ignoreNoFilesFound: false, 
  documentForm: 'documentPerLine') ~> source1 
source1 sink(
  allowSchemaDrift: true, 
  validateSchema: false, 
  skipDuplicateMapInputs: true, 
  skipDuplicateMapOutputs: true) ~> sink1
""")