Viewing docs for AWS v5.43.0 (Older version)
published on Tuesday, Mar 10, 2026 by Pulumi
published on Tuesday, Mar 10, 2026 by Pulumi
Viewing docs for AWS v5.43.0 (Older version)
published on Tuesday, Mar 10, 2026 by Pulumi
published on Tuesday, Mar 10, 2026 by Pulumi
Provides a Kinesis Firehose Delivery Stream resource. Amazon Kinesis Firehose is a fully managed, elastic service to easily deliver real-time data streams to destinations such as Amazon S3 and Amazon Redshift.
For more details, see the Amazon Kinesis Firehose Documentation.
Example Usage
Extended S3 Destination
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Aws = Pulumi.Aws;
return await Deployment.RunAsync(() =>
{
var bucket = new Aws.S3.BucketV2("bucket");
var firehoseAssumeRole = Aws.Iam.GetPolicyDocument.Invoke(new()
{
Statements = new[]
{
new Aws.Iam.Inputs.GetPolicyDocumentStatementInputArgs
{
Effect = "Allow",
Principals = new[]
{
new Aws.Iam.Inputs.GetPolicyDocumentStatementPrincipalInputArgs
{
Type = "Service",
Identifiers = new[]
{
"firehose.amazonaws.com",
},
},
},
Actions = new[]
{
"sts:AssumeRole",
},
},
},
});
var firehoseRole = new Aws.Iam.Role("firehoseRole", new()
{
AssumeRolePolicy = firehoseAssumeRole.Apply(getPolicyDocumentResult => getPolicyDocumentResult.Json),
});
var lambdaAssumeRole = Aws.Iam.GetPolicyDocument.Invoke(new()
{
Statements = new[]
{
new Aws.Iam.Inputs.GetPolicyDocumentStatementInputArgs
{
Effect = "Allow",
Principals = new[]
{
new Aws.Iam.Inputs.GetPolicyDocumentStatementPrincipalInputArgs
{
Type = "Service",
Identifiers = new[]
{
"lambda.amazonaws.com",
},
},
},
Actions = new[]
{
"sts:AssumeRole",
},
},
},
});
var lambdaIam = new Aws.Iam.Role("lambdaIam", new()
{
AssumeRolePolicy = lambdaAssumeRole.Apply(getPolicyDocumentResult => getPolicyDocumentResult.Json),
});
var lambdaProcessor = new Aws.Lambda.Function("lambdaProcessor", new()
{
Code = new FileArchive("lambda.zip"),
Role = lambdaIam.Arn,
Handler = "exports.handler",
Runtime = "nodejs16.x",
});
var extendedS3Stream = new Aws.Kinesis.FirehoseDeliveryStream("extendedS3Stream", new()
{
Destination = "extended_s3",
ExtendedS3Configuration = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamExtendedS3ConfigurationArgs
{
RoleArn = firehoseRole.Arn,
BucketArn = bucket.Arn,
ProcessingConfiguration = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationArgs
{
Enabled = true,
Processors = new[]
{
new Aws.Kinesis.Inputs.FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationProcessorArgs
{
Type = "Lambda",
Parameters = new[]
{
new Aws.Kinesis.Inputs.FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationProcessorParameterArgs
{
ParameterName = "LambdaArn",
ParameterValue = lambdaProcessor.Arn.Apply(arn => $"{arn}:$LATEST"),
},
},
},
},
},
},
});
var bucketAcl = new Aws.S3.BucketAclV2("bucketAcl", new()
{
Bucket = bucket.Id,
Acl = "private",
});
});
package main
import (
"fmt"
"github.com/pulumi/pulumi-aws/sdk/v5/go/aws/iam"
"github.com/pulumi/pulumi-aws/sdk/v5/go/aws/kinesis"
"github.com/pulumi/pulumi-aws/sdk/v5/go/aws/lambda"
"github.com/pulumi/pulumi-aws/sdk/v5/go/aws/s3"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
bucket, err := s3.NewBucketV2(ctx, "bucket", nil)
if err != nil {
return err
}
firehoseAssumeRole, err := iam.GetPolicyDocument(ctx, &iam.GetPolicyDocumentArgs{
Statements: []iam.GetPolicyDocumentStatement{
{
Effect: pulumi.StringRef("Allow"),
Principals: []iam.GetPolicyDocumentStatementPrincipal{
{
Type: "Service",
Identifiers: []string{
"firehose.amazonaws.com",
},
},
},
Actions: []string{
"sts:AssumeRole",
},
},
},
}, nil)
if err != nil {
return err
}
firehoseRole, err := iam.NewRole(ctx, "firehoseRole", &iam.RoleArgs{
AssumeRolePolicy: *pulumi.String(firehoseAssumeRole.Json),
})
if err != nil {
return err
}
lambdaAssumeRole, err := iam.GetPolicyDocument(ctx, &iam.GetPolicyDocumentArgs{
Statements: []iam.GetPolicyDocumentStatement{
{
Effect: pulumi.StringRef("Allow"),
Principals: []iam.GetPolicyDocumentStatementPrincipal{
{
Type: "Service",
Identifiers: []string{
"lambda.amazonaws.com",
},
},
},
Actions: []string{
"sts:AssumeRole",
},
},
},
}, nil)
if err != nil {
return err
}
lambdaIam, err := iam.NewRole(ctx, "lambdaIam", &iam.RoleArgs{
AssumeRolePolicy: *pulumi.String(lambdaAssumeRole.Json),
})
if err != nil {
return err
}
lambdaProcessor, err := lambda.NewFunction(ctx, "lambdaProcessor", &lambda.FunctionArgs{
Code: pulumi.NewFileArchive("lambda.zip"),
Role: lambdaIam.Arn,
Handler: pulumi.String("exports.handler"),
Runtime: pulumi.String("nodejs16.x"),
})
if err != nil {
return err
}
_, err = kinesis.NewFirehoseDeliveryStream(ctx, "extendedS3Stream", &kinesis.FirehoseDeliveryStreamArgs{
Destination: pulumi.String("extended_s3"),
ExtendedS3Configuration: &kinesis.FirehoseDeliveryStreamExtendedS3ConfigurationArgs{
RoleArn: firehoseRole.Arn,
BucketArn: bucket.Arn,
ProcessingConfiguration: &kinesis.FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationArgs{
Enabled: pulumi.Bool(true),
Processors: kinesis.FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationProcessorArray{
&kinesis.FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationProcessorArgs{
Type: pulumi.String("Lambda"),
Parameters: kinesis.FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationProcessorParameterArray{
&kinesis.FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationProcessorParameterArgs{
ParameterName: pulumi.String("LambdaArn"),
ParameterValue: lambdaProcessor.Arn.ApplyT(func(arn string) (string, error) {
return fmt.Sprintf("%v:$LATEST", arn), nil
}).(pulumi.StringOutput),
},
},
},
},
},
},
})
if err != nil {
return err
}
_, err = s3.NewBucketAclV2(ctx, "bucketAcl", &s3.BucketAclV2Args{
Bucket: bucket.ID(),
Acl: pulumi.String("private"),
})
if err != nil {
return err
}
return nil
})
}
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.s3.BucketV2;
import com.pulumi.aws.iam.IamFunctions;
import com.pulumi.aws.iam.inputs.GetPolicyDocumentArgs;
import com.pulumi.aws.iam.Role;
import com.pulumi.aws.iam.RoleArgs;
import com.pulumi.aws.lambda.Function;
import com.pulumi.aws.lambda.FunctionArgs;
import com.pulumi.aws.kinesis.FirehoseDeliveryStream;
import com.pulumi.aws.kinesis.FirehoseDeliveryStreamArgs;
import com.pulumi.aws.kinesis.inputs.FirehoseDeliveryStreamExtendedS3ConfigurationArgs;
import com.pulumi.aws.kinesis.inputs.FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationArgs;
import com.pulumi.aws.s3.BucketAclV2;
import com.pulumi.aws.s3.BucketAclV2Args;
import com.pulumi.asset.FileArchive;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var bucket = new BucketV2("bucket");
final var firehoseAssumeRole = IamFunctions.getPolicyDocument(GetPolicyDocumentArgs.builder()
.statements(GetPolicyDocumentStatementArgs.builder()
.effect("Allow")
.principals(GetPolicyDocumentStatementPrincipalArgs.builder()
.type("Service")
.identifiers("firehose.amazonaws.com")
.build())
.actions("sts:AssumeRole")
.build())
.build());
var firehoseRole = new Role("firehoseRole", RoleArgs.builder()
.assumeRolePolicy(firehoseAssumeRole.applyValue(getPolicyDocumentResult -> getPolicyDocumentResult.json()))
.build());
final var lambdaAssumeRole = IamFunctions.getPolicyDocument(GetPolicyDocumentArgs.builder()
.statements(GetPolicyDocumentStatementArgs.builder()
.effect("Allow")
.principals(GetPolicyDocumentStatementPrincipalArgs.builder()
.type("Service")
.identifiers("lambda.amazonaws.com")
.build())
.actions("sts:AssumeRole")
.build())
.build());
var lambdaIam = new Role("lambdaIam", RoleArgs.builder()
.assumeRolePolicy(lambdaAssumeRole.applyValue(getPolicyDocumentResult -> getPolicyDocumentResult.json()))
.build());
var lambdaProcessor = new Function("lambdaProcessor", FunctionArgs.builder()
.code(new FileArchive("lambda.zip"))
.role(lambdaIam.arn())
.handler("exports.handler")
.runtime("nodejs16.x")
.build());
var extendedS3Stream = new FirehoseDeliveryStream("extendedS3Stream", FirehoseDeliveryStreamArgs.builder()
.destination("extended_s3")
.extendedS3Configuration(FirehoseDeliveryStreamExtendedS3ConfigurationArgs.builder()
.roleArn(firehoseRole.arn())
.bucketArn(bucket.arn())
.processingConfiguration(FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationArgs.builder()
.enabled("true")
.processors(FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationProcessorArgs.builder()
.type("Lambda")
.parameters(FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationProcessorParameterArgs.builder()
.parameterName("LambdaArn")
.parameterValue(lambdaProcessor.arn().applyValue(arn -> String.format("%s:$LATEST", arn)))
.build())
.build())
.build())
.build())
.build());
var bucketAcl = new BucketAclV2("bucketAcl", BucketAclV2Args.builder()
.bucket(bucket.id())
.acl("private")
.build());
}
}
import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
const bucket = new aws.s3.BucketV2("bucket", {});
const firehoseAssumeRole = aws.iam.getPolicyDocument({
statements: [{
effect: "Allow",
principals: [{
type: "Service",
identifiers: ["firehose.amazonaws.com"],
}],
actions: ["sts:AssumeRole"],
}],
});
const firehoseRole = new aws.iam.Role("firehoseRole", {assumeRolePolicy: firehoseAssumeRole.then(firehoseAssumeRole => firehoseAssumeRole.json)});
const lambdaAssumeRole = aws.iam.getPolicyDocument({
statements: [{
effect: "Allow",
principals: [{
type: "Service",
identifiers: ["lambda.amazonaws.com"],
}],
actions: ["sts:AssumeRole"],
}],
});
const lambdaIam = new aws.iam.Role("lambdaIam", {assumeRolePolicy: lambdaAssumeRole.then(lambdaAssumeRole => lambdaAssumeRole.json)});
const lambdaProcessor = new aws.lambda.Function("lambdaProcessor", {
code: new pulumi.asset.FileArchive("lambda.zip"),
role: lambdaIam.arn,
handler: "exports.handler",
runtime: "nodejs16.x",
});
const extendedS3Stream = new aws.kinesis.FirehoseDeliveryStream("extendedS3Stream", {
destination: "extended_s3",
extendedS3Configuration: {
roleArn: firehoseRole.arn,
bucketArn: bucket.arn,
processingConfiguration: {
enabled: true,
processors: [{
type: "Lambda",
parameters: [{
parameterName: "LambdaArn",
parameterValue: pulumi.interpolate`${lambdaProcessor.arn}:$LATEST`,
}],
}],
},
},
});
const bucketAcl = new aws.s3.BucketAclV2("bucketAcl", {
bucket: bucket.id,
acl: "private",
});
import pulumi
import pulumi_aws as aws
bucket = aws.s3.BucketV2("bucket")
firehose_assume_role = aws.iam.get_policy_document(statements=[aws.iam.GetPolicyDocumentStatementArgs(
effect="Allow",
principals=[aws.iam.GetPolicyDocumentStatementPrincipalArgs(
type="Service",
identifiers=["firehose.amazonaws.com"],
)],
actions=["sts:AssumeRole"],
)])
firehose_role = aws.iam.Role("firehoseRole", assume_role_policy=firehose_assume_role.json)
lambda_assume_role = aws.iam.get_policy_document(statements=[aws.iam.GetPolicyDocumentStatementArgs(
effect="Allow",
principals=[aws.iam.GetPolicyDocumentStatementPrincipalArgs(
type="Service",
identifiers=["lambda.amazonaws.com"],
)],
actions=["sts:AssumeRole"],
)])
lambda_iam = aws.iam.Role("lambdaIam", assume_role_policy=lambda_assume_role.json)
lambda_processor = aws.lambda_.Function("lambdaProcessor",
code=pulumi.FileArchive("lambda.zip"),
role=lambda_iam.arn,
handler="exports.handler",
runtime="nodejs16.x")
extended_s3_stream = aws.kinesis.FirehoseDeliveryStream("extendedS3Stream",
destination="extended_s3",
extended_s3_configuration=aws.kinesis.FirehoseDeliveryStreamExtendedS3ConfigurationArgs(
role_arn=firehose_role.arn,
bucket_arn=bucket.arn,
processing_configuration=aws.kinesis.FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationArgs(
enabled=True,
processors=[aws.kinesis.FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationProcessorArgs(
type="Lambda",
parameters=[aws.kinesis.FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationProcessorParameterArgs(
parameter_name="LambdaArn",
parameter_value=lambda_processor.arn.apply(lambda arn: f"{arn}:$LATEST"),
)],
)],
),
))
bucket_acl = aws.s3.BucketAclV2("bucketAcl",
bucket=bucket.id,
acl="private")
resources:
extendedS3Stream:
type: aws:kinesis:FirehoseDeliveryStream
properties:
destination: extended_s3
extendedS3Configuration:
roleArn: ${firehoseRole.arn}
bucketArn: ${bucket.arn}
processingConfiguration:
enabled: 'true'
processors:
- type: Lambda
parameters:
- parameterName: LambdaArn
parameterValue: ${lambdaProcessor.arn}:$LATEST
bucket:
type: aws:s3:BucketV2
bucketAcl:
type: aws:s3:BucketAclV2
properties:
bucket: ${bucket.id}
acl: private
firehoseRole:
type: aws:iam:Role
properties:
assumeRolePolicy: ${firehoseAssumeRole.json}
lambdaIam:
type: aws:iam:Role
properties:
assumeRolePolicy: ${lambdaAssumeRole.json}
lambdaProcessor:
type: aws:lambda:Function
properties:
code:
fn::FileArchive: lambda.zip
role: ${lambdaIam.arn}
handler: exports.handler
runtime: nodejs16.x
variables:
firehoseAssumeRole:
fn::invoke:
Function: aws:iam:getPolicyDocument
Arguments:
statements:
- effect: Allow
principals:
- type: Service
identifiers:
- firehose.amazonaws.com
actions:
- sts:AssumeRole
lambdaAssumeRole:
fn::invoke:
Function: aws:iam:getPolicyDocument
Arguments:
statements:
- effect: Allow
principals:
- type: Service
identifiers:
- lambda.amazonaws.com
actions:
- sts:AssumeRole
Extended S3 Destination with dynamic partitioning
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Aws = Pulumi.Aws;
return await Deployment.RunAsync(() =>
{
var extendedS3Stream = new Aws.Kinesis.FirehoseDeliveryStream("extendedS3Stream", new()
{
Destination = "extended_s3",
ExtendedS3Configuration = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamExtendedS3ConfigurationArgs
{
RoleArn = aws_iam_role.Firehose_role.Arn,
BucketArn = aws_s3_bucket.Bucket.Arn,
BufferSize = 64,
DynamicPartitioningConfiguration = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamExtendedS3ConfigurationDynamicPartitioningConfigurationArgs
{
Enabled = true,
},
Prefix = "data/customer_id=!{partitionKeyFromQuery:customer_id}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/",
ErrorOutputPrefix = "errors/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/!{firehose:error-output-type}/",
ProcessingConfiguration = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationArgs
{
Enabled = true,
Processors = new[]
{
new Aws.Kinesis.Inputs.FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationProcessorArgs
{
Type = "RecordDeAggregation",
Parameters = new[]
{
new Aws.Kinesis.Inputs.FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationProcessorParameterArgs
{
ParameterName = "SubRecordType",
ParameterValue = "JSON",
},
},
},
new Aws.Kinesis.Inputs.FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationProcessorArgs
{
Type = "AppendDelimiterToRecord",
},
new Aws.Kinesis.Inputs.FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationProcessorArgs
{
Type = "MetadataExtraction",
Parameters = new[]
{
new Aws.Kinesis.Inputs.FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationProcessorParameterArgs
{
ParameterName = "JsonParsingEngine",
ParameterValue = "JQ-1.6",
},
new Aws.Kinesis.Inputs.FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationProcessorParameterArgs
{
ParameterName = "MetadataExtractionQuery",
ParameterValue = "{customer_id:.customer_id}",
},
},
},
},
},
},
});
});
package main
import (
"github.com/pulumi/pulumi-aws/sdk/v5/go/aws/kinesis"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
_, err := kinesis.NewFirehoseDeliveryStream(ctx, "extendedS3Stream", &kinesis.FirehoseDeliveryStreamArgs{
Destination: pulumi.String("extended_s3"),
ExtendedS3Configuration: &kinesis.FirehoseDeliveryStreamExtendedS3ConfigurationArgs{
RoleArn: pulumi.Any(aws_iam_role.Firehose_role.Arn),
BucketArn: pulumi.Any(aws_s3_bucket.Bucket.Arn),
BufferSize: pulumi.Int(64),
DynamicPartitioningConfiguration: &kinesis.FirehoseDeliveryStreamExtendedS3ConfigurationDynamicPartitioningConfigurationArgs{
Enabled: pulumi.Bool(true),
},
Prefix: pulumi.String("data/customer_id=!{partitionKeyFromQuery:customer_id}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/"),
ErrorOutputPrefix: pulumi.String("errors/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/!{firehose:error-output-type}/"),
ProcessingConfiguration: &kinesis.FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationArgs{
Enabled: pulumi.Bool(true),
Processors: kinesis.FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationProcessorArray{
&kinesis.FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationProcessorArgs{
Type: pulumi.String("RecordDeAggregation"),
Parameters: kinesis.FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationProcessorParameterArray{
&kinesis.FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationProcessorParameterArgs{
ParameterName: pulumi.String("SubRecordType"),
ParameterValue: pulumi.String("JSON"),
},
},
},
&kinesis.FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationProcessorArgs{
Type: pulumi.String("AppendDelimiterToRecord"),
},
&kinesis.FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationProcessorArgs{
Type: pulumi.String("MetadataExtraction"),
Parameters: kinesis.FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationProcessorParameterArray{
&kinesis.FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationProcessorParameterArgs{
ParameterName: pulumi.String("JsonParsingEngine"),
ParameterValue: pulumi.String("JQ-1.6"),
},
&kinesis.FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationProcessorParameterArgs{
ParameterName: pulumi.String("MetadataExtractionQuery"),
ParameterValue: pulumi.String("{customer_id:.customer_id}"),
},
},
},
},
},
},
})
if err != nil {
return err
}
return nil
})
}
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.kinesis.FirehoseDeliveryStream;
import com.pulumi.aws.kinesis.FirehoseDeliveryStreamArgs;
import com.pulumi.aws.kinesis.inputs.FirehoseDeliveryStreamExtendedS3ConfigurationArgs;
import com.pulumi.aws.kinesis.inputs.FirehoseDeliveryStreamExtendedS3ConfigurationDynamicPartitioningConfigurationArgs;
import com.pulumi.aws.kinesis.inputs.FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var extendedS3Stream = new FirehoseDeliveryStream("extendedS3Stream", FirehoseDeliveryStreamArgs.builder()
.destination("extended_s3")
.extendedS3Configuration(FirehoseDeliveryStreamExtendedS3ConfigurationArgs.builder()
.roleArn(aws_iam_role.firehose_role().arn())
.bucketArn(aws_s3_bucket.bucket().arn())
.bufferSize(64)
.dynamicPartitioningConfiguration(FirehoseDeliveryStreamExtendedS3ConfigurationDynamicPartitioningConfigurationArgs.builder()
.enabled("true")
.build())
.prefix("data/customer_id=!{partitionKeyFromQuery:customer_id}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/")
.errorOutputPrefix("errors/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/!{firehose:error-output-type}/")
.processingConfiguration(FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationArgs.builder()
.enabled("true")
.processors(
FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationProcessorArgs.builder()
.type("RecordDeAggregation")
.parameters(FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationProcessorParameterArgs.builder()
.parameterName("SubRecordType")
.parameterValue("JSON")
.build())
.build(),
FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationProcessorArgs.builder()
.type("AppendDelimiterToRecord")
.build(),
FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationProcessorArgs.builder()
.type("MetadataExtraction")
.parameters(
FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationProcessorParameterArgs.builder()
.parameterName("JsonParsingEngine")
.parameterValue("JQ-1.6")
.build(),
FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationProcessorParameterArgs.builder()
.parameterName("MetadataExtractionQuery")
.parameterValue("{customer_id:.customer_id}")
.build())
.build())
.build())
.build())
.build());
}
}
import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
const extendedS3Stream = new aws.kinesis.FirehoseDeliveryStream("extendedS3Stream", {
destination: "extended_s3",
extendedS3Configuration: {
roleArn: aws_iam_role.firehose_role.arn,
bucketArn: aws_s3_bucket.bucket.arn,
bufferSize: 64,
dynamicPartitioningConfiguration: {
enabled: true,
},
prefix: "data/customer_id=!{partitionKeyFromQuery:customer_id}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/",
errorOutputPrefix: "errors/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/!{firehose:error-output-type}/",
processingConfiguration: {
enabled: true,
processors: [
{
type: "RecordDeAggregation",
parameters: [{
parameterName: "SubRecordType",
parameterValue: "JSON",
}],
},
{
type: "AppendDelimiterToRecord",
},
{
type: "MetadataExtraction",
parameters: [
{
parameterName: "JsonParsingEngine",
parameterValue: "JQ-1.6",
},
{
parameterName: "MetadataExtractionQuery",
parameterValue: "{customer_id:.customer_id}",
},
],
},
],
},
},
});
import pulumi
import pulumi_aws as aws
extended_s3_stream = aws.kinesis.FirehoseDeliveryStream("extendedS3Stream",
destination="extended_s3",
extended_s3_configuration=aws.kinesis.FirehoseDeliveryStreamExtendedS3ConfigurationArgs(
role_arn=aws_iam_role["firehose_role"]["arn"],
bucket_arn=aws_s3_bucket["bucket"]["arn"],
buffer_size=64,
dynamic_partitioning_configuration=aws.kinesis.FirehoseDeliveryStreamExtendedS3ConfigurationDynamicPartitioningConfigurationArgs(
enabled=True,
),
prefix="data/customer_id=!{partitionKeyFromQuery:customer_id}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/",
error_output_prefix="errors/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/!{firehose:error-output-type}/",
processing_configuration=aws.kinesis.FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationArgs(
enabled=True,
processors=[
aws.kinesis.FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationProcessorArgs(
type="RecordDeAggregation",
parameters=[aws.kinesis.FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationProcessorParameterArgs(
parameter_name="SubRecordType",
parameter_value="JSON",
)],
),
aws.kinesis.FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationProcessorArgs(
type="AppendDelimiterToRecord",
),
aws.kinesis.FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationProcessorArgs(
type="MetadataExtraction",
parameters=[
aws.kinesis.FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationProcessorParameterArgs(
parameter_name="JsonParsingEngine",
parameter_value="JQ-1.6",
),
aws.kinesis.FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationProcessorParameterArgs(
parameter_name="MetadataExtractionQuery",
parameter_value="{customer_id:.customer_id}",
),
],
),
],
),
))
resources:
extendedS3Stream:
type: aws:kinesis:FirehoseDeliveryStream
properties:
destination: extended_s3
extendedS3Configuration:
roleArn: ${aws_iam_role.firehose_role.arn}
bucketArn: ${aws_s3_bucket.bucket.arn}
bufferSize: 64
dynamicPartitioningConfiguration:
enabled: 'true'
prefix: data/customer_id=!{partitionKeyFromQuery:customer_id}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/
errorOutputPrefix: errors/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/!{firehose:error-output-type}/
processingConfiguration:
enabled: 'true'
processors:
- type: RecordDeAggregation
parameters:
- parameterName: SubRecordType
parameterValue: JSON
- type: AppendDelimiterToRecord
- type: MetadataExtraction
parameters:
- parameterName: JsonParsingEngine
parameterValue: JQ-1.6
- parameterName: MetadataExtractionQuery
parameterValue: '{customer_id:.customer_id}'
S3 Destination (deprecated)
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Aws = Pulumi.Aws;
return await Deployment.RunAsync(() =>
{
var bucket = new Aws.S3.BucketV2("bucket");
var bucketAcl = new Aws.S3.BucketAclV2("bucketAcl", new()
{
Bucket = bucket.Id,
Acl = "private",
});
var assumeRole = Aws.Iam.GetPolicyDocument.Invoke(new()
{
Statements = new[]
{
new Aws.Iam.Inputs.GetPolicyDocumentStatementInputArgs
{
Effect = "Allow",
Principals = new[]
{
new Aws.Iam.Inputs.GetPolicyDocumentStatementPrincipalInputArgs
{
Type = "Service",
Identifiers = new[]
{
"firehose.amazonaws.com",
},
},
},
Actions = new[]
{
"sts:AssumeRole",
},
},
},
});
var firehoseRole = new Aws.Iam.Role("firehoseRole", new()
{
AssumeRolePolicy = assumeRole.Apply(getPolicyDocumentResult => getPolicyDocumentResult.Json),
});
var testStream = new Aws.Kinesis.FirehoseDeliveryStream("testStream", new()
{
Destination = "s3",
S3Configuration = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamS3ConfigurationArgs
{
RoleArn = firehoseRole.Arn,
BucketArn = bucket.Arn,
},
});
});
package main
import (
"github.com/pulumi/pulumi-aws/sdk/v5/go/aws/iam"
"github.com/pulumi/pulumi-aws/sdk/v5/go/aws/kinesis"
"github.com/pulumi/pulumi-aws/sdk/v5/go/aws/s3"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
bucket, err := s3.NewBucketV2(ctx, "bucket", nil)
if err != nil {
return err
}
_, err = s3.NewBucketAclV2(ctx, "bucketAcl", &s3.BucketAclV2Args{
Bucket: bucket.ID(),
Acl: pulumi.String("private"),
})
if err != nil {
return err
}
assumeRole, err := iam.GetPolicyDocument(ctx, &iam.GetPolicyDocumentArgs{
Statements: []iam.GetPolicyDocumentStatement{
{
Effect: pulumi.StringRef("Allow"),
Principals: []iam.GetPolicyDocumentStatementPrincipal{
{
Type: "Service",
Identifiers: []string{
"firehose.amazonaws.com",
},
},
},
Actions: []string{
"sts:AssumeRole",
},
},
},
}, nil)
if err != nil {
return err
}
firehoseRole, err := iam.NewRole(ctx, "firehoseRole", &iam.RoleArgs{
AssumeRolePolicy: *pulumi.String(assumeRole.Json),
})
if err != nil {
return err
}
_, err = kinesis.NewFirehoseDeliveryStream(ctx, "testStream", &kinesis.FirehoseDeliveryStreamArgs{
Destination: pulumi.String("s3"),
S3Configuration: &kinesis.FirehoseDeliveryStreamS3ConfigurationArgs{
RoleArn: firehoseRole.Arn,
BucketArn: bucket.Arn,
},
})
if err != nil {
return err
}
return nil
})
}
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.s3.BucketV2;
import com.pulumi.aws.s3.BucketAclV2;
import com.pulumi.aws.s3.BucketAclV2Args;
import com.pulumi.aws.iam.IamFunctions;
import com.pulumi.aws.iam.inputs.GetPolicyDocumentArgs;
import com.pulumi.aws.iam.Role;
import com.pulumi.aws.iam.RoleArgs;
import com.pulumi.aws.kinesis.FirehoseDeliveryStream;
import com.pulumi.aws.kinesis.FirehoseDeliveryStreamArgs;
import com.pulumi.aws.kinesis.inputs.FirehoseDeliveryStreamS3ConfigurationArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var bucket = new BucketV2("bucket");
var bucketAcl = new BucketAclV2("bucketAcl", BucketAclV2Args.builder()
.bucket(bucket.id())
.acl("private")
.build());
final var assumeRole = IamFunctions.getPolicyDocument(GetPolicyDocumentArgs.builder()
.statements(GetPolicyDocumentStatementArgs.builder()
.effect("Allow")
.principals(GetPolicyDocumentStatementPrincipalArgs.builder()
.type("Service")
.identifiers("firehose.amazonaws.com")
.build())
.actions("sts:AssumeRole")
.build())
.build());
var firehoseRole = new Role("firehoseRole", RoleArgs.builder()
.assumeRolePolicy(assumeRole.applyValue(getPolicyDocumentResult -> getPolicyDocumentResult.json()))
.build());
var testStream = new FirehoseDeliveryStream("testStream", FirehoseDeliveryStreamArgs.builder()
.destination("s3")
.s3Configuration(FirehoseDeliveryStreamS3ConfigurationArgs.builder()
.roleArn(firehoseRole.arn())
.bucketArn(bucket.arn())
.build())
.build());
}
}
import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
const bucket = new aws.s3.BucketV2("bucket", {});
const bucketAcl = new aws.s3.BucketAclV2("bucketAcl", {
bucket: bucket.id,
acl: "private",
});
const assumeRole = aws.iam.getPolicyDocument({
statements: [{
effect: "Allow",
principals: [{
type: "Service",
identifiers: ["firehose.amazonaws.com"],
}],
actions: ["sts:AssumeRole"],
}],
});
const firehoseRole = new aws.iam.Role("firehoseRole", {assumeRolePolicy: assumeRole.then(assumeRole => assumeRole.json)});
const testStream = new aws.kinesis.FirehoseDeliveryStream("testStream", {
destination: "s3",
s3Configuration: {
roleArn: firehoseRole.arn,
bucketArn: bucket.arn,
},
});
import pulumi
import pulumi_aws as aws
bucket = aws.s3.BucketV2("bucket")
bucket_acl = aws.s3.BucketAclV2("bucketAcl",
bucket=bucket.id,
acl="private")
assume_role = aws.iam.get_policy_document(statements=[aws.iam.GetPolicyDocumentStatementArgs(
effect="Allow",
principals=[aws.iam.GetPolicyDocumentStatementPrincipalArgs(
type="Service",
identifiers=["firehose.amazonaws.com"],
)],
actions=["sts:AssumeRole"],
)])
firehose_role = aws.iam.Role("firehoseRole", assume_role_policy=assume_role.json)
test_stream = aws.kinesis.FirehoseDeliveryStream("testStream",
destination="s3",
s3_configuration=aws.kinesis.FirehoseDeliveryStreamS3ConfigurationArgs(
role_arn=firehose_role.arn,
bucket_arn=bucket.arn,
))
resources:
bucket:
type: aws:s3:BucketV2
bucketAcl:
type: aws:s3:BucketAclV2
properties:
bucket: ${bucket.id}
acl: private
firehoseRole:
type: aws:iam:Role
properties:
assumeRolePolicy: ${assumeRole.json}
testStream:
type: aws:kinesis:FirehoseDeliveryStream
properties:
destination: s3
s3Configuration:
roleArn: ${firehoseRole.arn}
bucketArn: ${bucket.arn}
variables:
assumeRole:
fn::invoke:
Function: aws:iam:getPolicyDocument
Arguments:
statements:
- effect: Allow
principals:
- type: Service
identifiers:
- firehose.amazonaws.com
actions:
- sts:AssumeRole
Redshift Destination
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Aws = Pulumi.Aws;
return await Deployment.RunAsync(() =>
{
var testCluster = new Aws.RedShift.Cluster("testCluster", new()
{
ClusterIdentifier = "tf-redshift-cluster",
DatabaseName = "test",
MasterUsername = "testuser",
MasterPassword = "T3stPass",
NodeType = "dc1.large",
ClusterType = "single-node",
});
var testStream = new Aws.Kinesis.FirehoseDeliveryStream("testStream", new()
{
Destination = "redshift",
S3Configuration = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamS3ConfigurationArgs
{
RoleArn = aws_iam_role.Firehose_role.Arn,
BucketArn = aws_s3_bucket.Bucket.Arn,
BufferSize = 10,
BufferInterval = 400,
CompressionFormat = "GZIP",
},
RedshiftConfiguration = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamRedshiftConfigurationArgs
{
RoleArn = aws_iam_role.Firehose_role.Arn,
ClusterJdbcurl = Output.Tuple(testCluster.Endpoint, testCluster.DatabaseName).Apply(values =>
{
var endpoint = values.Item1;
var databaseName = values.Item2;
return $"jdbc:redshift://{endpoint}/{databaseName}";
}),
Username = "testuser",
Password = "T3stPass",
DataTableName = "test-table",
CopyOptions = "delimiter '|'",
DataTableColumns = "test-col",
S3BackupMode = "Enabled",
S3BackupConfiguration = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamRedshiftConfigurationS3BackupConfigurationArgs
{
RoleArn = aws_iam_role.Firehose_role.Arn,
BucketArn = aws_s3_bucket.Bucket.Arn,
BufferSize = 15,
BufferInterval = 300,
CompressionFormat = "GZIP",
},
},
});
});
package main
import (
"fmt"
"github.com/pulumi/pulumi-aws/sdk/v5/go/aws/kinesis"
"github.com/pulumi/pulumi-aws/sdk/v5/go/aws/redshift"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
testCluster, err := redshift.NewCluster(ctx, "testCluster", &redshift.ClusterArgs{
ClusterIdentifier: pulumi.String("tf-redshift-cluster"),
DatabaseName: pulumi.String("test"),
MasterUsername: pulumi.String("testuser"),
MasterPassword: pulumi.String("T3stPass"),
NodeType: pulumi.String("dc1.large"),
ClusterType: pulumi.String("single-node"),
})
if err != nil {
return err
}
_, err = kinesis.NewFirehoseDeliveryStream(ctx, "testStream", &kinesis.FirehoseDeliveryStreamArgs{
Destination: pulumi.String("redshift"),
S3Configuration: &kinesis.FirehoseDeliveryStreamS3ConfigurationArgs{
RoleArn: pulumi.Any(aws_iam_role.Firehose_role.Arn),
BucketArn: pulumi.Any(aws_s3_bucket.Bucket.Arn),
BufferSize: pulumi.Int(10),
BufferInterval: pulumi.Int(400),
CompressionFormat: pulumi.String("GZIP"),
},
RedshiftConfiguration: &kinesis.FirehoseDeliveryStreamRedshiftConfigurationArgs{
RoleArn: pulumi.Any(aws_iam_role.Firehose_role.Arn),
ClusterJdbcurl: pulumi.All(testCluster.Endpoint, testCluster.DatabaseName).ApplyT(func(_args []interface{}) (string, error) {
endpoint := _args[0].(string)
databaseName := _args[1].(string)
return fmt.Sprintf("jdbc:redshift://%v/%v", endpoint, databaseName), nil
}).(pulumi.StringOutput),
Username: pulumi.String("testuser"),
Password: pulumi.String("T3stPass"),
DataTableName: pulumi.String("test-table"),
CopyOptions: pulumi.String("delimiter '|'"),
DataTableColumns: pulumi.String("test-col"),
S3BackupMode: pulumi.String("Enabled"),
S3BackupConfiguration: &kinesis.FirehoseDeliveryStreamRedshiftConfigurationS3BackupConfigurationArgs{
RoleArn: pulumi.Any(aws_iam_role.Firehose_role.Arn),
BucketArn: pulumi.Any(aws_s3_bucket.Bucket.Arn),
BufferSize: pulumi.Int(15),
BufferInterval: pulumi.Int(300),
CompressionFormat: pulumi.String("GZIP"),
},
},
})
if err != nil {
return err
}
return nil
})
}
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.redshift.Cluster;
import com.pulumi.aws.redshift.ClusterArgs;
import com.pulumi.aws.kinesis.FirehoseDeliveryStream;
import com.pulumi.aws.kinesis.FirehoseDeliveryStreamArgs;
import com.pulumi.aws.kinesis.inputs.FirehoseDeliveryStreamS3ConfigurationArgs;
import com.pulumi.aws.kinesis.inputs.FirehoseDeliveryStreamRedshiftConfigurationArgs;
import com.pulumi.aws.kinesis.inputs.FirehoseDeliveryStreamRedshiftConfigurationS3BackupConfigurationArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var testCluster = new Cluster("testCluster", ClusterArgs.builder()
.clusterIdentifier("tf-redshift-cluster")
.databaseName("test")
.masterUsername("testuser")
.masterPassword("T3stPass")
.nodeType("dc1.large")
.clusterType("single-node")
.build());
var testStream = new FirehoseDeliveryStream("testStream", FirehoseDeliveryStreamArgs.builder()
.destination("redshift")
.s3Configuration(FirehoseDeliveryStreamS3ConfigurationArgs.builder()
.roleArn(aws_iam_role.firehose_role().arn())
.bucketArn(aws_s3_bucket.bucket().arn())
.bufferSize(10)
.bufferInterval(400)
.compressionFormat("GZIP")
.build())
.redshiftConfiguration(FirehoseDeliveryStreamRedshiftConfigurationArgs.builder()
.roleArn(aws_iam_role.firehose_role().arn())
.clusterJdbcurl(Output.tuple(testCluster.endpoint(), testCluster.databaseName()).applyValue(values -> {
var endpoint = values.t1;
var databaseName = values.t2;
return String.format("jdbc:redshift://%s/%s", endpoint,databaseName);
}))
.username("testuser")
.password("T3stPass")
.dataTableName("test-table")
.copyOptions("delimiter '|'")
.dataTableColumns("test-col")
.s3BackupMode("Enabled")
.s3BackupConfiguration(FirehoseDeliveryStreamRedshiftConfigurationS3BackupConfigurationArgs.builder()
.roleArn(aws_iam_role.firehose_role().arn())
.bucketArn(aws_s3_bucket.bucket().arn())
.bufferSize(15)
.bufferInterval(300)
.compressionFormat("GZIP")
.build())
.build())
.build());
}
}
import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
const testCluster = new aws.redshift.Cluster("testCluster", {
clusterIdentifier: "tf-redshift-cluster",
databaseName: "test",
masterUsername: "testuser",
masterPassword: "T3stPass",
nodeType: "dc1.large",
clusterType: "single-node",
});
const testStream = new aws.kinesis.FirehoseDeliveryStream("testStream", {
destination: "redshift",
s3Configuration: {
roleArn: aws_iam_role.firehose_role.arn,
bucketArn: aws_s3_bucket.bucket.arn,
bufferSize: 10,
bufferInterval: 400,
compressionFormat: "GZIP",
},
redshiftConfiguration: {
roleArn: aws_iam_role.firehose_role.arn,
clusterJdbcurl: pulumi.interpolate`jdbc:redshift://${testCluster.endpoint}/${testCluster.databaseName}`,
username: "testuser",
password: "T3stPass",
dataTableName: "test-table",
copyOptions: "delimiter '|'",
dataTableColumns: "test-col",
s3BackupMode: "Enabled",
s3BackupConfiguration: {
roleArn: aws_iam_role.firehose_role.arn,
bucketArn: aws_s3_bucket.bucket.arn,
bufferSize: 15,
bufferInterval: 300,
compressionFormat: "GZIP",
},
},
});
import pulumi
import pulumi_aws as aws
test_cluster = aws.redshift.Cluster("testCluster",
cluster_identifier="tf-redshift-cluster",
database_name="test",
master_username="testuser",
master_password="T3stPass",
node_type="dc1.large",
cluster_type="single-node")
test_stream = aws.kinesis.FirehoseDeliveryStream("testStream",
destination="redshift",
s3_configuration=aws.kinesis.FirehoseDeliveryStreamS3ConfigurationArgs(
role_arn=aws_iam_role["firehose_role"]["arn"],
bucket_arn=aws_s3_bucket["bucket"]["arn"],
buffer_size=10,
buffer_interval=400,
compression_format="GZIP",
),
redshift_configuration=aws.kinesis.FirehoseDeliveryStreamRedshiftConfigurationArgs(
role_arn=aws_iam_role["firehose_role"]["arn"],
cluster_jdbcurl=pulumi.Output.all(test_cluster.endpoint, test_cluster.database_name).apply(lambda endpoint, database_name: f"jdbc:redshift://{endpoint}/{database_name}"),
username="testuser",
password="T3stPass",
data_table_name="test-table",
copy_options="delimiter '|'",
data_table_columns="test-col",
s3_backup_mode="Enabled",
s3_backup_configuration=aws.kinesis.FirehoseDeliveryStreamRedshiftConfigurationS3BackupConfigurationArgs(
role_arn=aws_iam_role["firehose_role"]["arn"],
bucket_arn=aws_s3_bucket["bucket"]["arn"],
buffer_size=15,
buffer_interval=300,
compression_format="GZIP",
),
))
resources:
testCluster:
type: aws:redshift:Cluster
properties:
clusterIdentifier: tf-redshift-cluster
databaseName: test
masterUsername: testuser
masterPassword: T3stPass
nodeType: dc1.large
clusterType: single-node
testStream:
type: aws:kinesis:FirehoseDeliveryStream
properties:
destination: redshift
s3Configuration:
roleArn: ${aws_iam_role.firehose_role.arn}
bucketArn: ${aws_s3_bucket.bucket.arn}
bufferSize: 10
bufferInterval: 400
compressionFormat: GZIP
redshiftConfiguration:
roleArn: ${aws_iam_role.firehose_role.arn}
clusterJdbcurl: jdbc:redshift://${testCluster.endpoint}/${testCluster.databaseName}
username: testuser
password: T3stPass
dataTableName: test-table
copyOptions: delimiter '|'
dataTableColumns: test-col
s3BackupMode: Enabled
s3BackupConfiguration:
roleArn: ${aws_iam_role.firehose_role.arn}
bucketArn: ${aws_s3_bucket.bucket.arn}
bufferSize: 15
bufferInterval: 300
compressionFormat: GZIP
Elasticsearch Destination
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Aws = Pulumi.Aws;
return await Deployment.RunAsync(() =>
{
var testCluster = new Aws.ElasticSearch.Domain("testCluster");
var testStream = new Aws.Kinesis.FirehoseDeliveryStream("testStream", new()
{
Destination = "elasticsearch",
S3Configuration = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamS3ConfigurationArgs
{
RoleArn = aws_iam_role.Firehose_role.Arn,
BucketArn = aws_s3_bucket.Bucket.Arn,
BufferSize = 10,
BufferInterval = 400,
CompressionFormat = "GZIP",
},
ElasticsearchConfiguration = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamElasticsearchConfigurationArgs
{
DomainArn = testCluster.Arn,
RoleArn = aws_iam_role.Firehose_role.Arn,
IndexName = "test",
TypeName = "test",
ProcessingConfiguration = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamElasticsearchConfigurationProcessingConfigurationArgs
{
Enabled = true,
Processors = new[]
{
new Aws.Kinesis.Inputs.FirehoseDeliveryStreamElasticsearchConfigurationProcessingConfigurationProcessorArgs
{
Type = "Lambda",
Parameters = new[]
{
new Aws.Kinesis.Inputs.FirehoseDeliveryStreamElasticsearchConfigurationProcessingConfigurationProcessorParameterArgs
{
ParameterName = "LambdaArn",
ParameterValue = $"{aws_lambda_function.Lambda_processor.Arn}:$LATEST",
},
},
},
},
},
},
});
});
package main
import (
"fmt"
"github.com/pulumi/pulumi-aws/sdk/v5/go/aws/elasticsearch"
"github.com/pulumi/pulumi-aws/sdk/v5/go/aws/kinesis"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
testCluster, err := elasticsearch.NewDomain(ctx, "testCluster", nil)
if err != nil {
return err
}
_, err = kinesis.NewFirehoseDeliveryStream(ctx, "testStream", &kinesis.FirehoseDeliveryStreamArgs{
Destination: pulumi.String("elasticsearch"),
S3Configuration: &kinesis.FirehoseDeliveryStreamS3ConfigurationArgs{
RoleArn: pulumi.Any(aws_iam_role.Firehose_role.Arn),
BucketArn: pulumi.Any(aws_s3_bucket.Bucket.Arn),
BufferSize: pulumi.Int(10),
BufferInterval: pulumi.Int(400),
CompressionFormat: pulumi.String("GZIP"),
},
ElasticsearchConfiguration: &kinesis.FirehoseDeliveryStreamElasticsearchConfigurationArgs{
DomainArn: testCluster.Arn,
RoleArn: pulumi.Any(aws_iam_role.Firehose_role.Arn),
IndexName: pulumi.String("test"),
TypeName: pulumi.String("test"),
ProcessingConfiguration: &kinesis.FirehoseDeliveryStreamElasticsearchConfigurationProcessingConfigurationArgs{
Enabled: pulumi.Bool(true),
Processors: kinesis.FirehoseDeliveryStreamElasticsearchConfigurationProcessingConfigurationProcessorArray{
&kinesis.FirehoseDeliveryStreamElasticsearchConfigurationProcessingConfigurationProcessorArgs{
Type: pulumi.String("Lambda"),
Parameters: kinesis.FirehoseDeliveryStreamElasticsearchConfigurationProcessingConfigurationProcessorParameterArray{
&kinesis.FirehoseDeliveryStreamElasticsearchConfigurationProcessingConfigurationProcessorParameterArgs{
ParameterName: pulumi.String("LambdaArn"),
ParameterValue: pulumi.String(fmt.Sprintf("%v:$LATEST", aws_lambda_function.Lambda_processor.Arn)),
},
},
},
},
},
},
})
if err != nil {
return err
}
return nil
})
}
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.elasticsearch.Domain;
import com.pulumi.aws.kinesis.FirehoseDeliveryStream;
import com.pulumi.aws.kinesis.FirehoseDeliveryStreamArgs;
import com.pulumi.aws.kinesis.inputs.FirehoseDeliveryStreamS3ConfigurationArgs;
import com.pulumi.aws.kinesis.inputs.FirehoseDeliveryStreamElasticsearchConfigurationArgs;
import com.pulumi.aws.kinesis.inputs.FirehoseDeliveryStreamElasticsearchConfigurationProcessingConfigurationArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var testCluster = new Domain("testCluster");
var testStream = new FirehoseDeliveryStream("testStream", FirehoseDeliveryStreamArgs.builder()
.destination("elasticsearch")
.s3Configuration(FirehoseDeliveryStreamS3ConfigurationArgs.builder()
.roleArn(aws_iam_role.firehose_role().arn())
.bucketArn(aws_s3_bucket.bucket().arn())
.bufferSize(10)
.bufferInterval(400)
.compressionFormat("GZIP")
.build())
.elasticsearchConfiguration(FirehoseDeliveryStreamElasticsearchConfigurationArgs.builder()
.domainArn(testCluster.arn())
.roleArn(aws_iam_role.firehose_role().arn())
.indexName("test")
.typeName("test")
.processingConfiguration(FirehoseDeliveryStreamElasticsearchConfigurationProcessingConfigurationArgs.builder()
.enabled("true")
.processors(FirehoseDeliveryStreamElasticsearchConfigurationProcessingConfigurationProcessorArgs.builder()
.type("Lambda")
.parameters(FirehoseDeliveryStreamElasticsearchConfigurationProcessingConfigurationProcessorParameterArgs.builder()
.parameterName("LambdaArn")
.parameterValue(String.format("%s:$LATEST", aws_lambda_function.lambda_processor().arn()))
.build())
.build())
.build())
.build())
.build());
}
}
import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
const testCluster = new aws.elasticsearch.Domain("testCluster", {});
const testStream = new aws.kinesis.FirehoseDeliveryStream("testStream", {
destination: "elasticsearch",
s3Configuration: {
roleArn: aws_iam_role.firehose_role.arn,
bucketArn: aws_s3_bucket.bucket.arn,
bufferSize: 10,
bufferInterval: 400,
compressionFormat: "GZIP",
},
elasticsearchConfiguration: {
domainArn: testCluster.arn,
roleArn: aws_iam_role.firehose_role.arn,
indexName: "test",
typeName: "test",
processingConfiguration: {
enabled: true,
processors: [{
type: "Lambda",
parameters: [{
parameterName: "LambdaArn",
parameterValue: `${aws_lambda_function.lambda_processor.arn}:$LATEST`,
}],
}],
},
},
});
import pulumi
import pulumi_aws as aws
test_cluster = aws.elasticsearch.Domain("testCluster")
test_stream = aws.kinesis.FirehoseDeliveryStream("testStream",
destination="elasticsearch",
s3_configuration=aws.kinesis.FirehoseDeliveryStreamS3ConfigurationArgs(
role_arn=aws_iam_role["firehose_role"]["arn"],
bucket_arn=aws_s3_bucket["bucket"]["arn"],
buffer_size=10,
buffer_interval=400,
compression_format="GZIP",
),
elasticsearch_configuration=aws.kinesis.FirehoseDeliveryStreamElasticsearchConfigurationArgs(
domain_arn=test_cluster.arn,
role_arn=aws_iam_role["firehose_role"]["arn"],
index_name="test",
type_name="test",
processing_configuration=aws.kinesis.FirehoseDeliveryStreamElasticsearchConfigurationProcessingConfigurationArgs(
enabled=True,
processors=[aws.kinesis.FirehoseDeliveryStreamElasticsearchConfigurationProcessingConfigurationProcessorArgs(
type="Lambda",
parameters=[aws.kinesis.FirehoseDeliveryStreamElasticsearchConfigurationProcessingConfigurationProcessorParameterArgs(
parameter_name="LambdaArn",
parameter_value=f"{aws_lambda_function['lambda_processor']['arn']}:$LATEST",
)],
)],
),
))
resources:
testCluster:
type: aws:elasticsearch:Domain
testStream:
type: aws:kinesis:FirehoseDeliveryStream
properties:
destination: elasticsearch
s3Configuration:
roleArn: ${aws_iam_role.firehose_role.arn}
bucketArn: ${aws_s3_bucket.bucket.arn}
bufferSize: 10
bufferInterval: 400
compressionFormat: GZIP
elasticsearchConfiguration:
domainArn: ${testCluster.arn}
roleArn: ${aws_iam_role.firehose_role.arn}
indexName: test
typeName: test
processingConfiguration:
enabled: 'true'
processors:
- type: Lambda
parameters:
- parameterName: LambdaArn
parameterValue: ${aws_lambda_function.lambda_processor.arn}:$LATEST
Elasticsearch Destination With VPC
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Aws = Pulumi.Aws;
return await Deployment.RunAsync(() =>
{
var testCluster = new Aws.ElasticSearch.Domain("testCluster", new()
{
ClusterConfig = new Aws.ElasticSearch.Inputs.DomainClusterConfigArgs
{
InstanceCount = 2,
ZoneAwarenessEnabled = true,
InstanceType = "t2.small.elasticsearch",
},
EbsOptions = new Aws.ElasticSearch.Inputs.DomainEbsOptionsArgs
{
EbsEnabled = true,
VolumeSize = 10,
},
VpcOptions = new Aws.ElasticSearch.Inputs.DomainVpcOptionsArgs
{
SecurityGroupIds = new[]
{
aws_security_group.First.Id,
},
SubnetIds = new[]
{
aws_subnet.First.Id,
aws_subnet.Second.Id,
},
},
});
var firehose_elasticsearchPolicyDocument = Aws.Iam.GetPolicyDocument.Invoke(new()
{
Statements = new[]
{
new Aws.Iam.Inputs.GetPolicyDocumentStatementInputArgs
{
Effect = "Allow",
Actions = new[]
{
"es:*",
},
Resources = new[]
{
testCluster.Arn,
$"{testCluster.Arn}/*",
},
},
new Aws.Iam.Inputs.GetPolicyDocumentStatementInputArgs
{
Effect = "Allow",
Actions = new[]
{
"ec2:DescribeVpcs",
"ec2:DescribeVpcAttribute",
"ec2:DescribeSubnets",
"ec2:DescribeSecurityGroups",
"ec2:DescribeNetworkInterfaces",
"ec2:CreateNetworkInterface",
"ec2:CreateNetworkInterfacePermission",
"ec2:DeleteNetworkInterface",
},
Resources = new[]
{
"*",
},
},
},
});
var firehose_elasticsearchRolePolicy = new Aws.Iam.RolePolicy("firehose-elasticsearchRolePolicy", new()
{
Role = aws_iam_role.Firehose.Id,
Policy = firehose_elasticsearchPolicyDocument.Apply(firehose_elasticsearchPolicyDocument => firehose_elasticsearchPolicyDocument.Apply(getPolicyDocumentResult => getPolicyDocumentResult.Json)),
});
var test = new Aws.Kinesis.FirehoseDeliveryStream("test", new()
{
Destination = "elasticsearch",
S3Configuration = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamS3ConfigurationArgs
{
RoleArn = aws_iam_role.Firehose.Arn,
BucketArn = aws_s3_bucket.Bucket.Arn,
},
ElasticsearchConfiguration = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamElasticsearchConfigurationArgs
{
DomainArn = testCluster.Arn,
RoleArn = aws_iam_role.Firehose.Arn,
IndexName = "test",
TypeName = "test",
VpcConfig = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamElasticsearchConfigurationVpcConfigArgs
{
SubnetIds = new[]
{
aws_subnet.First.Id,
aws_subnet.Second.Id,
},
SecurityGroupIds = new[]
{
aws_security_group.First.Id,
},
RoleArn = aws_iam_role.Firehose.Arn,
},
},
}, new CustomResourceOptions
{
DependsOn = new[]
{
firehose_elasticsearchRolePolicy,
},
});
});
package main
import (
"fmt"
"github.com/pulumi/pulumi-aws/sdk/v5/go/aws/elasticsearch"
"github.com/pulumi/pulumi-aws/sdk/v5/go/aws/iam"
"github.com/pulumi/pulumi-aws/sdk/v5/go/aws/kinesis"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
testCluster, err := elasticsearch.NewDomain(ctx, "testCluster", &elasticsearch.DomainArgs{
ClusterConfig: &elasticsearch.DomainClusterConfigArgs{
InstanceCount: pulumi.Int(2),
ZoneAwarenessEnabled: pulumi.Bool(true),
InstanceType: pulumi.String("t2.small.elasticsearch"),
},
EbsOptions: &elasticsearch.DomainEbsOptionsArgs{
EbsEnabled: pulumi.Bool(true),
VolumeSize: pulumi.Int(10),
},
VpcOptions: &elasticsearch.DomainVpcOptionsArgs{
SecurityGroupIds: pulumi.StringArray{
aws_security_group.First.Id,
},
SubnetIds: pulumi.StringArray{
aws_subnet.First.Id,
aws_subnet.Second.Id,
},
},
})
if err != nil {
return err
}
firehose_elasticsearchPolicyDocument := iam.GetPolicyDocumentOutput(ctx, iam.GetPolicyDocumentOutputArgs{
Statements: iam.GetPolicyDocumentStatementArray{
&iam.GetPolicyDocumentStatementArgs{
Effect: pulumi.String("Allow"),
Actions: pulumi.StringArray{
pulumi.String("es:*"),
},
Resources: pulumi.StringArray{
testCluster.Arn,
testCluster.Arn.ApplyT(func(arn string) (string, error) {
return fmt.Sprintf("%v/*", arn), nil
}).(pulumi.StringOutput),
},
},
&iam.GetPolicyDocumentStatementArgs{
Effect: pulumi.String("Allow"),
Actions: pulumi.StringArray{
pulumi.String("ec2:DescribeVpcs"),
pulumi.String("ec2:DescribeVpcAttribute"),
pulumi.String("ec2:DescribeSubnets"),
pulumi.String("ec2:DescribeSecurityGroups"),
pulumi.String("ec2:DescribeNetworkInterfaces"),
pulumi.String("ec2:CreateNetworkInterface"),
pulumi.String("ec2:CreateNetworkInterfacePermission"),
pulumi.String("ec2:DeleteNetworkInterface"),
},
Resources: pulumi.StringArray{
pulumi.String("*"),
},
},
},
}, nil)
_, err = iam.NewRolePolicy(ctx, "firehose-elasticsearchRolePolicy", &iam.RolePolicyArgs{
Role: pulumi.Any(aws_iam_role.Firehose.Id),
Policy: firehose_elasticsearchPolicyDocument.ApplyT(func(firehose_elasticsearchPolicyDocument iam.GetPolicyDocumentResult) (*string, error) {
return &firehose_elasticsearchPolicyDocument.Json, nil
}).(pulumi.StringPtrOutput),
})
if err != nil {
return err
}
_, err = kinesis.NewFirehoseDeliveryStream(ctx, "test", &kinesis.FirehoseDeliveryStreamArgs{
Destination: pulumi.String("elasticsearch"),
S3Configuration: &kinesis.FirehoseDeliveryStreamS3ConfigurationArgs{
RoleArn: pulumi.Any(aws_iam_role.Firehose.Arn),
BucketArn: pulumi.Any(aws_s3_bucket.Bucket.Arn),
},
ElasticsearchConfiguration: &kinesis.FirehoseDeliveryStreamElasticsearchConfigurationArgs{
DomainArn: testCluster.Arn,
RoleArn: pulumi.Any(aws_iam_role.Firehose.Arn),
IndexName: pulumi.String("test"),
TypeName: pulumi.String("test"),
VpcConfig: &kinesis.FirehoseDeliveryStreamElasticsearchConfigurationVpcConfigArgs{
SubnetIds: pulumi.StringArray{
aws_subnet.First.Id,
aws_subnet.Second.Id,
},
SecurityGroupIds: pulumi.StringArray{
aws_security_group.First.Id,
},
RoleArn: pulumi.Any(aws_iam_role.Firehose.Arn),
},
},
}, pulumi.DependsOn([]pulumi.Resource{
firehose_elasticsearchRolePolicy,
}))
if err != nil {
return err
}
return nil
})
}
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.elasticsearch.Domain;
import com.pulumi.aws.elasticsearch.DomainArgs;
import com.pulumi.aws.elasticsearch.inputs.DomainClusterConfigArgs;
import com.pulumi.aws.elasticsearch.inputs.DomainEbsOptionsArgs;
import com.pulumi.aws.elasticsearch.inputs.DomainVpcOptionsArgs;
import com.pulumi.aws.iam.IamFunctions;
import com.pulumi.aws.iam.inputs.GetPolicyDocumentArgs;
import com.pulumi.aws.iam.RolePolicy;
import com.pulumi.aws.iam.RolePolicyArgs;
import com.pulumi.aws.kinesis.FirehoseDeliveryStream;
import com.pulumi.aws.kinesis.FirehoseDeliveryStreamArgs;
import com.pulumi.aws.kinesis.inputs.FirehoseDeliveryStreamS3ConfigurationArgs;
import com.pulumi.aws.kinesis.inputs.FirehoseDeliveryStreamElasticsearchConfigurationArgs;
import com.pulumi.aws.kinesis.inputs.FirehoseDeliveryStreamElasticsearchConfigurationVpcConfigArgs;
import com.pulumi.resources.CustomResourceOptions;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var testCluster = new Domain("testCluster", DomainArgs.builder()
.clusterConfig(DomainClusterConfigArgs.builder()
.instanceCount(2)
.zoneAwarenessEnabled(true)
.instanceType("t2.small.elasticsearch")
.build())
.ebsOptions(DomainEbsOptionsArgs.builder()
.ebsEnabled(true)
.volumeSize(10)
.build())
.vpcOptions(DomainVpcOptionsArgs.builder()
.securityGroupIds(aws_security_group.first().id())
.subnetIds(
aws_subnet.first().id(),
aws_subnet.second().id())
.build())
.build());
final var firehose-elasticsearchPolicyDocument = IamFunctions.getPolicyDocument(GetPolicyDocumentArgs.builder()
.statements(
GetPolicyDocumentStatementArgs.builder()
.effect("Allow")
.actions("es:*")
.resources(
testCluster.arn(),
testCluster.arn().applyValue(arn -> String.format("%s/*", arn)))
.build(),
GetPolicyDocumentStatementArgs.builder()
.effect("Allow")
.actions(
"ec2:DescribeVpcs",
"ec2:DescribeVpcAttribute",
"ec2:DescribeSubnets",
"ec2:DescribeSecurityGroups",
"ec2:DescribeNetworkInterfaces",
"ec2:CreateNetworkInterface",
"ec2:CreateNetworkInterfacePermission",
"ec2:DeleteNetworkInterface")
.resources("*")
.build())
.build());
var firehose_elasticsearchRolePolicy = new RolePolicy("firehose-elasticsearchRolePolicy", RolePolicyArgs.builder()
.role(aws_iam_role.firehose().id())
.policy(firehose_elasticsearchPolicyDocument.applyValue(firehose_elasticsearchPolicyDocument -> firehose_elasticsearchPolicyDocument.json()))
.build());
var test = new FirehoseDeliveryStream("test", FirehoseDeliveryStreamArgs.builder()
.destination("elasticsearch")
.s3Configuration(FirehoseDeliveryStreamS3ConfigurationArgs.builder()
.roleArn(aws_iam_role.firehose().arn())
.bucketArn(aws_s3_bucket.bucket().arn())
.build())
.elasticsearchConfiguration(FirehoseDeliveryStreamElasticsearchConfigurationArgs.builder()
.domainArn(testCluster.arn())
.roleArn(aws_iam_role.firehose().arn())
.indexName("test")
.typeName("test")
.vpcConfig(FirehoseDeliveryStreamElasticsearchConfigurationVpcConfigArgs.builder()
.subnetIds(
aws_subnet.first().id(),
aws_subnet.second().id())
.securityGroupIds(aws_security_group.first().id())
.roleArn(aws_iam_role.firehose().arn())
.build())
.build())
.build(), CustomResourceOptions.builder()
.dependsOn(firehose_elasticsearchRolePolicy)
.build());
}
}
import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
const testCluster = new aws.elasticsearch.Domain("testCluster", {
clusterConfig: {
instanceCount: 2,
zoneAwarenessEnabled: true,
instanceType: "t2.small.elasticsearch",
},
ebsOptions: {
ebsEnabled: true,
volumeSize: 10,
},
vpcOptions: {
securityGroupIds: [aws_security_group.first.id],
subnetIds: [
aws_subnet.first.id,
aws_subnet.second.id,
],
},
});
const firehose-elasticsearchPolicyDocument = aws.iam.getPolicyDocumentOutput({
statements: [
{
effect: "Allow",
actions: ["es:*"],
resources: [
testCluster.arn,
pulumi.interpolate`${testCluster.arn}/*`,
],
},
{
effect: "Allow",
actions: [
"ec2:DescribeVpcs",
"ec2:DescribeVpcAttribute",
"ec2:DescribeSubnets",
"ec2:DescribeSecurityGroups",
"ec2:DescribeNetworkInterfaces",
"ec2:CreateNetworkInterface",
"ec2:CreateNetworkInterfacePermission",
"ec2:DeleteNetworkInterface",
],
resources: ["*"],
},
],
});
const firehose_elasticsearchRolePolicy = new aws.iam.RolePolicy("firehose-elasticsearchRolePolicy", {
role: aws_iam_role.firehose.id,
policy: firehose_elasticsearchPolicyDocument.apply(firehose_elasticsearchPolicyDocument => firehose_elasticsearchPolicyDocument.json),
});
const test = new aws.kinesis.FirehoseDeliveryStream("test", {
destination: "elasticsearch",
s3Configuration: {
roleArn: aws_iam_role.firehose.arn,
bucketArn: aws_s3_bucket.bucket.arn,
},
elasticsearchConfiguration: {
domainArn: testCluster.arn,
roleArn: aws_iam_role.firehose.arn,
indexName: "test",
typeName: "test",
vpcConfig: {
subnetIds: [
aws_subnet.first.id,
aws_subnet.second.id,
],
securityGroupIds: [aws_security_group.first.id],
roleArn: aws_iam_role.firehose.arn,
},
},
}, {
dependsOn: [firehose_elasticsearchRolePolicy],
});
import pulumi
import pulumi_aws as aws
test_cluster = aws.elasticsearch.Domain("testCluster",
cluster_config=aws.elasticsearch.DomainClusterConfigArgs(
instance_count=2,
zone_awareness_enabled=True,
instance_type="t2.small.elasticsearch",
),
ebs_options=aws.elasticsearch.DomainEbsOptionsArgs(
ebs_enabled=True,
volume_size=10,
),
vpc_options=aws.elasticsearch.DomainVpcOptionsArgs(
security_group_ids=[aws_security_group["first"]["id"]],
subnet_ids=[
aws_subnet["first"]["id"],
aws_subnet["second"]["id"],
],
))
firehose_elasticsearch_policy_document = aws.iam.get_policy_document_output(statements=[
aws.iam.GetPolicyDocumentStatementArgs(
effect="Allow",
actions=["es:*"],
resources=[
test_cluster.arn,
test_cluster.arn.apply(lambda arn: f"{arn}/*"),
],
),
aws.iam.GetPolicyDocumentStatementArgs(
effect="Allow",
actions=[
"ec2:DescribeVpcs",
"ec2:DescribeVpcAttribute",
"ec2:DescribeSubnets",
"ec2:DescribeSecurityGroups",
"ec2:DescribeNetworkInterfaces",
"ec2:CreateNetworkInterface",
"ec2:CreateNetworkInterfacePermission",
"ec2:DeleteNetworkInterface",
],
resources=["*"],
),
])
firehose_elasticsearch_role_policy = aws.iam.RolePolicy("firehose-elasticsearchRolePolicy",
role=aws_iam_role["firehose"]["id"],
policy=firehose_elasticsearch_policy_document.json)
test = aws.kinesis.FirehoseDeliveryStream("test",
destination="elasticsearch",
s3_configuration=aws.kinesis.FirehoseDeliveryStreamS3ConfigurationArgs(
role_arn=aws_iam_role["firehose"]["arn"],
bucket_arn=aws_s3_bucket["bucket"]["arn"],
),
elasticsearch_configuration=aws.kinesis.FirehoseDeliveryStreamElasticsearchConfigurationArgs(
domain_arn=test_cluster.arn,
role_arn=aws_iam_role["firehose"]["arn"],
index_name="test",
type_name="test",
vpc_config=aws.kinesis.FirehoseDeliveryStreamElasticsearchConfigurationVpcConfigArgs(
subnet_ids=[
aws_subnet["first"]["id"],
aws_subnet["second"]["id"],
],
security_group_ids=[aws_security_group["first"]["id"]],
role_arn=aws_iam_role["firehose"]["arn"],
),
),
opts=pulumi.ResourceOptions(depends_on=[firehose_elasticsearch_role_policy]))
resources:
testCluster:
type: aws:elasticsearch:Domain
properties:
clusterConfig:
instanceCount: 2
zoneAwarenessEnabled: true
instanceType: t2.small.elasticsearch
ebsOptions:
ebsEnabled: true
volumeSize: 10
vpcOptions:
securityGroupIds:
- ${aws_security_group.first.id}
subnetIds:
- ${aws_subnet.first.id}
- ${aws_subnet.second.id}
firehose-elasticsearchRolePolicy:
type: aws:iam:RolePolicy
properties:
role: ${aws_iam_role.firehose.id}
policy: ${["firehose-elasticsearchPolicyDocument"].json}
test:
type: aws:kinesis:FirehoseDeliveryStream
properties:
destination: elasticsearch
s3Configuration:
roleArn: ${aws_iam_role.firehose.arn}
bucketArn: ${aws_s3_bucket.bucket.arn}
elasticsearchConfiguration:
domainArn: ${testCluster.arn}
roleArn: ${aws_iam_role.firehose.arn}
indexName: test
typeName: test
vpcConfig:
subnetIds:
- ${aws_subnet.first.id}
- ${aws_subnet.second.id}
securityGroupIds:
- ${aws_security_group.first.id}
roleArn: ${aws_iam_role.firehose.arn}
options:
dependson:
- ${["firehose-elasticsearchRolePolicy"]}
variables:
firehose-elasticsearchPolicyDocument:
fn::invoke:
Function: aws:iam:getPolicyDocument
Arguments:
statements:
- effect: Allow
actions:
- es:*
resources:
- ${testCluster.arn}
- ${testCluster.arn}/*
- effect: Allow
actions:
- ec2:DescribeVpcs
- ec2:DescribeVpcAttribute
- ec2:DescribeSubnets
- ec2:DescribeSecurityGroups
- ec2:DescribeNetworkInterfaces
- ec2:CreateNetworkInterface
- ec2:CreateNetworkInterfacePermission
- ec2:DeleteNetworkInterface
resources:
- '*'
Opensearch Destination
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Aws = Pulumi.Aws;
return await Deployment.RunAsync(() =>
{
var testCluster = new Aws.OpenSearch.Domain("testCluster");
var testStream = new Aws.Kinesis.FirehoseDeliveryStream("testStream", new()
{
Destination = "opensearch",
S3Configuration = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamS3ConfigurationArgs
{
RoleArn = aws_iam_role.Firehose_role.Arn,
BucketArn = aws_s3_bucket.Bucket.Arn,
BufferSize = 10,
BufferInterval = 400,
CompressionFormat = "GZIP",
},
OpensearchConfiguration = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamOpensearchConfigurationArgs
{
DomainArn = testCluster.Arn,
RoleArn = aws_iam_role.Firehose_role.Arn,
IndexName = "test",
ProcessingConfiguration = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamOpensearchConfigurationProcessingConfigurationArgs
{
Enabled = true,
Processors = new[]
{
new Aws.Kinesis.Inputs.FirehoseDeliveryStreamOpensearchConfigurationProcessingConfigurationProcessorArgs
{
Type = "Lambda",
Parameters = new[]
{
new Aws.Kinesis.Inputs.FirehoseDeliveryStreamOpensearchConfigurationProcessingConfigurationProcessorParameterArgs
{
ParameterName = "LambdaArn",
ParameterValue = $"{aws_lambda_function.Lambda_processor.Arn}:$LATEST",
},
},
},
},
},
},
});
});
package main
import (
"fmt"
"github.com/pulumi/pulumi-aws/sdk/v5/go/aws/kinesis"
"github.com/pulumi/pulumi-aws/sdk/v5/go/aws/opensearch"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
testCluster, err := opensearch.NewDomain(ctx, "testCluster", nil)
if err != nil {
return err
}
_, err = kinesis.NewFirehoseDeliveryStream(ctx, "testStream", &kinesis.FirehoseDeliveryStreamArgs{
Destination: pulumi.String("opensearch"),
S3Configuration: &kinesis.FirehoseDeliveryStreamS3ConfigurationArgs{
RoleArn: pulumi.Any(aws_iam_role.Firehose_role.Arn),
BucketArn: pulumi.Any(aws_s3_bucket.Bucket.Arn),
BufferSize: pulumi.Int(10),
BufferInterval: pulumi.Int(400),
CompressionFormat: pulumi.String("GZIP"),
},
OpensearchConfiguration: &kinesis.FirehoseDeliveryStreamOpensearchConfigurationArgs{
DomainArn: testCluster.Arn,
RoleArn: pulumi.Any(aws_iam_role.Firehose_role.Arn),
IndexName: pulumi.String("test"),
ProcessingConfiguration: &kinesis.FirehoseDeliveryStreamOpensearchConfigurationProcessingConfigurationArgs{
Enabled: pulumi.Bool(true),
Processors: kinesis.FirehoseDeliveryStreamOpensearchConfigurationProcessingConfigurationProcessorArray{
&kinesis.FirehoseDeliveryStreamOpensearchConfigurationProcessingConfigurationProcessorArgs{
Type: pulumi.String("Lambda"),
Parameters: kinesis.FirehoseDeliveryStreamOpensearchConfigurationProcessingConfigurationProcessorParameterArray{
&kinesis.FirehoseDeliveryStreamOpensearchConfigurationProcessingConfigurationProcessorParameterArgs{
ParameterName: pulumi.String("LambdaArn"),
ParameterValue: pulumi.String(fmt.Sprintf("%v:$LATEST", aws_lambda_function.Lambda_processor.Arn)),
},
},
},
},
},
},
})
if err != nil {
return err
}
return nil
})
}
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.opensearch.Domain;
import com.pulumi.aws.kinesis.FirehoseDeliveryStream;
import com.pulumi.aws.kinesis.FirehoseDeliveryStreamArgs;
import com.pulumi.aws.kinesis.inputs.FirehoseDeliveryStreamS3ConfigurationArgs;
import com.pulumi.aws.kinesis.inputs.FirehoseDeliveryStreamOpensearchConfigurationArgs;
import com.pulumi.aws.kinesis.inputs.FirehoseDeliveryStreamOpensearchConfigurationProcessingConfigurationArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var testCluster = new Domain("testCluster");
var testStream = new FirehoseDeliveryStream("testStream", FirehoseDeliveryStreamArgs.builder()
.destination("opensearch")
.s3Configuration(FirehoseDeliveryStreamS3ConfigurationArgs.builder()
.roleArn(aws_iam_role.firehose_role().arn())
.bucketArn(aws_s3_bucket.bucket().arn())
.bufferSize(10)
.bufferInterval(400)
.compressionFormat("GZIP")
.build())
.opensearchConfiguration(FirehoseDeliveryStreamOpensearchConfigurationArgs.builder()
.domainArn(testCluster.arn())
.roleArn(aws_iam_role.firehose_role().arn())
.indexName("test")
.processingConfiguration(FirehoseDeliveryStreamOpensearchConfigurationProcessingConfigurationArgs.builder()
.enabled("true")
.processors(FirehoseDeliveryStreamOpensearchConfigurationProcessingConfigurationProcessorArgs.builder()
.type("Lambda")
.parameters(FirehoseDeliveryStreamOpensearchConfigurationProcessingConfigurationProcessorParameterArgs.builder()
.parameterName("LambdaArn")
.parameterValue(String.format("%s:$LATEST", aws_lambda_function.lambda_processor().arn()))
.build())
.build())
.build())
.build())
.build());
}
}
import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
const testCluster = new aws.opensearch.Domain("testCluster", {});
const testStream = new aws.kinesis.FirehoseDeliveryStream("testStream", {
destination: "opensearch",
s3Configuration: {
roleArn: aws_iam_role.firehose_role.arn,
bucketArn: aws_s3_bucket.bucket.arn,
bufferSize: 10,
bufferInterval: 400,
compressionFormat: "GZIP",
},
opensearchConfiguration: {
domainArn: testCluster.arn,
roleArn: aws_iam_role.firehose_role.arn,
indexName: "test",
processingConfiguration: {
enabled: true,
processors: [{
type: "Lambda",
parameters: [{
parameterName: "LambdaArn",
parameterValue: `${aws_lambda_function.lambda_processor.arn}:$LATEST`,
}],
}],
},
},
});
import pulumi
import pulumi_aws as aws
test_cluster = aws.opensearch.Domain("testCluster")
test_stream = aws.kinesis.FirehoseDeliveryStream("testStream",
destination="opensearch",
s3_configuration=aws.kinesis.FirehoseDeliveryStreamS3ConfigurationArgs(
role_arn=aws_iam_role["firehose_role"]["arn"],
bucket_arn=aws_s3_bucket["bucket"]["arn"],
buffer_size=10,
buffer_interval=400,
compression_format="GZIP",
),
opensearch_configuration=aws.kinesis.FirehoseDeliveryStreamOpensearchConfigurationArgs(
domain_arn=test_cluster.arn,
role_arn=aws_iam_role["firehose_role"]["arn"],
index_name="test",
processing_configuration=aws.kinesis.FirehoseDeliveryStreamOpensearchConfigurationProcessingConfigurationArgs(
enabled=True,
processors=[aws.kinesis.FirehoseDeliveryStreamOpensearchConfigurationProcessingConfigurationProcessorArgs(
type="Lambda",
parameters=[aws.kinesis.FirehoseDeliveryStreamOpensearchConfigurationProcessingConfigurationProcessorParameterArgs(
parameter_name="LambdaArn",
parameter_value=f"{aws_lambda_function['lambda_processor']['arn']}:$LATEST",
)],
)],
),
))
resources:
testCluster:
type: aws:opensearch:Domain
testStream:
type: aws:kinesis:FirehoseDeliveryStream
properties:
destination: opensearch
s3Configuration:
roleArn: ${aws_iam_role.firehose_role.arn}
bucketArn: ${aws_s3_bucket.bucket.arn}
bufferSize: 10
bufferInterval: 400
compressionFormat: GZIP
opensearchConfiguration:
domainArn: ${testCluster.arn}
roleArn: ${aws_iam_role.firehose_role.arn}
indexName: test
processingConfiguration:
enabled: 'true'
processors:
- type: Lambda
parameters:
- parameterName: LambdaArn
parameterValue: ${aws_lambda_function.lambda_processor.arn}:$LATEST
Opensearch Destination With VPC
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Aws = Pulumi.Aws;
return await Deployment.RunAsync(() =>
{
var testCluster = new Aws.OpenSearch.Domain("testCluster", new()
{
ClusterConfig = new Aws.OpenSearch.Inputs.DomainClusterConfigArgs
{
InstanceCount = 2,
ZoneAwarenessEnabled = true,
InstanceType = "m4.large.search",
},
EbsOptions = new Aws.OpenSearch.Inputs.DomainEbsOptionsArgs
{
EbsEnabled = true,
VolumeSize = 10,
},
VpcOptions = new Aws.OpenSearch.Inputs.DomainVpcOptionsArgs
{
SecurityGroupIds = new[]
{
aws_security_group.First.Id,
},
SubnetIds = new[]
{
aws_subnet.First.Id,
aws_subnet.Second.Id,
},
},
});
var firehose_opensearch = new Aws.Iam.RolePolicy("firehose-opensearch", new()
{
Role = aws_iam_role.Firehose.Id,
Policy = Output.Tuple(testCluster.Arn, testCluster.Arn).Apply(values =>
{
var testClusterArn = values.Item1;
var testClusterArn1 = values.Item2;
return @$"{{
""Version"": ""2012-10-17"",
""Statement"": [
{{
""Effect"": ""Allow"",
""Action"": [
""es:*""
],
""Resource"": [
""{testClusterArn}"",
""{testClusterArn1}/*""
]
}},
{{
""Effect"": ""Allow"",
""Action"": [
""ec2:DescribeVpcs"",
""ec2:DescribeVpcAttribute"",
""ec2:DescribeSubnets"",
""ec2:DescribeSecurityGroups"",
""ec2:DescribeNetworkInterfaces"",
""ec2:CreateNetworkInterface"",
""ec2:CreateNetworkInterfacePermission"",
""ec2:DeleteNetworkInterface""
],
""Resource"": [
""*""
]
}}
]
}}
";
}),
});
var test = new Aws.Kinesis.FirehoseDeliveryStream("test", new()
{
Destination = "opensearch",
S3Configuration = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamS3ConfigurationArgs
{
RoleArn = aws_iam_role.Firehose.Arn,
BucketArn = aws_s3_bucket.Bucket.Arn,
},
OpensearchConfiguration = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamOpensearchConfigurationArgs
{
DomainArn = testCluster.Arn,
RoleArn = aws_iam_role.Firehose.Arn,
IndexName = "test",
VpcConfig = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamOpensearchConfigurationVpcConfigArgs
{
SubnetIds = new[]
{
aws_subnet.First.Id,
aws_subnet.Second.Id,
},
SecurityGroupIds = new[]
{
aws_security_group.First.Id,
},
RoleArn = aws_iam_role.Firehose.Arn,
},
},
}, new CustomResourceOptions
{
DependsOn = new[]
{
firehose_opensearch,
},
});
});
package main
import (
"fmt"
"github.com/pulumi/pulumi-aws/sdk/v5/go/aws/iam"
"github.com/pulumi/pulumi-aws/sdk/v5/go/aws/kinesis"
"github.com/pulumi/pulumi-aws/sdk/v5/go/aws/opensearch"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
testCluster, err := opensearch.NewDomain(ctx, "testCluster", &opensearch.DomainArgs{
ClusterConfig: &opensearch.DomainClusterConfigArgs{
InstanceCount: pulumi.Int(2),
ZoneAwarenessEnabled: pulumi.Bool(true),
InstanceType: pulumi.String("m4.large.search"),
},
EbsOptions: &opensearch.DomainEbsOptionsArgs{
EbsEnabled: pulumi.Bool(true),
VolumeSize: pulumi.Int(10),
},
VpcOptions: &opensearch.DomainVpcOptionsArgs{
SecurityGroupIds: pulumi.StringArray{
aws_security_group.First.Id,
},
SubnetIds: pulumi.StringArray{
aws_subnet.First.Id,
aws_subnet.Second.Id,
},
},
})
if err != nil {
return err
}
_, err = iam.NewRolePolicy(ctx, "firehose-opensearch", &iam.RolePolicyArgs{
Role: pulumi.Any(aws_iam_role.Firehose.Id),
Policy: pulumi.All(testCluster.Arn, testCluster.Arn).ApplyT(func(_args []interface{}) (string, error) {
testClusterArn := _args[0].(string)
testClusterArn1 := _args[1].(string)
return fmt.Sprintf(`{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"es:*"
],
"Resource": [
"%v",
"%v/*"
]
},
{
"Effect": "Allow",
"Action": [
"ec2:DescribeVpcs",
"ec2:DescribeVpcAttribute",
"ec2:DescribeSubnets",
"ec2:DescribeSecurityGroups",
"ec2:DescribeNetworkInterfaces",
"ec2:CreateNetworkInterface",
"ec2:CreateNetworkInterfacePermission",
"ec2:DeleteNetworkInterface"
],
"Resource": [
"*"
]
}
]
}
`, testClusterArn, testClusterArn1), nil
}).(pulumi.StringOutput),
})
if err != nil {
return err
}
_, err = kinesis.NewFirehoseDeliveryStream(ctx, "test", &kinesis.FirehoseDeliveryStreamArgs{
Destination: pulumi.String("opensearch"),
S3Configuration: &kinesis.FirehoseDeliveryStreamS3ConfigurationArgs{
RoleArn: pulumi.Any(aws_iam_role.Firehose.Arn),
BucketArn: pulumi.Any(aws_s3_bucket.Bucket.Arn),
},
OpensearchConfiguration: &kinesis.FirehoseDeliveryStreamOpensearchConfigurationArgs{
DomainArn: testCluster.Arn,
RoleArn: pulumi.Any(aws_iam_role.Firehose.Arn),
IndexName: pulumi.String("test"),
VpcConfig: &kinesis.FirehoseDeliveryStreamOpensearchConfigurationVpcConfigArgs{
SubnetIds: pulumi.StringArray{
aws_subnet.First.Id,
aws_subnet.Second.Id,
},
SecurityGroupIds: pulumi.StringArray{
aws_security_group.First.Id,
},
RoleArn: pulumi.Any(aws_iam_role.Firehose.Arn),
},
},
}, pulumi.DependsOn([]pulumi.Resource{
firehose_opensearch,
}))
if err != nil {
return err
}
return nil
})
}
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.opensearch.Domain;
import com.pulumi.aws.opensearch.DomainArgs;
import com.pulumi.aws.opensearch.inputs.DomainClusterConfigArgs;
import com.pulumi.aws.opensearch.inputs.DomainEbsOptionsArgs;
import com.pulumi.aws.opensearch.inputs.DomainVpcOptionsArgs;
import com.pulumi.aws.iam.RolePolicy;
import com.pulumi.aws.iam.RolePolicyArgs;
import com.pulumi.aws.kinesis.FirehoseDeliveryStream;
import com.pulumi.aws.kinesis.FirehoseDeliveryStreamArgs;
import com.pulumi.aws.kinesis.inputs.FirehoseDeliveryStreamS3ConfigurationArgs;
import com.pulumi.aws.kinesis.inputs.FirehoseDeliveryStreamOpensearchConfigurationArgs;
import com.pulumi.aws.kinesis.inputs.FirehoseDeliveryStreamOpensearchConfigurationVpcConfigArgs;
import com.pulumi.resources.CustomResourceOptions;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var testCluster = new Domain("testCluster", DomainArgs.builder()
.clusterConfig(DomainClusterConfigArgs.builder()
.instanceCount(2)
.zoneAwarenessEnabled(true)
.instanceType("m4.large.search")
.build())
.ebsOptions(DomainEbsOptionsArgs.builder()
.ebsEnabled(true)
.volumeSize(10)
.build())
.vpcOptions(DomainVpcOptionsArgs.builder()
.securityGroupIds(aws_security_group.first().id())
.subnetIds(
aws_subnet.first().id(),
aws_subnet.second().id())
.build())
.build());
var firehose_opensearch = new RolePolicy("firehose-opensearch", RolePolicyArgs.builder()
.role(aws_iam_role.firehose().id())
.policy(Output.tuple(testCluster.arn(), testCluster.arn()).applyValue(values -> {
var testClusterArn = values.t1;
var testClusterArn1 = values.t2;
return """
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"es:*"
],
"Resource": [
"%s",
"%s/*"
]
},
{
"Effect": "Allow",
"Action": [
"ec2:DescribeVpcs",
"ec2:DescribeVpcAttribute",
"ec2:DescribeSubnets",
"ec2:DescribeSecurityGroups",
"ec2:DescribeNetworkInterfaces",
"ec2:CreateNetworkInterface",
"ec2:CreateNetworkInterfacePermission",
"ec2:DeleteNetworkInterface"
],
"Resource": [
"*"
]
}
]
}
", testClusterArn,testClusterArn1);
}))
.build());
var test = new FirehoseDeliveryStream("test", FirehoseDeliveryStreamArgs.builder()
.destination("opensearch")
.s3Configuration(FirehoseDeliveryStreamS3ConfigurationArgs.builder()
.roleArn(aws_iam_role.firehose().arn())
.bucketArn(aws_s3_bucket.bucket().arn())
.build())
.opensearchConfiguration(FirehoseDeliveryStreamOpensearchConfigurationArgs.builder()
.domainArn(testCluster.arn())
.roleArn(aws_iam_role.firehose().arn())
.indexName("test")
.vpcConfig(FirehoseDeliveryStreamOpensearchConfigurationVpcConfigArgs.builder()
.subnetIds(
aws_subnet.first().id(),
aws_subnet.second().id())
.securityGroupIds(aws_security_group.first().id())
.roleArn(aws_iam_role.firehose().arn())
.build())
.build())
.build(), CustomResourceOptions.builder()
.dependsOn(firehose_opensearch)
.build());
}
}
import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
const testCluster = new aws.opensearch.Domain("testCluster", {
clusterConfig: {
instanceCount: 2,
zoneAwarenessEnabled: true,
instanceType: "m4.large.search",
},
ebsOptions: {
ebsEnabled: true,
volumeSize: 10,
},
vpcOptions: {
securityGroupIds: [aws_security_group.first.id],
subnetIds: [
aws_subnet.first.id,
aws_subnet.second.id,
],
},
});
const firehose_opensearch = new aws.iam.RolePolicy("firehose-opensearch", {
role: aws_iam_role.firehose.id,
policy: pulumi.interpolate`{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"es:*"
],
"Resource": [
"${testCluster.arn}",
"${testCluster.arn}/*"
]
},
{
"Effect": "Allow",
"Action": [
"ec2:DescribeVpcs",
"ec2:DescribeVpcAttribute",
"ec2:DescribeSubnets",
"ec2:DescribeSecurityGroups",
"ec2:DescribeNetworkInterfaces",
"ec2:CreateNetworkInterface",
"ec2:CreateNetworkInterfacePermission",
"ec2:DeleteNetworkInterface"
],
"Resource": [
"*"
]
}
]
}
`,
});
const test = new aws.kinesis.FirehoseDeliveryStream("test", {
destination: "opensearch",
s3Configuration: {
roleArn: aws_iam_role.firehose.arn,
bucketArn: aws_s3_bucket.bucket.arn,
},
opensearchConfiguration: {
domainArn: testCluster.arn,
roleArn: aws_iam_role.firehose.arn,
indexName: "test",
vpcConfig: {
subnetIds: [
aws_subnet.first.id,
aws_subnet.second.id,
],
securityGroupIds: [aws_security_group.first.id],
roleArn: aws_iam_role.firehose.arn,
},
},
}, {
dependsOn: [firehose_opensearch],
});
import pulumi
import pulumi_aws as aws
test_cluster = aws.opensearch.Domain("testCluster",
cluster_config=aws.opensearch.DomainClusterConfigArgs(
instance_count=2,
zone_awareness_enabled=True,
instance_type="m4.large.search",
),
ebs_options=aws.opensearch.DomainEbsOptionsArgs(
ebs_enabled=True,
volume_size=10,
),
vpc_options=aws.opensearch.DomainVpcOptionsArgs(
security_group_ids=[aws_security_group["first"]["id"]],
subnet_ids=[
aws_subnet["first"]["id"],
aws_subnet["second"]["id"],
],
))
firehose_opensearch = aws.iam.RolePolicy("firehose-opensearch",
role=aws_iam_role["firehose"]["id"],
policy=pulumi.Output.all(test_cluster.arn, test_cluster.arn).apply(lambda testClusterArn, testClusterArn1: f"""{{
"Version": "2012-10-17",
"Statement": [
{{
"Effect": "Allow",
"Action": [
"es:*"
],
"Resource": [
"{test_cluster_arn}",
"{test_cluster_arn1}/*"
]
}},
{{
"Effect": "Allow",
"Action": [
"ec2:DescribeVpcs",
"ec2:DescribeVpcAttribute",
"ec2:DescribeSubnets",
"ec2:DescribeSecurityGroups",
"ec2:DescribeNetworkInterfaces",
"ec2:CreateNetworkInterface",
"ec2:CreateNetworkInterfacePermission",
"ec2:DeleteNetworkInterface"
],
"Resource": [
"*"
]
}}
]
}}
"""))
test = aws.kinesis.FirehoseDeliveryStream("test",
destination="opensearch",
s3_configuration=aws.kinesis.FirehoseDeliveryStreamS3ConfigurationArgs(
role_arn=aws_iam_role["firehose"]["arn"],
bucket_arn=aws_s3_bucket["bucket"]["arn"],
),
opensearch_configuration=aws.kinesis.FirehoseDeliveryStreamOpensearchConfigurationArgs(
domain_arn=test_cluster.arn,
role_arn=aws_iam_role["firehose"]["arn"],
index_name="test",
vpc_config=aws.kinesis.FirehoseDeliveryStreamOpensearchConfigurationVpcConfigArgs(
subnet_ids=[
aws_subnet["first"]["id"],
aws_subnet["second"]["id"],
],
security_group_ids=[aws_security_group["first"]["id"]],
role_arn=aws_iam_role["firehose"]["arn"],
),
),
opts=pulumi.ResourceOptions(depends_on=[firehose_opensearch]))
resources:
testCluster:
type: aws:opensearch:Domain
properties:
clusterConfig:
instanceCount: 2
zoneAwarenessEnabled: true
instanceType: m4.large.search
ebsOptions:
ebsEnabled: true
volumeSize: 10
vpcOptions:
securityGroupIds:
- ${aws_security_group.first.id}
subnetIds:
- ${aws_subnet.first.id}
- ${aws_subnet.second.id}
firehose-opensearch:
type: aws:iam:RolePolicy
properties:
role: ${aws_iam_role.firehose.id}
policy: |
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"es:*"
],
"Resource": [
"${testCluster.arn}",
"${testCluster.arn}/*"
]
},
{
"Effect": "Allow",
"Action": [
"ec2:DescribeVpcs",
"ec2:DescribeVpcAttribute",
"ec2:DescribeSubnets",
"ec2:DescribeSecurityGroups",
"ec2:DescribeNetworkInterfaces",
"ec2:CreateNetworkInterface",
"ec2:CreateNetworkInterfacePermission",
"ec2:DeleteNetworkInterface"
],
"Resource": [
"*"
]
}
]
}
test:
type: aws:kinesis:FirehoseDeliveryStream
properties:
destination: opensearch
s3Configuration:
roleArn: ${aws_iam_role.firehose.arn}
bucketArn: ${aws_s3_bucket.bucket.arn}
opensearchConfiguration:
domainArn: ${testCluster.arn}
roleArn: ${aws_iam_role.firehose.arn}
indexName: test
vpcConfig:
subnetIds:
- ${aws_subnet.first.id}
- ${aws_subnet.second.id}
securityGroupIds:
- ${aws_security_group.first.id}
roleArn: ${aws_iam_role.firehose.arn}
options:
dependson:
- ${["firehose-opensearch"]}
Splunk Destination
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Aws = Pulumi.Aws;
return await Deployment.RunAsync(() =>
{
var testStream = new Aws.Kinesis.FirehoseDeliveryStream("testStream", new()
{
Destination = "splunk",
S3Configuration = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamS3ConfigurationArgs
{
RoleArn = aws_iam_role.Firehose.Arn,
BucketArn = aws_s3_bucket.Bucket.Arn,
BufferSize = 10,
BufferInterval = 400,
CompressionFormat = "GZIP",
},
SplunkConfiguration = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamSplunkConfigurationArgs
{
HecEndpoint = "https://http-inputs-mydomain.splunkcloud.com:443",
HecToken = "51D4DA16-C61B-4F5F-8EC7-ED4301342A4A",
HecAcknowledgmentTimeout = 600,
HecEndpointType = "Event",
S3BackupMode = "FailedEventsOnly",
},
});
});
package main
import (
"github.com/pulumi/pulumi-aws/sdk/v5/go/aws/kinesis"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
_, err := kinesis.NewFirehoseDeliveryStream(ctx, "testStream", &kinesis.FirehoseDeliveryStreamArgs{
Destination: pulumi.String("splunk"),
S3Configuration: &kinesis.FirehoseDeliveryStreamS3ConfigurationArgs{
RoleArn: pulumi.Any(aws_iam_role.Firehose.Arn),
BucketArn: pulumi.Any(aws_s3_bucket.Bucket.Arn),
BufferSize: pulumi.Int(10),
BufferInterval: pulumi.Int(400),
CompressionFormat: pulumi.String("GZIP"),
},
SplunkConfiguration: &kinesis.FirehoseDeliveryStreamSplunkConfigurationArgs{
HecEndpoint: pulumi.String("https://http-inputs-mydomain.splunkcloud.com:443"),
HecToken: pulumi.String("51D4DA16-C61B-4F5F-8EC7-ED4301342A4A"),
HecAcknowledgmentTimeout: pulumi.Int(600),
HecEndpointType: pulumi.String("Event"),
S3BackupMode: pulumi.String("FailedEventsOnly"),
},
})
if err != nil {
return err
}
return nil
})
}
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.kinesis.FirehoseDeliveryStream;
import com.pulumi.aws.kinesis.FirehoseDeliveryStreamArgs;
import com.pulumi.aws.kinesis.inputs.FirehoseDeliveryStreamS3ConfigurationArgs;
import com.pulumi.aws.kinesis.inputs.FirehoseDeliveryStreamSplunkConfigurationArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var testStream = new FirehoseDeliveryStream("testStream", FirehoseDeliveryStreamArgs.builder()
.destination("splunk")
.s3Configuration(FirehoseDeliveryStreamS3ConfigurationArgs.builder()
.roleArn(aws_iam_role.firehose().arn())
.bucketArn(aws_s3_bucket.bucket().arn())
.bufferSize(10)
.bufferInterval(400)
.compressionFormat("GZIP")
.build())
.splunkConfiguration(FirehoseDeliveryStreamSplunkConfigurationArgs.builder()
.hecEndpoint("https://http-inputs-mydomain.splunkcloud.com:443")
.hecToken("51D4DA16-C61B-4F5F-8EC7-ED4301342A4A")
.hecAcknowledgmentTimeout(600)
.hecEndpointType("Event")
.s3BackupMode("FailedEventsOnly")
.build())
.build());
}
}
import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
const testStream = new aws.kinesis.FirehoseDeliveryStream("testStream", {
destination: "splunk",
s3Configuration: {
roleArn: aws_iam_role.firehose.arn,
bucketArn: aws_s3_bucket.bucket.arn,
bufferSize: 10,
bufferInterval: 400,
compressionFormat: "GZIP",
},
splunkConfiguration: {
hecEndpoint: "https://http-inputs-mydomain.splunkcloud.com:443",
hecToken: "51D4DA16-C61B-4F5F-8EC7-ED4301342A4A",
hecAcknowledgmentTimeout: 600,
hecEndpointType: "Event",
s3BackupMode: "FailedEventsOnly",
},
});
import pulumi
import pulumi_aws as aws
test_stream = aws.kinesis.FirehoseDeliveryStream("testStream",
destination="splunk",
s3_configuration=aws.kinesis.FirehoseDeliveryStreamS3ConfigurationArgs(
role_arn=aws_iam_role["firehose"]["arn"],
bucket_arn=aws_s3_bucket["bucket"]["arn"],
buffer_size=10,
buffer_interval=400,
compression_format="GZIP",
),
splunk_configuration=aws.kinesis.FirehoseDeliveryStreamSplunkConfigurationArgs(
hec_endpoint="https://http-inputs-mydomain.splunkcloud.com:443",
hec_token="51D4DA16-C61B-4F5F-8EC7-ED4301342A4A",
hec_acknowledgment_timeout=600,
hec_endpoint_type="Event",
s3_backup_mode="FailedEventsOnly",
))
resources:
testStream:
type: aws:kinesis:FirehoseDeliveryStream
properties:
destination: splunk
s3Configuration:
roleArn: ${aws_iam_role.firehose.arn}
bucketArn: ${aws_s3_bucket.bucket.arn}
bufferSize: 10
bufferInterval: 400
compressionFormat: GZIP
splunkConfiguration:
hecEndpoint: https://http-inputs-mydomain.splunkcloud.com:443
hecToken: 51D4DA16-C61B-4F5F-8EC7-ED4301342A4A
hecAcknowledgmentTimeout: 600
hecEndpointType: Event
s3BackupMode: FailedEventsOnly
HTTP Endpoint (e.g., New Relic) Destination
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Aws = Pulumi.Aws;
return await Deployment.RunAsync(() =>
{
var testStream = new Aws.Kinesis.FirehoseDeliveryStream("testStream", new()
{
Destination = "http_endpoint",
S3Configuration = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamS3ConfigurationArgs
{
RoleArn = aws_iam_role.Firehose.Arn,
BucketArn = aws_s3_bucket.Bucket.Arn,
BufferSize = 10,
BufferInterval = 400,
CompressionFormat = "GZIP",
},
HttpEndpointConfiguration = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamHttpEndpointConfigurationArgs
{
Url = "https://aws-api.newrelic.com/firehose/v1",
Name = "New Relic",
AccessKey = "my-key",
BufferingSize = 15,
BufferingInterval = 600,
RoleArn = aws_iam_role.Firehose.Arn,
S3BackupMode = "FailedDataOnly",
RequestConfiguration = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamHttpEndpointConfigurationRequestConfigurationArgs
{
ContentEncoding = "GZIP",
CommonAttributes = new[]
{
new Aws.Kinesis.Inputs.FirehoseDeliveryStreamHttpEndpointConfigurationRequestConfigurationCommonAttributeArgs
{
Name = "testname",
Value = "testvalue",
},
new Aws.Kinesis.Inputs.FirehoseDeliveryStreamHttpEndpointConfigurationRequestConfigurationCommonAttributeArgs
{
Name = "testname2",
Value = "testvalue2",
},
},
},
},
});
});
package main
import (
"github.com/pulumi/pulumi-aws/sdk/v5/go/aws/kinesis"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
_, err := kinesis.NewFirehoseDeliveryStream(ctx, "testStream", &kinesis.FirehoseDeliveryStreamArgs{
Destination: pulumi.String("http_endpoint"),
S3Configuration: &kinesis.FirehoseDeliveryStreamS3ConfigurationArgs{
RoleArn: pulumi.Any(aws_iam_role.Firehose.Arn),
BucketArn: pulumi.Any(aws_s3_bucket.Bucket.Arn),
BufferSize: pulumi.Int(10),
BufferInterval: pulumi.Int(400),
CompressionFormat: pulumi.String("GZIP"),
},
HttpEndpointConfiguration: &kinesis.FirehoseDeliveryStreamHttpEndpointConfigurationArgs{
Url: pulumi.String("https://aws-api.newrelic.com/firehose/v1"),
Name: pulumi.String("New Relic"),
AccessKey: pulumi.String("my-key"),
BufferingSize: pulumi.Int(15),
BufferingInterval: pulumi.Int(600),
RoleArn: pulumi.Any(aws_iam_role.Firehose.Arn),
S3BackupMode: pulumi.String("FailedDataOnly"),
RequestConfiguration: &kinesis.FirehoseDeliveryStreamHttpEndpointConfigurationRequestConfigurationArgs{
ContentEncoding: pulumi.String("GZIP"),
CommonAttributes: kinesis.FirehoseDeliveryStreamHttpEndpointConfigurationRequestConfigurationCommonAttributeArray{
&kinesis.FirehoseDeliveryStreamHttpEndpointConfigurationRequestConfigurationCommonAttributeArgs{
Name: pulumi.String("testname"),
Value: pulumi.String("testvalue"),
},
&kinesis.FirehoseDeliveryStreamHttpEndpointConfigurationRequestConfigurationCommonAttributeArgs{
Name: pulumi.String("testname2"),
Value: pulumi.String("testvalue2"),
},
},
},
},
})
if err != nil {
return err
}
return nil
})
}
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.kinesis.FirehoseDeliveryStream;
import com.pulumi.aws.kinesis.FirehoseDeliveryStreamArgs;
import com.pulumi.aws.kinesis.inputs.FirehoseDeliveryStreamS3ConfigurationArgs;
import com.pulumi.aws.kinesis.inputs.FirehoseDeliveryStreamHttpEndpointConfigurationArgs;
import com.pulumi.aws.kinesis.inputs.FirehoseDeliveryStreamHttpEndpointConfigurationRequestConfigurationArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var testStream = new FirehoseDeliveryStream("testStream", FirehoseDeliveryStreamArgs.builder()
.destination("http_endpoint")
.s3Configuration(FirehoseDeliveryStreamS3ConfigurationArgs.builder()
.roleArn(aws_iam_role.firehose().arn())
.bucketArn(aws_s3_bucket.bucket().arn())
.bufferSize(10)
.bufferInterval(400)
.compressionFormat("GZIP")
.build())
.httpEndpointConfiguration(FirehoseDeliveryStreamHttpEndpointConfigurationArgs.builder()
.url("https://aws-api.newrelic.com/firehose/v1")
.name("New Relic")
.accessKey("my-key")
.bufferingSize(15)
.bufferingInterval(600)
.roleArn(aws_iam_role.firehose().arn())
.s3BackupMode("FailedDataOnly")
.requestConfiguration(FirehoseDeliveryStreamHttpEndpointConfigurationRequestConfigurationArgs.builder()
.contentEncoding("GZIP")
.commonAttributes(
FirehoseDeliveryStreamHttpEndpointConfigurationRequestConfigurationCommonAttributeArgs.builder()
.name("testname")
.value("testvalue")
.build(),
FirehoseDeliveryStreamHttpEndpointConfigurationRequestConfigurationCommonAttributeArgs.builder()
.name("testname2")
.value("testvalue2")
.build())
.build())
.build())
.build());
}
}
import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
const testStream = new aws.kinesis.FirehoseDeliveryStream("testStream", {
destination: "http_endpoint",
s3Configuration: {
roleArn: aws_iam_role.firehose.arn,
bucketArn: aws_s3_bucket.bucket.arn,
bufferSize: 10,
bufferInterval: 400,
compressionFormat: "GZIP",
},
httpEndpointConfiguration: {
url: "https://aws-api.newrelic.com/firehose/v1",
name: "New Relic",
accessKey: "my-key",
bufferingSize: 15,
bufferingInterval: 600,
roleArn: aws_iam_role.firehose.arn,
s3BackupMode: "FailedDataOnly",
requestConfiguration: {
contentEncoding: "GZIP",
commonAttributes: [
{
name: "testname",
value: "testvalue",
},
{
name: "testname2",
value: "testvalue2",
},
],
},
},
});
import pulumi
import pulumi_aws as aws
test_stream = aws.kinesis.FirehoseDeliveryStream("testStream",
destination="http_endpoint",
s3_configuration=aws.kinesis.FirehoseDeliveryStreamS3ConfigurationArgs(
role_arn=aws_iam_role["firehose"]["arn"],
bucket_arn=aws_s3_bucket["bucket"]["arn"],
buffer_size=10,
buffer_interval=400,
compression_format="GZIP",
),
http_endpoint_configuration=aws.kinesis.FirehoseDeliveryStreamHttpEndpointConfigurationArgs(
url="https://aws-api.newrelic.com/firehose/v1",
name="New Relic",
access_key="my-key",
buffering_size=15,
buffering_interval=600,
role_arn=aws_iam_role["firehose"]["arn"],
s3_backup_mode="FailedDataOnly",
request_configuration=aws.kinesis.FirehoseDeliveryStreamHttpEndpointConfigurationRequestConfigurationArgs(
content_encoding="GZIP",
common_attributes=[
aws.kinesis.FirehoseDeliveryStreamHttpEndpointConfigurationRequestConfigurationCommonAttributeArgs(
name="testname",
value="testvalue",
),
aws.kinesis.FirehoseDeliveryStreamHttpEndpointConfigurationRequestConfigurationCommonAttributeArgs(
name="testname2",
value="testvalue2",
),
],
),
))
resources:
testStream:
type: aws:kinesis:FirehoseDeliveryStream
properties:
destination: http_endpoint
s3Configuration:
roleArn: ${aws_iam_role.firehose.arn}
bucketArn: ${aws_s3_bucket.bucket.arn}
bufferSize: 10
bufferInterval: 400
compressionFormat: GZIP
httpEndpointConfiguration:
url: https://aws-api.newrelic.com/firehose/v1
name: New Relic
accessKey: my-key
bufferingSize: 15
bufferingInterval: 600
roleArn: ${aws_iam_role.firehose.arn}
s3BackupMode: FailedDataOnly
requestConfiguration:
contentEncoding: GZIP
commonAttributes:
- name: testname
value: testvalue
- name: testname2
value: testvalue2
Create FirehoseDeliveryStream Resource
Resources are created with functions called constructors. To learn more about declaring and configuring resources, see Resources.
Constructor syntax
new FirehoseDeliveryStream(name: string, args: FirehoseDeliveryStreamArgs, opts?: CustomResourceOptions);@overload
def FirehoseDeliveryStream(resource_name: str,
args: FirehoseDeliveryStreamArgs,
opts: Optional[ResourceOptions] = None)
@overload
def FirehoseDeliveryStream(resource_name: str,
opts: Optional[ResourceOptions] = None,
destination: Optional[str] = None,
kinesis_source_configuration: Optional[FirehoseDeliveryStreamKinesisSourceConfigurationArgs] = None,
destination_id: Optional[str] = None,
elasticsearch_configuration: Optional[FirehoseDeliveryStreamElasticsearchConfigurationArgs] = None,
extended_s3_configuration: Optional[FirehoseDeliveryStreamExtendedS3ConfigurationArgs] = None,
http_endpoint_configuration: Optional[FirehoseDeliveryStreamHttpEndpointConfigurationArgs] = None,
arn: Optional[str] = None,
name: Optional[str] = None,
opensearch_configuration: Optional[FirehoseDeliveryStreamOpensearchConfigurationArgs] = None,
redshift_configuration: Optional[FirehoseDeliveryStreamRedshiftConfigurationArgs] = None,
s3_configuration: Optional[FirehoseDeliveryStreamS3ConfigurationArgs] = None,
server_side_encryption: Optional[FirehoseDeliveryStreamServerSideEncryptionArgs] = None,
splunk_configuration: Optional[FirehoseDeliveryStreamSplunkConfigurationArgs] = None,
tags: Optional[Mapping[str, str]] = None,
version_id: Optional[str] = None)func NewFirehoseDeliveryStream(ctx *Context, name string, args FirehoseDeliveryStreamArgs, opts ...ResourceOption) (*FirehoseDeliveryStream, error)public FirehoseDeliveryStream(string name, FirehoseDeliveryStreamArgs args, CustomResourceOptions? opts = null)
public FirehoseDeliveryStream(String name, FirehoseDeliveryStreamArgs args)
public FirehoseDeliveryStream(String name, FirehoseDeliveryStreamArgs args, CustomResourceOptions options)
type: aws:kinesis:FirehoseDeliveryStream
properties: # The arguments to resource properties.
options: # Bag of options to control resource's behavior.
Parameters
- name string
- The unique name of the resource.
- args FirehoseDeliveryStreamArgs
- The arguments to resource properties.
- opts CustomResourceOptions
- Bag of options to control resource's behavior.
- resource_name str
- The unique name of the resource.
- args FirehoseDeliveryStreamArgs
- The arguments to resource properties.
- opts ResourceOptions
- Bag of options to control resource's behavior.
- ctx Context
- Context object for the current deployment.
- name string
- The unique name of the resource.
- args FirehoseDeliveryStreamArgs
- The arguments to resource properties.
- opts ResourceOption
- Bag of options to control resource's behavior.
- name string
- The unique name of the resource.
- args FirehoseDeliveryStreamArgs
- The arguments to resource properties.
- opts CustomResourceOptions
- Bag of options to control resource's behavior.
- name String
- The unique name of the resource.
- args FirehoseDeliveryStreamArgs
- The arguments to resource properties.
- options CustomResourceOptions
- Bag of options to control resource's behavior.
Constructor example
The following reference example uses placeholder values for all input properties.
var firehoseDeliveryStreamResource = new Aws.Kinesis.FirehoseDeliveryStream("firehoseDeliveryStreamResource", new()
{
Destination = "string",
KinesisSourceConfiguration = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamKinesisSourceConfigurationArgs
{
KinesisStreamArn = "string",
RoleArn = "string",
},
DestinationId = "string",
ElasticsearchConfiguration = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamElasticsearchConfigurationArgs
{
IndexName = "string",
RoleArn = "string",
ClusterEndpoint = "string",
BufferingInterval = 0,
DomainArn = "string",
CloudwatchLoggingOptions = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamElasticsearchConfigurationCloudwatchLoggingOptionsArgs
{
Enabled = false,
LogGroupName = "string",
LogStreamName = "string",
},
IndexRotationPeriod = "string",
ProcessingConfiguration = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamElasticsearchConfigurationProcessingConfigurationArgs
{
Enabled = false,
Processors = new[]
{
new Aws.Kinesis.Inputs.FirehoseDeliveryStreamElasticsearchConfigurationProcessingConfigurationProcessorArgs
{
Type = "string",
Parameters = new[]
{
new Aws.Kinesis.Inputs.FirehoseDeliveryStreamElasticsearchConfigurationProcessingConfigurationProcessorParameterArgs
{
ParameterName = "string",
ParameterValue = "string",
},
},
},
},
},
RetryDuration = 0,
BufferingSize = 0,
S3BackupMode = "string",
TypeName = "string",
VpcConfig = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamElasticsearchConfigurationVpcConfigArgs
{
RoleArn = "string",
SecurityGroupIds = new[]
{
"string",
},
SubnetIds = new[]
{
"string",
},
VpcId = "string",
},
},
ExtendedS3Configuration = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamExtendedS3ConfigurationArgs
{
BucketArn = "string",
RoleArn = "string",
DynamicPartitioningConfiguration = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamExtendedS3ConfigurationDynamicPartitioningConfigurationArgs
{
Enabled = false,
RetryDuration = 0,
},
CloudwatchLoggingOptions = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamExtendedS3ConfigurationCloudwatchLoggingOptionsArgs
{
Enabled = false,
LogGroupName = "string",
LogStreamName = "string",
},
CompressionFormat = "string",
DataFormatConversionConfiguration = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamExtendedS3ConfigurationDataFormatConversionConfigurationArgs
{
InputFormatConfiguration = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamExtendedS3ConfigurationDataFormatConversionConfigurationInputFormatConfigurationArgs
{
Deserializer = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamExtendedS3ConfigurationDataFormatConversionConfigurationInputFormatConfigurationDeserializerArgs
{
HiveJsonSerDe = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamExtendedS3ConfigurationDataFormatConversionConfigurationInputFormatConfigurationDeserializerHiveJsonSerDeArgs
{
TimestampFormats = new[]
{
"string",
},
},
OpenXJsonSerDe = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamExtendedS3ConfigurationDataFormatConversionConfigurationInputFormatConfigurationDeserializerOpenXJsonSerDeArgs
{
CaseInsensitive = false,
ColumnToJsonKeyMappings =
{
{ "string", "string" },
},
ConvertDotsInJsonKeysToUnderscores = false,
},
},
},
OutputFormatConfiguration = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamExtendedS3ConfigurationDataFormatConversionConfigurationOutputFormatConfigurationArgs
{
Serializer = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamExtendedS3ConfigurationDataFormatConversionConfigurationOutputFormatConfigurationSerializerArgs
{
OrcSerDe = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamExtendedS3ConfigurationDataFormatConversionConfigurationOutputFormatConfigurationSerializerOrcSerDeArgs
{
BlockSizeBytes = 0,
BloomFilterColumns = new[]
{
"string",
},
BloomFilterFalsePositiveProbability = 0,
Compression = "string",
DictionaryKeyThreshold = 0,
EnablePadding = false,
FormatVersion = "string",
PaddingTolerance = 0,
RowIndexStride = 0,
StripeSizeBytes = 0,
},
ParquetSerDe = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamExtendedS3ConfigurationDataFormatConversionConfigurationOutputFormatConfigurationSerializerParquetSerDeArgs
{
BlockSizeBytes = 0,
Compression = "string",
EnableDictionaryCompression = false,
MaxPaddingBytes = 0,
PageSizeBytes = 0,
WriterVersion = "string",
},
},
},
SchemaConfiguration = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamExtendedS3ConfigurationDataFormatConversionConfigurationSchemaConfigurationArgs
{
DatabaseName = "string",
RoleArn = "string",
TableName = "string",
CatalogId = "string",
Region = "string",
VersionId = "string",
},
Enabled = false,
},
BufferSize = 0,
ErrorOutputPrefix = "string",
KmsKeyArn = "string",
Prefix = "string",
ProcessingConfiguration = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationArgs
{
Enabled = false,
Processors = new[]
{
new Aws.Kinesis.Inputs.FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationProcessorArgs
{
Type = "string",
Parameters = new[]
{
new Aws.Kinesis.Inputs.FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationProcessorParameterArgs
{
ParameterName = "string",
ParameterValue = "string",
},
},
},
},
},
BufferInterval = 0,
S3BackupConfiguration = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamExtendedS3ConfigurationS3BackupConfigurationArgs
{
BucketArn = "string",
RoleArn = "string",
BufferInterval = 0,
BufferSize = 0,
CloudwatchLoggingOptions = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamExtendedS3ConfigurationS3BackupConfigurationCloudwatchLoggingOptionsArgs
{
Enabled = false,
LogGroupName = "string",
LogStreamName = "string",
},
CompressionFormat = "string",
ErrorOutputPrefix = "string",
KmsKeyArn = "string",
Prefix = "string",
},
S3BackupMode = "string",
},
HttpEndpointConfiguration = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamHttpEndpointConfigurationArgs
{
Url = "string",
AccessKey = "string",
BufferingInterval = 0,
BufferingSize = 0,
CloudwatchLoggingOptions = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamHttpEndpointConfigurationCloudwatchLoggingOptionsArgs
{
Enabled = false,
LogGroupName = "string",
LogStreamName = "string",
},
Name = "string",
ProcessingConfiguration = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamHttpEndpointConfigurationProcessingConfigurationArgs
{
Enabled = false,
Processors = new[]
{
new Aws.Kinesis.Inputs.FirehoseDeliveryStreamHttpEndpointConfigurationProcessingConfigurationProcessorArgs
{
Type = "string",
Parameters = new[]
{
new Aws.Kinesis.Inputs.FirehoseDeliveryStreamHttpEndpointConfigurationProcessingConfigurationProcessorParameterArgs
{
ParameterName = "string",
ParameterValue = "string",
},
},
},
},
},
RequestConfiguration = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamHttpEndpointConfigurationRequestConfigurationArgs
{
CommonAttributes = new[]
{
new Aws.Kinesis.Inputs.FirehoseDeliveryStreamHttpEndpointConfigurationRequestConfigurationCommonAttributeArgs
{
Name = "string",
Value = "string",
},
},
ContentEncoding = "string",
},
RetryDuration = 0,
RoleArn = "string",
S3BackupMode = "string",
},
Arn = "string",
Name = "string",
OpensearchConfiguration = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamOpensearchConfigurationArgs
{
IndexName = "string",
RoleArn = "string",
ClusterEndpoint = "string",
BufferingInterval = 0,
DomainArn = "string",
CloudwatchLoggingOptions = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamOpensearchConfigurationCloudwatchLoggingOptionsArgs
{
Enabled = false,
LogGroupName = "string",
LogStreamName = "string",
},
IndexRotationPeriod = "string",
ProcessingConfiguration = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamOpensearchConfigurationProcessingConfigurationArgs
{
Enabled = false,
Processors = new[]
{
new Aws.Kinesis.Inputs.FirehoseDeliveryStreamOpensearchConfigurationProcessingConfigurationProcessorArgs
{
Type = "string",
Parameters = new[]
{
new Aws.Kinesis.Inputs.FirehoseDeliveryStreamOpensearchConfigurationProcessingConfigurationProcessorParameterArgs
{
ParameterName = "string",
ParameterValue = "string",
},
},
},
},
},
RetryDuration = 0,
BufferingSize = 0,
S3BackupMode = "string",
TypeName = "string",
VpcConfig = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamOpensearchConfigurationVpcConfigArgs
{
RoleArn = "string",
SecurityGroupIds = new[]
{
"string",
},
SubnetIds = new[]
{
"string",
},
VpcId = "string",
},
},
RedshiftConfiguration = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamRedshiftConfigurationArgs
{
ClusterJdbcurl = "string",
DataTableName = "string",
Password = "string",
RoleArn = "string",
Username = "string",
CloudwatchLoggingOptions = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamRedshiftConfigurationCloudwatchLoggingOptionsArgs
{
Enabled = false,
LogGroupName = "string",
LogStreamName = "string",
},
CopyOptions = "string",
DataTableColumns = "string",
ProcessingConfiguration = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamRedshiftConfigurationProcessingConfigurationArgs
{
Enabled = false,
Processors = new[]
{
new Aws.Kinesis.Inputs.FirehoseDeliveryStreamRedshiftConfigurationProcessingConfigurationProcessorArgs
{
Type = "string",
Parameters = new[]
{
new Aws.Kinesis.Inputs.FirehoseDeliveryStreamRedshiftConfigurationProcessingConfigurationProcessorParameterArgs
{
ParameterName = "string",
ParameterValue = "string",
},
},
},
},
},
RetryDuration = 0,
S3BackupConfiguration = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamRedshiftConfigurationS3BackupConfigurationArgs
{
BucketArn = "string",
RoleArn = "string",
BufferInterval = 0,
BufferSize = 0,
CloudwatchLoggingOptions = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamRedshiftConfigurationS3BackupConfigurationCloudwatchLoggingOptionsArgs
{
Enabled = false,
LogGroupName = "string",
LogStreamName = "string",
},
CompressionFormat = "string",
ErrorOutputPrefix = "string",
KmsKeyArn = "string",
Prefix = "string",
},
S3BackupMode = "string",
},
S3Configuration = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamS3ConfigurationArgs
{
BucketArn = "string",
RoleArn = "string",
BufferInterval = 0,
BufferSize = 0,
CloudwatchLoggingOptions = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamS3ConfigurationCloudwatchLoggingOptionsArgs
{
Enabled = false,
LogGroupName = "string",
LogStreamName = "string",
},
CompressionFormat = "string",
ErrorOutputPrefix = "string",
KmsKeyArn = "string",
Prefix = "string",
},
ServerSideEncryption = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamServerSideEncryptionArgs
{
Enabled = false,
KeyArn = "string",
KeyType = "string",
},
SplunkConfiguration = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamSplunkConfigurationArgs
{
HecEndpoint = "string",
HecToken = "string",
CloudwatchLoggingOptions = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamSplunkConfigurationCloudwatchLoggingOptionsArgs
{
Enabled = false,
LogGroupName = "string",
LogStreamName = "string",
},
HecAcknowledgmentTimeout = 0,
HecEndpointType = "string",
ProcessingConfiguration = new Aws.Kinesis.Inputs.FirehoseDeliveryStreamSplunkConfigurationProcessingConfigurationArgs
{
Enabled = false,
Processors = new[]
{
new Aws.Kinesis.Inputs.FirehoseDeliveryStreamSplunkConfigurationProcessingConfigurationProcessorArgs
{
Type = "string",
Parameters = new[]
{
new Aws.Kinesis.Inputs.FirehoseDeliveryStreamSplunkConfigurationProcessingConfigurationProcessorParameterArgs
{
ParameterName = "string",
ParameterValue = "string",
},
},
},
},
},
RetryDuration = 0,
S3BackupMode = "string",
},
Tags =
{
{ "string", "string" },
},
VersionId = "string",
});
example, err := kinesis.NewFirehoseDeliveryStream(ctx, "firehoseDeliveryStreamResource", &kinesis.FirehoseDeliveryStreamArgs{
Destination: pulumi.String("string"),
KinesisSourceConfiguration: &kinesis.FirehoseDeliveryStreamKinesisSourceConfigurationArgs{
KinesisStreamArn: pulumi.String("string"),
RoleArn: pulumi.String("string"),
},
DestinationId: pulumi.String("string"),
ElasticsearchConfiguration: &kinesis.FirehoseDeliveryStreamElasticsearchConfigurationArgs{
IndexName: pulumi.String("string"),
RoleArn: pulumi.String("string"),
ClusterEndpoint: pulumi.String("string"),
BufferingInterval: pulumi.Int(0),
DomainArn: pulumi.String("string"),
CloudwatchLoggingOptions: &kinesis.FirehoseDeliveryStreamElasticsearchConfigurationCloudwatchLoggingOptionsArgs{
Enabled: pulumi.Bool(false),
LogGroupName: pulumi.String("string"),
LogStreamName: pulumi.String("string"),
},
IndexRotationPeriod: pulumi.String("string"),
ProcessingConfiguration: &kinesis.FirehoseDeliveryStreamElasticsearchConfigurationProcessingConfigurationArgs{
Enabled: pulumi.Bool(false),
Processors: kinesis.FirehoseDeliveryStreamElasticsearchConfigurationProcessingConfigurationProcessorArray{
&kinesis.FirehoseDeliveryStreamElasticsearchConfigurationProcessingConfigurationProcessorArgs{
Type: pulumi.String("string"),
Parameters: kinesis.FirehoseDeliveryStreamElasticsearchConfigurationProcessingConfigurationProcessorParameterArray{
&kinesis.FirehoseDeliveryStreamElasticsearchConfigurationProcessingConfigurationProcessorParameterArgs{
ParameterName: pulumi.String("string"),
ParameterValue: pulumi.String("string"),
},
},
},
},
},
RetryDuration: pulumi.Int(0),
BufferingSize: pulumi.Int(0),
S3BackupMode: pulumi.String("string"),
TypeName: pulumi.String("string"),
VpcConfig: &kinesis.FirehoseDeliveryStreamElasticsearchConfigurationVpcConfigArgs{
RoleArn: pulumi.String("string"),
SecurityGroupIds: pulumi.StringArray{
pulumi.String("string"),
},
SubnetIds: pulumi.StringArray{
pulumi.String("string"),
},
VpcId: pulumi.String("string"),
},
},
ExtendedS3Configuration: &kinesis.FirehoseDeliveryStreamExtendedS3ConfigurationArgs{
BucketArn: pulumi.String("string"),
RoleArn: pulumi.String("string"),
DynamicPartitioningConfiguration: &kinesis.FirehoseDeliveryStreamExtendedS3ConfigurationDynamicPartitioningConfigurationArgs{
Enabled: pulumi.Bool(false),
RetryDuration: pulumi.Int(0),
},
CloudwatchLoggingOptions: &kinesis.FirehoseDeliveryStreamExtendedS3ConfigurationCloudwatchLoggingOptionsArgs{
Enabled: pulumi.Bool(false),
LogGroupName: pulumi.String("string"),
LogStreamName: pulumi.String("string"),
},
CompressionFormat: pulumi.String("string"),
DataFormatConversionConfiguration: &kinesis.FirehoseDeliveryStreamExtendedS3ConfigurationDataFormatConversionConfigurationArgs{
InputFormatConfiguration: &kinesis.FirehoseDeliveryStreamExtendedS3ConfigurationDataFormatConversionConfigurationInputFormatConfigurationArgs{
Deserializer: &kinesis.FirehoseDeliveryStreamExtendedS3ConfigurationDataFormatConversionConfigurationInputFormatConfigurationDeserializerArgs{
HiveJsonSerDe: &kinesis.FirehoseDeliveryStreamExtendedS3ConfigurationDataFormatConversionConfigurationInputFormatConfigurationDeserializerHiveJsonSerDeArgs{
TimestampFormats: pulumi.StringArray{
pulumi.String("string"),
},
},
OpenXJsonSerDe: &kinesis.FirehoseDeliveryStreamExtendedS3ConfigurationDataFormatConversionConfigurationInputFormatConfigurationDeserializerOpenXJsonSerDeArgs{
CaseInsensitive: pulumi.Bool(false),
ColumnToJsonKeyMappings: pulumi.StringMap{
"string": pulumi.String("string"),
},
ConvertDotsInJsonKeysToUnderscores: pulumi.Bool(false),
},
},
},
OutputFormatConfiguration: &kinesis.FirehoseDeliveryStreamExtendedS3ConfigurationDataFormatConversionConfigurationOutputFormatConfigurationArgs{
Serializer: &kinesis.FirehoseDeliveryStreamExtendedS3ConfigurationDataFormatConversionConfigurationOutputFormatConfigurationSerializerArgs{
OrcSerDe: &kinesis.FirehoseDeliveryStreamExtendedS3ConfigurationDataFormatConversionConfigurationOutputFormatConfigurationSerializerOrcSerDeArgs{
BlockSizeBytes: pulumi.Int(0),
BloomFilterColumns: pulumi.StringArray{
pulumi.String("string"),
},
BloomFilterFalsePositiveProbability: pulumi.Float64(0),
Compression: pulumi.String("string"),
DictionaryKeyThreshold: pulumi.Float64(0),
EnablePadding: pulumi.Bool(false),
FormatVersion: pulumi.String("string"),
PaddingTolerance: pulumi.Float64(0),
RowIndexStride: pulumi.Int(0),
StripeSizeBytes: pulumi.Int(0),
},
ParquetSerDe: &kinesis.FirehoseDeliveryStreamExtendedS3ConfigurationDataFormatConversionConfigurationOutputFormatConfigurationSerializerParquetSerDeArgs{
BlockSizeBytes: pulumi.Int(0),
Compression: pulumi.String("string"),
EnableDictionaryCompression: pulumi.Bool(false),
MaxPaddingBytes: pulumi.Int(0),
PageSizeBytes: pulumi.Int(0),
WriterVersion: pulumi.String("string"),
},
},
},
SchemaConfiguration: &kinesis.FirehoseDeliveryStreamExtendedS3ConfigurationDataFormatConversionConfigurationSchemaConfigurationArgs{
DatabaseName: pulumi.String("string"),
RoleArn: pulumi.String("string"),
TableName: pulumi.String("string"),
CatalogId: pulumi.String("string"),
Region: pulumi.String("string"),
VersionId: pulumi.String("string"),
},
Enabled: pulumi.Bool(false),
},
BufferSize: pulumi.Int(0),
ErrorOutputPrefix: pulumi.String("string"),
KmsKeyArn: pulumi.String("string"),
Prefix: pulumi.String("string"),
ProcessingConfiguration: &kinesis.FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationArgs{
Enabled: pulumi.Bool(false),
Processors: kinesis.FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationProcessorArray{
&kinesis.FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationProcessorArgs{
Type: pulumi.String("string"),
Parameters: kinesis.FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationProcessorParameterArray{
&kinesis.FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationProcessorParameterArgs{
ParameterName: pulumi.String("string"),
ParameterValue: pulumi.String("string"),
},
},
},
},
},
BufferInterval: pulumi.Int(0),
S3BackupConfiguration: &kinesis.FirehoseDeliveryStreamExtendedS3ConfigurationS3BackupConfigurationArgs{
BucketArn: pulumi.String("string"),
RoleArn: pulumi.String("string"),
BufferInterval: pulumi.Int(0),
BufferSize: pulumi.Int(0),
CloudwatchLoggingOptions: &kinesis.FirehoseDeliveryStreamExtendedS3ConfigurationS3BackupConfigurationCloudwatchLoggingOptionsArgs{
Enabled: pulumi.Bool(false),
LogGroupName: pulumi.String("string"),
LogStreamName: pulumi.String("string"),
},
CompressionFormat: pulumi.String("string"),
ErrorOutputPrefix: pulumi.String("string"),
KmsKeyArn: pulumi.String("string"),
Prefix: pulumi.String("string"),
},
S3BackupMode: pulumi.String("string"),
},
HttpEndpointConfiguration: &kinesis.FirehoseDeliveryStreamHttpEndpointConfigurationArgs{
Url: pulumi.String("string"),
AccessKey: pulumi.String("string"),
BufferingInterval: pulumi.Int(0),
BufferingSize: pulumi.Int(0),
CloudwatchLoggingOptions: &kinesis.FirehoseDeliveryStreamHttpEndpointConfigurationCloudwatchLoggingOptionsArgs{
Enabled: pulumi.Bool(false),
LogGroupName: pulumi.String("string"),
LogStreamName: pulumi.String("string"),
},
Name: pulumi.String("string"),
ProcessingConfiguration: &kinesis.FirehoseDeliveryStreamHttpEndpointConfigurationProcessingConfigurationArgs{
Enabled: pulumi.Bool(false),
Processors: kinesis.FirehoseDeliveryStreamHttpEndpointConfigurationProcessingConfigurationProcessorArray{
&kinesis.FirehoseDeliveryStreamHttpEndpointConfigurationProcessingConfigurationProcessorArgs{
Type: pulumi.String("string"),
Parameters: kinesis.FirehoseDeliveryStreamHttpEndpointConfigurationProcessingConfigurationProcessorParameterArray{
&kinesis.FirehoseDeliveryStreamHttpEndpointConfigurationProcessingConfigurationProcessorParameterArgs{
ParameterName: pulumi.String("string"),
ParameterValue: pulumi.String("string"),
},
},
},
},
},
RequestConfiguration: &kinesis.FirehoseDeliveryStreamHttpEndpointConfigurationRequestConfigurationArgs{
CommonAttributes: kinesis.FirehoseDeliveryStreamHttpEndpointConfigurationRequestConfigurationCommonAttributeArray{
&kinesis.FirehoseDeliveryStreamHttpEndpointConfigurationRequestConfigurationCommonAttributeArgs{
Name: pulumi.String("string"),
Value: pulumi.String("string"),
},
},
ContentEncoding: pulumi.String("string"),
},
RetryDuration: pulumi.Int(0),
RoleArn: pulumi.String("string"),
S3BackupMode: pulumi.String("string"),
},
Arn: pulumi.String("string"),
Name: pulumi.String("string"),
OpensearchConfiguration: &kinesis.FirehoseDeliveryStreamOpensearchConfigurationArgs{
IndexName: pulumi.String("string"),
RoleArn: pulumi.String("string"),
ClusterEndpoint: pulumi.String("string"),
BufferingInterval: pulumi.Int(0),
DomainArn: pulumi.String("string"),
CloudwatchLoggingOptions: &kinesis.FirehoseDeliveryStreamOpensearchConfigurationCloudwatchLoggingOptionsArgs{
Enabled: pulumi.Bool(false),
LogGroupName: pulumi.String("string"),
LogStreamName: pulumi.String("string"),
},
IndexRotationPeriod: pulumi.String("string"),
ProcessingConfiguration: &kinesis.FirehoseDeliveryStreamOpensearchConfigurationProcessingConfigurationArgs{
Enabled: pulumi.Bool(false),
Processors: kinesis.FirehoseDeliveryStreamOpensearchConfigurationProcessingConfigurationProcessorArray{
&kinesis.FirehoseDeliveryStreamOpensearchConfigurationProcessingConfigurationProcessorArgs{
Type: pulumi.String("string"),
Parameters: kinesis.FirehoseDeliveryStreamOpensearchConfigurationProcessingConfigurationProcessorParameterArray{
&kinesis.FirehoseDeliveryStreamOpensearchConfigurationProcessingConfigurationProcessorParameterArgs{
ParameterName: pulumi.String("string"),
ParameterValue: pulumi.String("string"),
},
},
},
},
},
RetryDuration: pulumi.Int(0),
BufferingSize: pulumi.Int(0),
S3BackupMode: pulumi.String("string"),
TypeName: pulumi.String("string"),
VpcConfig: &kinesis.FirehoseDeliveryStreamOpensearchConfigurationVpcConfigArgs{
RoleArn: pulumi.String("string"),
SecurityGroupIds: pulumi.StringArray{
pulumi.String("string"),
},
SubnetIds: pulumi.StringArray{
pulumi.String("string"),
},
VpcId: pulumi.String("string"),
},
},
RedshiftConfiguration: &kinesis.FirehoseDeliveryStreamRedshiftConfigurationArgs{
ClusterJdbcurl: pulumi.String("string"),
DataTableName: pulumi.String("string"),
Password: pulumi.String("string"),
RoleArn: pulumi.String("string"),
Username: pulumi.String("string"),
CloudwatchLoggingOptions: &kinesis.FirehoseDeliveryStreamRedshiftConfigurationCloudwatchLoggingOptionsArgs{
Enabled: pulumi.Bool(false),
LogGroupName: pulumi.String("string"),
LogStreamName: pulumi.String("string"),
},
CopyOptions: pulumi.String("string"),
DataTableColumns: pulumi.String("string"),
ProcessingConfiguration: &kinesis.FirehoseDeliveryStreamRedshiftConfigurationProcessingConfigurationArgs{
Enabled: pulumi.Bool(false),
Processors: kinesis.FirehoseDeliveryStreamRedshiftConfigurationProcessingConfigurationProcessorArray{
&kinesis.FirehoseDeliveryStreamRedshiftConfigurationProcessingConfigurationProcessorArgs{
Type: pulumi.String("string"),
Parameters: kinesis.FirehoseDeliveryStreamRedshiftConfigurationProcessingConfigurationProcessorParameterArray{
&kinesis.FirehoseDeliveryStreamRedshiftConfigurationProcessingConfigurationProcessorParameterArgs{
ParameterName: pulumi.String("string"),
ParameterValue: pulumi.String("string"),
},
},
},
},
},
RetryDuration: pulumi.Int(0),
S3BackupConfiguration: &kinesis.FirehoseDeliveryStreamRedshiftConfigurationS3BackupConfigurationArgs{
BucketArn: pulumi.String("string"),
RoleArn: pulumi.String("string"),
BufferInterval: pulumi.Int(0),
BufferSize: pulumi.Int(0),
CloudwatchLoggingOptions: &kinesis.FirehoseDeliveryStreamRedshiftConfigurationS3BackupConfigurationCloudwatchLoggingOptionsArgs{
Enabled: pulumi.Bool(false),
LogGroupName: pulumi.String("string"),
LogStreamName: pulumi.String("string"),
},
CompressionFormat: pulumi.String("string"),
ErrorOutputPrefix: pulumi.String("string"),
KmsKeyArn: pulumi.String("string"),
Prefix: pulumi.String("string"),
},
S3BackupMode: pulumi.String("string"),
},
S3Configuration: &kinesis.FirehoseDeliveryStreamS3ConfigurationArgs{
BucketArn: pulumi.String("string"),
RoleArn: pulumi.String("string"),
BufferInterval: pulumi.Int(0),
BufferSize: pulumi.Int(0),
CloudwatchLoggingOptions: &kinesis.FirehoseDeliveryStreamS3ConfigurationCloudwatchLoggingOptionsArgs{
Enabled: pulumi.Bool(false),
LogGroupName: pulumi.String("string"),
LogStreamName: pulumi.String("string"),
},
CompressionFormat: pulumi.String("string"),
ErrorOutputPrefix: pulumi.String("string"),
KmsKeyArn: pulumi.String("string"),
Prefix: pulumi.String("string"),
},
ServerSideEncryption: &kinesis.FirehoseDeliveryStreamServerSideEncryptionArgs{
Enabled: pulumi.Bool(false),
KeyArn: pulumi.String("string"),
KeyType: pulumi.String("string"),
},
SplunkConfiguration: &kinesis.FirehoseDeliveryStreamSplunkConfigurationArgs{
HecEndpoint: pulumi.String("string"),
HecToken: pulumi.String("string"),
CloudwatchLoggingOptions: &kinesis.FirehoseDeliveryStreamSplunkConfigurationCloudwatchLoggingOptionsArgs{
Enabled: pulumi.Bool(false),
LogGroupName: pulumi.String("string"),
LogStreamName: pulumi.String("string"),
},
HecAcknowledgmentTimeout: pulumi.Int(0),
HecEndpointType: pulumi.String("string"),
ProcessingConfiguration: &kinesis.FirehoseDeliveryStreamSplunkConfigurationProcessingConfigurationArgs{
Enabled: pulumi.Bool(false),
Processors: kinesis.FirehoseDeliveryStreamSplunkConfigurationProcessingConfigurationProcessorArray{
&kinesis.FirehoseDeliveryStreamSplunkConfigurationProcessingConfigurationProcessorArgs{
Type: pulumi.String("string"),
Parameters: kinesis.FirehoseDeliveryStreamSplunkConfigurationProcessingConfigurationProcessorParameterArray{
&kinesis.FirehoseDeliveryStreamSplunkConfigurationProcessingConfigurationProcessorParameterArgs{
ParameterName: pulumi.String("string"),
ParameterValue: pulumi.String("string"),
},
},
},
},
},
RetryDuration: pulumi.Int(0),
S3BackupMode: pulumi.String("string"),
},
Tags: pulumi.StringMap{
"string": pulumi.String("string"),
},
VersionId: pulumi.String("string"),
})
var firehoseDeliveryStreamResource = new FirehoseDeliveryStream("firehoseDeliveryStreamResource", FirehoseDeliveryStreamArgs.builder()
.destination("string")
.kinesisSourceConfiguration(FirehoseDeliveryStreamKinesisSourceConfigurationArgs.builder()
.kinesisStreamArn("string")
.roleArn("string")
.build())
.destinationId("string")
.elasticsearchConfiguration(FirehoseDeliveryStreamElasticsearchConfigurationArgs.builder()
.indexName("string")
.roleArn("string")
.clusterEndpoint("string")
.bufferingInterval(0)
.domainArn("string")
.cloudwatchLoggingOptions(FirehoseDeliveryStreamElasticsearchConfigurationCloudwatchLoggingOptionsArgs.builder()
.enabled(false)
.logGroupName("string")
.logStreamName("string")
.build())
.indexRotationPeriod("string")
.processingConfiguration(FirehoseDeliveryStreamElasticsearchConfigurationProcessingConfigurationArgs.builder()
.enabled(false)
.processors(FirehoseDeliveryStreamElasticsearchConfigurationProcessingConfigurationProcessorArgs.builder()
.type("string")
.parameters(FirehoseDeliveryStreamElasticsearchConfigurationProcessingConfigurationProcessorParameterArgs.builder()
.parameterName("string")
.parameterValue("string")
.build())
.build())
.build())
.retryDuration(0)
.bufferingSize(0)
.s3BackupMode("string")
.typeName("string")
.vpcConfig(FirehoseDeliveryStreamElasticsearchConfigurationVpcConfigArgs.builder()
.roleArn("string")
.securityGroupIds("string")
.subnetIds("string")
.vpcId("string")
.build())
.build())
.extendedS3Configuration(FirehoseDeliveryStreamExtendedS3ConfigurationArgs.builder()
.bucketArn("string")
.roleArn("string")
.dynamicPartitioningConfiguration(FirehoseDeliveryStreamExtendedS3ConfigurationDynamicPartitioningConfigurationArgs.builder()
.enabled(false)
.retryDuration(0)
.build())
.cloudwatchLoggingOptions(FirehoseDeliveryStreamExtendedS3ConfigurationCloudwatchLoggingOptionsArgs.builder()
.enabled(false)
.logGroupName("string")
.logStreamName("string")
.build())
.compressionFormat("string")
.dataFormatConversionConfiguration(FirehoseDeliveryStreamExtendedS3ConfigurationDataFormatConversionConfigurationArgs.builder()
.inputFormatConfiguration(FirehoseDeliveryStreamExtendedS3ConfigurationDataFormatConversionConfigurationInputFormatConfigurationArgs.builder()
.deserializer(FirehoseDeliveryStreamExtendedS3ConfigurationDataFormatConversionConfigurationInputFormatConfigurationDeserializerArgs.builder()
.hiveJsonSerDe(FirehoseDeliveryStreamExtendedS3ConfigurationDataFormatConversionConfigurationInputFormatConfigurationDeserializerHiveJsonSerDeArgs.builder()
.timestampFormats("string")
.build())
.openXJsonSerDe(FirehoseDeliveryStreamExtendedS3ConfigurationDataFormatConversionConfigurationInputFormatConfigurationDeserializerOpenXJsonSerDeArgs.builder()
.caseInsensitive(false)
.columnToJsonKeyMappings(Map.of("string", "string"))
.convertDotsInJsonKeysToUnderscores(false)
.build())
.build())
.build())
.outputFormatConfiguration(FirehoseDeliveryStreamExtendedS3ConfigurationDataFormatConversionConfigurationOutputFormatConfigurationArgs.builder()
.serializer(FirehoseDeliveryStreamExtendedS3ConfigurationDataFormatConversionConfigurationOutputFormatConfigurationSerializerArgs.builder()
.orcSerDe(FirehoseDeliveryStreamExtendedS3ConfigurationDataFormatConversionConfigurationOutputFormatConfigurationSerializerOrcSerDeArgs.builder()
.blockSizeBytes(0)
.bloomFilterColumns("string")
.bloomFilterFalsePositiveProbability(0.0)
.compression("string")
.dictionaryKeyThreshold(0.0)
.enablePadding(false)
.formatVersion("string")
.paddingTolerance(0.0)
.rowIndexStride(0)
.stripeSizeBytes(0)
.build())
.parquetSerDe(FirehoseDeliveryStreamExtendedS3ConfigurationDataFormatConversionConfigurationOutputFormatConfigurationSerializerParquetSerDeArgs.builder()
.blockSizeBytes(0)
.compression("string")
.enableDictionaryCompression(false)
.maxPaddingBytes(0)
.pageSizeBytes(0)
.writerVersion("string")
.build())
.build())
.build())
.schemaConfiguration(FirehoseDeliveryStreamExtendedS3ConfigurationDataFormatConversionConfigurationSchemaConfigurationArgs.builder()
.databaseName("string")
.roleArn("string")
.tableName("string")
.catalogId("string")
.region("string")
.versionId("string")
.build())
.enabled(false)
.build())
.bufferSize(0)
.errorOutputPrefix("string")
.kmsKeyArn("string")
.prefix("string")
.processingConfiguration(FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationArgs.builder()
.enabled(false)
.processors(FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationProcessorArgs.builder()
.type("string")
.parameters(FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationProcessorParameterArgs.builder()
.parameterName("string")
.parameterValue("string")
.build())
.build())
.build())
.bufferInterval(0)
.s3BackupConfiguration(FirehoseDeliveryStreamExtendedS3ConfigurationS3BackupConfigurationArgs.builder()
.bucketArn("string")
.roleArn("string")
.bufferInterval(0)
.bufferSize(0)
.cloudwatchLoggingOptions(FirehoseDeliveryStreamExtendedS3ConfigurationS3BackupConfigurationCloudwatchLoggingOptionsArgs.builder()
.enabled(false)
.logGroupName("string")
.logStreamName("string")
.build())
.compressionFormat("string")
.errorOutputPrefix("string")
.kmsKeyArn("string")
.prefix("string")
.build())
.s3BackupMode("string")
.build())
.httpEndpointConfiguration(FirehoseDeliveryStreamHttpEndpointConfigurationArgs.builder()
.url("string")
.accessKey("string")
.bufferingInterval(0)
.bufferingSize(0)
.cloudwatchLoggingOptions(FirehoseDeliveryStreamHttpEndpointConfigurationCloudwatchLoggingOptionsArgs.builder()
.enabled(false)
.logGroupName("string")
.logStreamName("string")
.build())
.name("string")
.processingConfiguration(FirehoseDeliveryStreamHttpEndpointConfigurationProcessingConfigurationArgs.builder()
.enabled(false)
.processors(FirehoseDeliveryStreamHttpEndpointConfigurationProcessingConfigurationProcessorArgs.builder()
.type("string")
.parameters(FirehoseDeliveryStreamHttpEndpointConfigurationProcessingConfigurationProcessorParameterArgs.builder()
.parameterName("string")
.parameterValue("string")
.build())
.build())
.build())
.requestConfiguration(FirehoseDeliveryStreamHttpEndpointConfigurationRequestConfigurationArgs.builder()
.commonAttributes(FirehoseDeliveryStreamHttpEndpointConfigurationRequestConfigurationCommonAttributeArgs.builder()
.name("string")
.value("string")
.build())
.contentEncoding("string")
.build())
.retryDuration(0)
.roleArn("string")
.s3BackupMode("string")
.build())
.arn("string")
.name("string")
.opensearchConfiguration(FirehoseDeliveryStreamOpensearchConfigurationArgs.builder()
.indexName("string")
.roleArn("string")
.clusterEndpoint("string")
.bufferingInterval(0)
.domainArn("string")
.cloudwatchLoggingOptions(FirehoseDeliveryStreamOpensearchConfigurationCloudwatchLoggingOptionsArgs.builder()
.enabled(false)
.logGroupName("string")
.logStreamName("string")
.build())
.indexRotationPeriod("string")
.processingConfiguration(FirehoseDeliveryStreamOpensearchConfigurationProcessingConfigurationArgs.builder()
.enabled(false)
.processors(FirehoseDeliveryStreamOpensearchConfigurationProcessingConfigurationProcessorArgs.builder()
.type("string")
.parameters(FirehoseDeliveryStreamOpensearchConfigurationProcessingConfigurationProcessorParameterArgs.builder()
.parameterName("string")
.parameterValue("string")
.build())
.build())
.build())
.retryDuration(0)
.bufferingSize(0)
.s3BackupMode("string")
.typeName("string")
.vpcConfig(FirehoseDeliveryStreamOpensearchConfigurationVpcConfigArgs.builder()
.roleArn("string")
.securityGroupIds("string")
.subnetIds("string")
.vpcId("string")
.build())
.build())
.redshiftConfiguration(FirehoseDeliveryStreamRedshiftConfigurationArgs.builder()
.clusterJdbcurl("string")
.dataTableName("string")
.password("string")
.roleArn("string")
.username("string")
.cloudwatchLoggingOptions(FirehoseDeliveryStreamRedshiftConfigurationCloudwatchLoggingOptionsArgs.builder()
.enabled(false)
.logGroupName("string")
.logStreamName("string")
.build())
.copyOptions("string")
.dataTableColumns("string")
.processingConfiguration(FirehoseDeliveryStreamRedshiftConfigurationProcessingConfigurationArgs.builder()
.enabled(false)
.processors(FirehoseDeliveryStreamRedshiftConfigurationProcessingConfigurationProcessorArgs.builder()
.type("string")
.parameters(FirehoseDeliveryStreamRedshiftConfigurationProcessingConfigurationProcessorParameterArgs.builder()
.parameterName("string")
.parameterValue("string")
.build())
.build())
.build())
.retryDuration(0)
.s3BackupConfiguration(FirehoseDeliveryStreamRedshiftConfigurationS3BackupConfigurationArgs.builder()
.bucketArn("string")
.roleArn("string")
.bufferInterval(0)
.bufferSize(0)
.cloudwatchLoggingOptions(FirehoseDeliveryStreamRedshiftConfigurationS3BackupConfigurationCloudwatchLoggingOptionsArgs.builder()
.enabled(false)
.logGroupName("string")
.logStreamName("string")
.build())
.compressionFormat("string")
.errorOutputPrefix("string")
.kmsKeyArn("string")
.prefix("string")
.build())
.s3BackupMode("string")
.build())
.s3Configuration(FirehoseDeliveryStreamS3ConfigurationArgs.builder()
.bucketArn("string")
.roleArn("string")
.bufferInterval(0)
.bufferSize(0)
.cloudwatchLoggingOptions(FirehoseDeliveryStreamS3ConfigurationCloudwatchLoggingOptionsArgs.builder()
.enabled(false)
.logGroupName("string")
.logStreamName("string")
.build())
.compressionFormat("string")
.errorOutputPrefix("string")
.kmsKeyArn("string")
.prefix("string")
.build())
.serverSideEncryption(FirehoseDeliveryStreamServerSideEncryptionArgs.builder()
.enabled(false)
.keyArn("string")
.keyType("string")
.build())
.splunkConfiguration(FirehoseDeliveryStreamSplunkConfigurationArgs.builder()
.hecEndpoint("string")
.hecToken("string")
.cloudwatchLoggingOptions(FirehoseDeliveryStreamSplunkConfigurationCloudwatchLoggingOptionsArgs.builder()
.enabled(false)
.logGroupName("string")
.logStreamName("string")
.build())
.hecAcknowledgmentTimeout(0)
.hecEndpointType("string")
.processingConfiguration(FirehoseDeliveryStreamSplunkConfigurationProcessingConfigurationArgs.builder()
.enabled(false)
.processors(FirehoseDeliveryStreamSplunkConfigurationProcessingConfigurationProcessorArgs.builder()
.type("string")
.parameters(FirehoseDeliveryStreamSplunkConfigurationProcessingConfigurationProcessorParameterArgs.builder()
.parameterName("string")
.parameterValue("string")
.build())
.build())
.build())
.retryDuration(0)
.s3BackupMode("string")
.build())
.tags(Map.of("string", "string"))
.versionId("string")
.build());
firehose_delivery_stream_resource = aws.kinesis.FirehoseDeliveryStream("firehoseDeliveryStreamResource",
destination="string",
kinesis_source_configuration={
"kinesis_stream_arn": "string",
"role_arn": "string",
},
destination_id="string",
elasticsearch_configuration={
"index_name": "string",
"role_arn": "string",
"cluster_endpoint": "string",
"buffering_interval": 0,
"domain_arn": "string",
"cloudwatch_logging_options": {
"enabled": False,
"log_group_name": "string",
"log_stream_name": "string",
},
"index_rotation_period": "string",
"processing_configuration": {
"enabled": False,
"processors": [{
"type": "string",
"parameters": [{
"parameter_name": "string",
"parameter_value": "string",
}],
}],
},
"retry_duration": 0,
"buffering_size": 0,
"s3_backup_mode": "string",
"type_name": "string",
"vpc_config": {
"role_arn": "string",
"security_group_ids": ["string"],
"subnet_ids": ["string"],
"vpc_id": "string",
},
},
extended_s3_configuration={
"bucket_arn": "string",
"role_arn": "string",
"dynamic_partitioning_configuration": {
"enabled": False,
"retry_duration": 0,
},
"cloudwatch_logging_options": {
"enabled": False,
"log_group_name": "string",
"log_stream_name": "string",
},
"compression_format": "string",
"data_format_conversion_configuration": {
"input_format_configuration": {
"deserializer": {
"hive_json_ser_de": {
"timestamp_formats": ["string"],
},
"open_x_json_ser_de": {
"case_insensitive": False,
"column_to_json_key_mappings": {
"string": "string",
},
"convert_dots_in_json_keys_to_underscores": False,
},
},
},
"output_format_configuration": {
"serializer": {
"orc_ser_de": {
"block_size_bytes": 0,
"bloom_filter_columns": ["string"],
"bloom_filter_false_positive_probability": 0,
"compression": "string",
"dictionary_key_threshold": 0,
"enable_padding": False,
"format_version": "string",
"padding_tolerance": 0,
"row_index_stride": 0,
"stripe_size_bytes": 0,
},
"parquet_ser_de": {
"block_size_bytes": 0,
"compression": "string",
"enable_dictionary_compression": False,
"max_padding_bytes": 0,
"page_size_bytes": 0,
"writer_version": "string",
},
},
},
"schema_configuration": {
"database_name": "string",
"role_arn": "string",
"table_name": "string",
"catalog_id": "string",
"region": "string",
"version_id": "string",
},
"enabled": False,
},
"buffer_size": 0,
"error_output_prefix": "string",
"kms_key_arn": "string",
"prefix": "string",
"processing_configuration": {
"enabled": False,
"processors": [{
"type": "string",
"parameters": [{
"parameter_name": "string",
"parameter_value": "string",
}],
}],
},
"buffer_interval": 0,
"s3_backup_configuration": {
"bucket_arn": "string",
"role_arn": "string",
"buffer_interval": 0,
"buffer_size": 0,
"cloudwatch_logging_options": {
"enabled": False,
"log_group_name": "string",
"log_stream_name": "string",
},
"compression_format": "string",
"error_output_prefix": "string",
"kms_key_arn": "string",
"prefix": "string",
},
"s3_backup_mode": "string",
},
http_endpoint_configuration={
"url": "string",
"access_key": "string",
"buffering_interval": 0,
"buffering_size": 0,
"cloudwatch_logging_options": {
"enabled": False,
"log_group_name": "string",
"log_stream_name": "string",
},
"name": "string",
"processing_configuration": {
"enabled": False,
"processors": [{
"type": "string",
"parameters": [{
"parameter_name": "string",
"parameter_value": "string",
}],
}],
},
"request_configuration": {
"common_attributes": [{
"name": "string",
"value": "string",
}],
"content_encoding": "string",
},
"retry_duration": 0,
"role_arn": "string",
"s3_backup_mode": "string",
},
arn="string",
name="string",
opensearch_configuration={
"index_name": "string",
"role_arn": "string",
"cluster_endpoint": "string",
"buffering_interval": 0,
"domain_arn": "string",
"cloudwatch_logging_options": {
"enabled": False,
"log_group_name": "string",
"log_stream_name": "string",
},
"index_rotation_period": "string",
"processing_configuration": {
"enabled": False,
"processors": [{
"type": "string",
"parameters": [{
"parameter_name": "string",
"parameter_value": "string",
}],
}],
},
"retry_duration": 0,
"buffering_size": 0,
"s3_backup_mode": "string",
"type_name": "string",
"vpc_config": {
"role_arn": "string",
"security_group_ids": ["string"],
"subnet_ids": ["string"],
"vpc_id": "string",
},
},
redshift_configuration={
"cluster_jdbcurl": "string",
"data_table_name": "string",
"password": "string",
"role_arn": "string",
"username": "string",
"cloudwatch_logging_options": {
"enabled": False,
"log_group_name": "string",
"log_stream_name": "string",
},
"copy_options": "string",
"data_table_columns": "string",
"processing_configuration": {
"enabled": False,
"processors": [{
"type": "string",
"parameters": [{
"parameter_name": "string",
"parameter_value": "string",
}],
}],
},
"retry_duration": 0,
"s3_backup_configuration": {
"bucket_arn": "string",
"role_arn": "string",
"buffer_interval": 0,
"buffer_size": 0,
"cloudwatch_logging_options": {
"enabled": False,
"log_group_name": "string",
"log_stream_name": "string",
},
"compression_format": "string",
"error_output_prefix": "string",
"kms_key_arn": "string",
"prefix": "string",
},
"s3_backup_mode": "string",
},
s3_configuration={
"bucket_arn": "string",
"role_arn": "string",
"buffer_interval": 0,
"buffer_size": 0,
"cloudwatch_logging_options": {
"enabled": False,
"log_group_name": "string",
"log_stream_name": "string",
},
"compression_format": "string",
"error_output_prefix": "string",
"kms_key_arn": "string",
"prefix": "string",
},
server_side_encryption={
"enabled": False,
"key_arn": "string",
"key_type": "string",
},
splunk_configuration={
"hec_endpoint": "string",
"hec_token": "string",
"cloudwatch_logging_options": {
"enabled": False,
"log_group_name": "string",
"log_stream_name": "string",
},
"hec_acknowledgment_timeout": 0,
"hec_endpoint_type": "string",
"processing_configuration": {
"enabled": False,
"processors": [{
"type": "string",
"parameters": [{
"parameter_name": "string",
"parameter_value": "string",
}],
}],
},
"retry_duration": 0,
"s3_backup_mode": "string",
},
tags={
"string": "string",
},
version_id="string")
const firehoseDeliveryStreamResource = new aws.kinesis.FirehoseDeliveryStream("firehoseDeliveryStreamResource", {
destination: "string",
kinesisSourceConfiguration: {
kinesisStreamArn: "string",
roleArn: "string",
},
destinationId: "string",
elasticsearchConfiguration: {
indexName: "string",
roleArn: "string",
clusterEndpoint: "string",
bufferingInterval: 0,
domainArn: "string",
cloudwatchLoggingOptions: {
enabled: false,
logGroupName: "string",
logStreamName: "string",
},
indexRotationPeriod: "string",
processingConfiguration: {
enabled: false,
processors: [{
type: "string",
parameters: [{
parameterName: "string",
parameterValue: "string",
}],
}],
},
retryDuration: 0,
bufferingSize: 0,
s3BackupMode: "string",
typeName: "string",
vpcConfig: {
roleArn: "string",
securityGroupIds: ["string"],
subnetIds: ["string"],
vpcId: "string",
},
},
extendedS3Configuration: {
bucketArn: "string",
roleArn: "string",
dynamicPartitioningConfiguration: {
enabled: false,
retryDuration: 0,
},
cloudwatchLoggingOptions: {
enabled: false,
logGroupName: "string",
logStreamName: "string",
},
compressionFormat: "string",
dataFormatConversionConfiguration: {
inputFormatConfiguration: {
deserializer: {
hiveJsonSerDe: {
timestampFormats: ["string"],
},
openXJsonSerDe: {
caseInsensitive: false,
columnToJsonKeyMappings: {
string: "string",
},
convertDotsInJsonKeysToUnderscores: false,
},
},
},
outputFormatConfiguration: {
serializer: {
orcSerDe: {
blockSizeBytes: 0,
bloomFilterColumns: ["string"],
bloomFilterFalsePositiveProbability: 0,
compression: "string",
dictionaryKeyThreshold: 0,
enablePadding: false,
formatVersion: "string",
paddingTolerance: 0,
rowIndexStride: 0,
stripeSizeBytes: 0,
},
parquetSerDe: {
blockSizeBytes: 0,
compression: "string",
enableDictionaryCompression: false,
maxPaddingBytes: 0,
pageSizeBytes: 0,
writerVersion: "string",
},
},
},
schemaConfiguration: {
databaseName: "string",
roleArn: "string",
tableName: "string",
catalogId: "string",
region: "string",
versionId: "string",
},
enabled: false,
},
bufferSize: 0,
errorOutputPrefix: "string",
kmsKeyArn: "string",
prefix: "string",
processingConfiguration: {
enabled: false,
processors: [{
type: "string",
parameters: [{
parameterName: "string",
parameterValue: "string",
}],
}],
},
bufferInterval: 0,
s3BackupConfiguration: {
bucketArn: "string",
roleArn: "string",
bufferInterval: 0,
bufferSize: 0,
cloudwatchLoggingOptions: {
enabled: false,
logGroupName: "string",
logStreamName: "string",
},
compressionFormat: "string",
errorOutputPrefix: "string",
kmsKeyArn: "string",
prefix: "string",
},
s3BackupMode: "string",
},
httpEndpointConfiguration: {
url: "string",
accessKey: "string",
bufferingInterval: 0,
bufferingSize: 0,
cloudwatchLoggingOptions: {
enabled: false,
logGroupName: "string",
logStreamName: "string",
},
name: "string",
processingConfiguration: {
enabled: false,
processors: [{
type: "string",
parameters: [{
parameterName: "string",
parameterValue: "string",
}],
}],
},
requestConfiguration: {
commonAttributes: [{
name: "string",
value: "string",
}],
contentEncoding: "string",
},
retryDuration: 0,
roleArn: "string",
s3BackupMode: "string",
},
arn: "string",
name: "string",
opensearchConfiguration: {
indexName: "string",
roleArn: "string",
clusterEndpoint: "string",
bufferingInterval: 0,
domainArn: "string",
cloudwatchLoggingOptions: {
enabled: false,
logGroupName: "string",
logStreamName: "string",
},
indexRotationPeriod: "string",
processingConfiguration: {
enabled: false,
processors: [{
type: "string",
parameters: [{
parameterName: "string",
parameterValue: "string",
}],
}],
},
retryDuration: 0,
bufferingSize: 0,
s3BackupMode: "string",
typeName: "string",
vpcConfig: {
roleArn: "string",
securityGroupIds: ["string"],
subnetIds: ["string"],
vpcId: "string",
},
},
redshiftConfiguration: {
clusterJdbcurl: "string",
dataTableName: "string",
password: "string",
roleArn: "string",
username: "string",
cloudwatchLoggingOptions: {
enabled: false,
logGroupName: "string",
logStreamName: "string",
},
copyOptions: "string",
dataTableColumns: "string",
processingConfiguration: {
enabled: false,
processors: [{
type: "string",
parameters: [{
parameterName: "string",
parameterValue: "string",
}],
}],
},
retryDuration: 0,
s3BackupConfiguration: {
bucketArn: "string",
roleArn: "string",
bufferInterval: 0,
bufferSize: 0,
cloudwatchLoggingOptions: {
enabled: false,
logGroupName: "string",
logStreamName: "string",
},
compressionFormat: "string",
errorOutputPrefix: "string",
kmsKeyArn: "string",
prefix: "string",
},
s3BackupMode: "string",
},
s3Configuration: {
bucketArn: "string",
roleArn: "string",
bufferInterval: 0,
bufferSize: 0,
cloudwatchLoggingOptions: {
enabled: false,
logGroupName: "string",
logStreamName: "string",
},
compressionFormat: "string",
errorOutputPrefix: "string",
kmsKeyArn: "string",
prefix: "string",
},
serverSideEncryption: {
enabled: false,
keyArn: "string",
keyType: "string",
},
splunkConfiguration: {
hecEndpoint: "string",
hecToken: "string",
cloudwatchLoggingOptions: {
enabled: false,
logGroupName: "string",
logStreamName: "string",
},
hecAcknowledgmentTimeout: 0,
hecEndpointType: "string",
processingConfiguration: {
enabled: false,
processors: [{
type: "string",
parameters: [{
parameterName: "string",
parameterValue: "string",
}],
}],
},
retryDuration: 0,
s3BackupMode: "string",
},
tags: {
string: "string",
},
versionId: "string",
});
type: aws:kinesis:FirehoseDeliveryStream
properties:
arn: string
destination: string
destinationId: string
elasticsearchConfiguration:
bufferingInterval: 0
bufferingSize: 0
cloudwatchLoggingOptions:
enabled: false
logGroupName: string
logStreamName: string
clusterEndpoint: string
domainArn: string
indexName: string
indexRotationPeriod: string
processingConfiguration:
enabled: false
processors:
- parameters:
- parameterName: string
parameterValue: string
type: string
retryDuration: 0
roleArn: string
s3BackupMode: string
typeName: string
vpcConfig:
roleArn: string
securityGroupIds:
- string
subnetIds:
- string
vpcId: string
extendedS3Configuration:
bucketArn: string
bufferInterval: 0
bufferSize: 0
cloudwatchLoggingOptions:
enabled: false
logGroupName: string
logStreamName: string
compressionFormat: string
dataFormatConversionConfiguration:
enabled: false
inputFormatConfiguration:
deserializer:
hiveJsonSerDe:
timestampFormats:
- string
openXJsonSerDe:
caseInsensitive: false
columnToJsonKeyMappings:
string: string
convertDotsInJsonKeysToUnderscores: false
outputFormatConfiguration:
serializer:
orcSerDe:
blockSizeBytes: 0
bloomFilterColumns:
- string
bloomFilterFalsePositiveProbability: 0
compression: string
dictionaryKeyThreshold: 0
enablePadding: false
formatVersion: string
paddingTolerance: 0
rowIndexStride: 0
stripeSizeBytes: 0
parquetSerDe:
blockSizeBytes: 0
compression: string
enableDictionaryCompression: false
maxPaddingBytes: 0
pageSizeBytes: 0
writerVersion: string
schemaConfiguration:
catalogId: string
databaseName: string
region: string
roleArn: string
tableName: string
versionId: string
dynamicPartitioningConfiguration:
enabled: false
retryDuration: 0
errorOutputPrefix: string
kmsKeyArn: string
prefix: string
processingConfiguration:
enabled: false
processors:
- parameters:
- parameterName: string
parameterValue: string
type: string
roleArn: string
s3BackupConfiguration:
bucketArn: string
bufferInterval: 0
bufferSize: 0
cloudwatchLoggingOptions:
enabled: false
logGroupName: string
logStreamName: string
compressionFormat: string
errorOutputPrefix: string
kmsKeyArn: string
prefix: string
roleArn: string
s3BackupMode: string
httpEndpointConfiguration:
accessKey: string
bufferingInterval: 0
bufferingSize: 0
cloudwatchLoggingOptions:
enabled: false
logGroupName: string
logStreamName: string
name: string
processingConfiguration:
enabled: false
processors:
- parameters:
- parameterName: string
parameterValue: string
type: string
requestConfiguration:
commonAttributes:
- name: string
value: string
contentEncoding: string
retryDuration: 0
roleArn: string
s3BackupMode: string
url: string
kinesisSourceConfiguration:
kinesisStreamArn: string
roleArn: string
name: string
opensearchConfiguration:
bufferingInterval: 0
bufferingSize: 0
cloudwatchLoggingOptions:
enabled: false
logGroupName: string
logStreamName: string
clusterEndpoint: string
domainArn: string
indexName: string
indexRotationPeriod: string
processingConfiguration:
enabled: false
processors:
- parameters:
- parameterName: string
parameterValue: string
type: string
retryDuration: 0
roleArn: string
s3BackupMode: string
typeName: string
vpcConfig:
roleArn: string
securityGroupIds:
- string
subnetIds:
- string
vpcId: string
redshiftConfiguration:
cloudwatchLoggingOptions:
enabled: false
logGroupName: string
logStreamName: string
clusterJdbcurl: string
copyOptions: string
dataTableColumns: string
dataTableName: string
password: string
processingConfiguration:
enabled: false
processors:
- parameters:
- parameterName: string
parameterValue: string
type: string
retryDuration: 0
roleArn: string
s3BackupConfiguration:
bucketArn: string
bufferInterval: 0
bufferSize: 0
cloudwatchLoggingOptions:
enabled: false
logGroupName: string
logStreamName: string
compressionFormat: string
errorOutputPrefix: string
kmsKeyArn: string
prefix: string
roleArn: string
s3BackupMode: string
username: string
s3Configuration:
bucketArn: string
bufferInterval: 0
bufferSize: 0
cloudwatchLoggingOptions:
enabled: false
logGroupName: string
logStreamName: string
compressionFormat: string
errorOutputPrefix: string
kmsKeyArn: string
prefix: string
roleArn: string
serverSideEncryption:
enabled: false
keyArn: string
keyType: string
splunkConfiguration:
cloudwatchLoggingOptions:
enabled: false
logGroupName: string
logStreamName: string
hecAcknowledgmentTimeout: 0
hecEndpoint: string
hecEndpointType: string
hecToken: string
processingConfiguration:
enabled: false
processors:
- parameters:
- parameterName: string
parameterValue: string
type: string
retryDuration: 0
s3BackupMode: string
tags:
string: string
versionId: string
FirehoseDeliveryStream Resource Properties
To learn more about resource properties and how to use them, see Inputs and Outputs in the Architecture and Concepts docs.
Inputs
In Python, inputs that are objects can be passed either as argument classes or as dictionary literals.
The FirehoseDeliveryStream resource accepts the following input properties:
- Destination string
- This is the destination to where the data is delivered. The only options are
s3(Deprecated, useextended_s3instead),extended_s3,redshift,elasticsearch,splunk,http_endpointandopensearch. - Arn string
- The Amazon Resource Name (ARN) specifying the Stream
- Destination
Id string - Elasticsearch
Configuration FirehoseDelivery Stream Elasticsearch Configuration - Configuration options if elasticsearch is the destination. More details are given below.
- Extended
S3Configuration FirehoseDelivery Stream Extended S3Configuration - Enhanced configuration options for the s3 destination. More details are given below.
- Http
Endpoint FirehoseConfiguration Delivery Stream Http Endpoint Configuration - Configuration options if http_endpoint is the destination. requires the user to also specify a
s3_configurationblock. More details are given below. - Kinesis
Source FirehoseConfiguration Delivery Stream Kinesis Source Configuration - Allows the ability to specify the kinesis stream that is used as the source of the firehose delivery stream.
- Name string
- A name to identify the stream. This is unique to the AWS account and region the Stream is created in. When using for WAF logging, name must be prefixed with
aws-waf-logs-. See AWS Documentation for more details. - Opensearch
Configuration FirehoseDelivery Stream Opensearch Configuration - Configuration options if opensearch is the destination. More details are given below.
- Redshift
Configuration FirehoseDelivery Stream Redshift Configuration - Configuration options if redshift is the destination.
Using
redshift_configurationrequires the user to also specify as3_configurationblock. More details are given below. - S3Configuration
Firehose
Delivery Stream S3Configuration - Required for non-S3 destinations. For S3 destination, use
extended_s3_configurationinstead. Configuration options for the s3 destination (or the intermediate bucket if the destination is redshift). More details are given below. - Server
Side FirehoseEncryption Delivery Stream Server Side Encryption - Encrypt at rest options. Server-side encryption should not be enabled when a kinesis stream is configured as the source of the firehose delivery stream.
- Splunk
Configuration FirehoseDelivery Stream Splunk Configuration - Configuration options if splunk is the destination. More details are given below.
- Dictionary<string, string>
- A map of tags to assign to the resource. If configured with a provider
default_tagsconfiguration block present, tags with matching keys will overwrite those defined at the provider-level. - Version
Id string - Specifies the table version for the output data schema. Defaults to
LATEST.
- Destination string
- This is the destination to where the data is delivered. The only options are
s3(Deprecated, useextended_s3instead),extended_s3,redshift,elasticsearch,splunk,http_endpointandopensearch. - Arn string
- The Amazon Resource Name (ARN) specifying the Stream
- Destination
Id string - Elasticsearch
Configuration FirehoseDelivery Stream Elasticsearch Configuration Args - Configuration options if elasticsearch is the destination. More details are given below.
- Extended
S3Configuration FirehoseDelivery Stream Extended S3Configuration Args - Enhanced configuration options for the s3 destination. More details are given below.
- Http
Endpoint FirehoseConfiguration Delivery Stream Http Endpoint Configuration Args - Configuration options if http_endpoint is the destination. requires the user to also specify a
s3_configurationblock. More details are given below. - Kinesis
Source FirehoseConfiguration Delivery Stream Kinesis Source Configuration Args - Allows the ability to specify the kinesis stream that is used as the source of the firehose delivery stream.
- Name string
- A name to identify the stream. This is unique to the AWS account and region the Stream is created in. When using for WAF logging, name must be prefixed with
aws-waf-logs-. See AWS Documentation for more details. - Opensearch
Configuration FirehoseDelivery Stream Opensearch Configuration Args - Configuration options if opensearch is the destination. More details are given below.
- Redshift
Configuration FirehoseDelivery Stream Redshift Configuration Args - Configuration options if redshift is the destination.
Using
redshift_configurationrequires the user to also specify as3_configurationblock. More details are given below. - S3Configuration
Firehose
Delivery Stream S3Configuration Args - Required for non-S3 destinations. For S3 destination, use
extended_s3_configurationinstead. Configuration options for the s3 destination (or the intermediate bucket if the destination is redshift). More details are given below. - Server
Side FirehoseEncryption Delivery Stream Server Side Encryption Args - Encrypt at rest options. Server-side encryption should not be enabled when a kinesis stream is configured as the source of the firehose delivery stream.
- Splunk
Configuration FirehoseDelivery Stream Splunk Configuration Args - Configuration options if splunk is the destination. More details are given below.
- map[string]string
- A map of tags to assign to the resource. If configured with a provider
default_tagsconfiguration block present, tags with matching keys will overwrite those defined at the provider-level. - Version
Id string - Specifies the table version for the output data schema. Defaults to
LATEST.
- destination String
- This is the destination to where the data is delivered. The only options are
s3(Deprecated, useextended_s3instead),extended_s3,redshift,elasticsearch,splunk,http_endpointandopensearch. - arn String
- The Amazon Resource Name (ARN) specifying the Stream
- destination
Id String - elasticsearch
Configuration FirehoseDelivery Stream Elasticsearch Configuration - Configuration options if elasticsearch is the destination. More details are given below.
- extended
S3Configuration FirehoseDelivery Stream Extended S3Configuration - Enhanced configuration options for the s3 destination. More details are given below.
- http
Endpoint FirehoseConfiguration Delivery Stream Http Endpoint Configuration - Configuration options if http_endpoint is the destination. requires the user to also specify a
s3_configurationblock. More details are given below. - kinesis
Source FirehoseConfiguration Delivery Stream Kinesis Source Configuration - Allows the ability to specify the kinesis stream that is used as the source of the firehose delivery stream.
- name String
- A name to identify the stream. This is unique to the AWS account and region the Stream is created in. When using for WAF logging, name must be prefixed with
aws-waf-logs-. See AWS Documentation for more details. - opensearch
Configuration FirehoseDelivery Stream Opensearch Configuration - Configuration options if opensearch is the destination. More details are given below.
- redshift
Configuration FirehoseDelivery Stream Redshift Configuration - Configuration options if redshift is the destination.
Using
redshift_configurationrequires the user to also specify as3_configurationblock. More details are given below. - s3Configuration
Firehose
Delivery Stream S3Configuration - Required for non-S3 destinations. For S3 destination, use
extended_s3_configurationinstead. Configuration options for the s3 destination (or the intermediate bucket if the destination is redshift). More details are given below. - server
Side FirehoseEncryption Delivery Stream Server Side Encryption - Encrypt at rest options. Server-side encryption should not be enabled when a kinesis stream is configured as the source of the firehose delivery stream.
- splunk
Configuration FirehoseDelivery Stream Splunk Configuration - Configuration options if splunk is the destination. More details are given below.
- Map<String,String>
- A map of tags to assign to the resource. If configured with a provider
default_tagsconfiguration block present, tags with matching keys will overwrite those defined at the provider-level. - version
Id String - Specifies the table version for the output data schema. Defaults to
LATEST.
- destination string
- This is the destination to where the data is delivered. The only options are
s3(Deprecated, useextended_s3instead),extended_s3,redshift,elasticsearch,splunk,http_endpointandopensearch. - arn string
- The Amazon Resource Name (ARN) specifying the Stream
- destination
Id string - elasticsearch
Configuration FirehoseDelivery Stream Elasticsearch Configuration - Configuration options if elasticsearch is the destination. More details are given below.
- extended
S3Configuration FirehoseDelivery Stream Extended S3Configuration - Enhanced configuration options for the s3 destination. More details are given below.
- http
Endpoint FirehoseConfiguration Delivery Stream Http Endpoint Configuration - Configuration options if http_endpoint is the destination. requires the user to also specify a
s3_configurationblock. More details are given below. - kinesis
Source FirehoseConfiguration Delivery Stream Kinesis Source Configuration - Allows the ability to specify the kinesis stream that is used as the source of the firehose delivery stream.
- name string
- A name to identify the stream. This is unique to the AWS account and region the Stream is created in. When using for WAF logging, name must be prefixed with
aws-waf-logs-. See AWS Documentation for more details. - opensearch
Configuration FirehoseDelivery Stream Opensearch Configuration - Configuration options if opensearch is the destination. More details are given below.
- redshift
Configuration FirehoseDelivery Stream Redshift Configuration - Configuration options if redshift is the destination.
Using
redshift_configurationrequires the user to also specify as3_configurationblock. More details are given below. - s3Configuration
Firehose
Delivery Stream S3Configuration - Required for non-S3 destinations. For S3 destination, use
extended_s3_configurationinstead. Configuration options for the s3 destination (or the intermediate bucket if the destination is redshift). More details are given below. - server
Side FirehoseEncryption Delivery Stream Server Side Encryption - Encrypt at rest options. Server-side encryption should not be enabled when a kinesis stream is configured as the source of the firehose delivery stream.
- splunk
Configuration FirehoseDelivery Stream Splunk Configuration - Configuration options if splunk is the destination. More details are given below.
- {[key: string]: string}
- A map of tags to assign to the resource. If configured with a provider
default_tagsconfiguration block present, tags with matching keys will overwrite those defined at the provider-level. - version
Id string - Specifies the table version for the output data schema. Defaults to
LATEST.
- destination str
- This is the destination to where the data is delivered. The only options are
s3(Deprecated, useextended_s3instead),extended_s3,redshift,elasticsearch,splunk,http_endpointandopensearch. - arn str
- The Amazon Resource Name (ARN) specifying the Stream
- destination_
id str - elasticsearch_
configuration FirehoseDelivery Stream Elasticsearch Configuration Args - Configuration options if elasticsearch is the destination. More details are given below.
- extended_
s3_ Firehoseconfiguration Delivery Stream Extended S3Configuration Args - Enhanced configuration options for the s3 destination. More details are given below.
- http_
endpoint_ Firehoseconfiguration Delivery Stream Http Endpoint Configuration Args - Configuration options if http_endpoint is the destination. requires the user to also specify a
s3_configurationblock. More details are given below. - kinesis_
source_ Firehoseconfiguration Delivery Stream Kinesis Source Configuration Args - Allows the ability to specify the kinesis stream that is used as the source of the firehose delivery stream.
- name str
- A name to identify the stream. This is unique to the AWS account and region the Stream is created in. When using for WAF logging, name must be prefixed with
aws-waf-logs-. See AWS Documentation for more details. - opensearch_
configuration FirehoseDelivery Stream Opensearch Configuration Args - Configuration options if opensearch is the destination. More details are given below.
- redshift_
configuration FirehoseDelivery Stream Redshift Configuration Args - Configuration options if redshift is the destination.
Using
redshift_configurationrequires the user to also specify as3_configurationblock. More details are given below. - s3_
configuration FirehoseDelivery Stream S3Configuration Args - Required for non-S3 destinations. For S3 destination, use
extended_s3_configurationinstead. Configuration options for the s3 destination (or the intermediate bucket if the destination is redshift). More details are given below. - server_
side_ Firehoseencryption Delivery Stream Server Side Encryption Args - Encrypt at rest options. Server-side encryption should not be enabled when a kinesis stream is configured as the source of the firehose delivery stream.
- splunk_
configuration FirehoseDelivery Stream Splunk Configuration Args - Configuration options if splunk is the destination. More details are given below.
- Mapping[str, str]
- A map of tags to assign to the resource. If configured with a provider
default_tagsconfiguration block present, tags with matching keys will overwrite those defined at the provider-level. - version_
id str - Specifies the table version for the output data schema. Defaults to
LATEST.
- destination String
- This is the destination to where the data is delivered. The only options are
s3(Deprecated, useextended_s3instead),extended_s3,redshift,elasticsearch,splunk,http_endpointandopensearch. - arn String
- The Amazon Resource Name (ARN) specifying the Stream
- destination
Id String - elasticsearch
Configuration Property Map - Configuration options if elasticsearch is the destination. More details are given below.
- extended
S3Configuration Property Map - Enhanced configuration options for the s3 destination. More details are given below.
- http
Endpoint Property MapConfiguration - Configuration options if http_endpoint is the destination. requires the user to also specify a
s3_configurationblock. More details are given below. - kinesis
Source Property MapConfiguration - Allows the ability to specify the kinesis stream that is used as the source of the firehose delivery stream.
- name String
- A name to identify the stream. This is unique to the AWS account and region the Stream is created in. When using for WAF logging, name must be prefixed with
aws-waf-logs-. See AWS Documentation for more details. - opensearch
Configuration Property Map - Configuration options if opensearch is the destination. More details are given below.
- redshift
Configuration Property Map - Configuration options if redshift is the destination.
Using
redshift_configurationrequires the user to also specify as3_configurationblock. More details are given below. - s3Configuration Property Map
- Required for non-S3 destinations. For S3 destination, use
extended_s3_configurationinstead. Configuration options for the s3 destination (or the intermediate bucket if the destination is redshift). More details are given below. - server
Side Property MapEncryption - Encrypt at rest options. Server-side encryption should not be enabled when a kinesis stream is configured as the source of the firehose delivery stream.
- splunk
Configuration Property Map - Configuration options if splunk is the destination. More details are given below.
- Map<String>
- A map of tags to assign to the resource. If configured with a provider
default_tagsconfiguration block present, tags with matching keys will overwrite those defined at the provider-level. - version
Id String - Specifies the table version for the output data schema. Defaults to
LATEST.
Outputs
All input properties are implicitly available as output properties. Additionally, the FirehoseDeliveryStream resource produces the following output properties:
- Id string
- The provider-assigned unique ID for this managed resource.
- Dictionary<string, string>
- A map of tags assigned to the resource, including those inherited from the provider
default_tagsconfiguration block.
- Id string
- The provider-assigned unique ID for this managed resource.
- map[string]string
- A map of tags assigned to the resource, including those inherited from the provider
default_tagsconfiguration block.
- id String
- The provider-assigned unique ID for this managed resource.
- Map<String,String>
- A map of tags assigned to the resource, including those inherited from the provider
default_tagsconfiguration block.
- id string
- The provider-assigned unique ID for this managed resource.
- {[key: string]: string}
- A map of tags assigned to the resource, including those inherited from the provider
default_tagsconfiguration block.
- id str
- The provider-assigned unique ID for this managed resource.
- Mapping[str, str]
- A map of tags assigned to the resource, including those inherited from the provider
default_tagsconfiguration block.
- id String
- The provider-assigned unique ID for this managed resource.
- Map<String>
- A map of tags assigned to the resource, including those inherited from the provider
default_tagsconfiguration block.
Look up Existing FirehoseDeliveryStream Resource
Get an existing FirehoseDeliveryStream resource’s state with the given name, ID, and optional extra properties used to qualify the lookup.
public static get(name: string, id: Input<ID>, state?: FirehoseDeliveryStreamState, opts?: CustomResourceOptions): FirehoseDeliveryStream@staticmethod
def get(resource_name: str,
id: str,
opts: Optional[ResourceOptions] = None,
arn: Optional[str] = None,
destination: Optional[str] = None,
destination_id: Optional[str] = None,
elasticsearch_configuration: Optional[FirehoseDeliveryStreamElasticsearchConfigurationArgs] = None,
extended_s3_configuration: Optional[FirehoseDeliveryStreamExtendedS3ConfigurationArgs] = None,
http_endpoint_configuration: Optional[FirehoseDeliveryStreamHttpEndpointConfigurationArgs] = None,
kinesis_source_configuration: Optional[FirehoseDeliveryStreamKinesisSourceConfigurationArgs] = None,
name: Optional[str] = None,
opensearch_configuration: Optional[FirehoseDeliveryStreamOpensearchConfigurationArgs] = None,
redshift_configuration: Optional[FirehoseDeliveryStreamRedshiftConfigurationArgs] = None,
s3_configuration: Optional[FirehoseDeliveryStreamS3ConfigurationArgs] = None,
server_side_encryption: Optional[FirehoseDeliveryStreamServerSideEncryptionArgs] = None,
splunk_configuration: Optional[FirehoseDeliveryStreamSplunkConfigurationArgs] = None,
tags: Optional[Mapping[str, str]] = None,
tags_all: Optional[Mapping[str, str]] = None,
version_id: Optional[str] = None) -> FirehoseDeliveryStreamfunc GetFirehoseDeliveryStream(ctx *Context, name string, id IDInput, state *FirehoseDeliveryStreamState, opts ...ResourceOption) (*FirehoseDeliveryStream, error)public static FirehoseDeliveryStream Get(string name, Input<string> id, FirehoseDeliveryStreamState? state, CustomResourceOptions? opts = null)public static FirehoseDeliveryStream get(String name, Output<String> id, FirehoseDeliveryStreamState state, CustomResourceOptions options)resources: _: type: aws:kinesis:FirehoseDeliveryStream get: id: ${id}- name
- The unique name of the resulting resource.
- id
- The unique provider ID of the resource to lookup.
- state
- Any extra arguments used during the lookup.
- opts
- A bag of options that control this resource's behavior.
- resource_name
- The unique name of the resulting resource.
- id
- The unique provider ID of the resource to lookup.
- name
- The unique name of the resulting resource.
- id
- The unique provider ID of the resource to lookup.
- state
- Any extra arguments used during the lookup.
- opts
- A bag of options that control this resource's behavior.
- name
- The unique name of the resulting resource.
- id
- The unique provider ID of the resource to lookup.
- state
- Any extra arguments used during the lookup.
- opts
- A bag of options that control this resource's behavior.
- name
- The unique name of the resulting resource.
- id
- The unique provider ID of the resource to lookup.
- state
- Any extra arguments used during the lookup.
- opts
- A bag of options that control this resource's behavior.
- Arn string
- The Amazon Resource Name (ARN) specifying the Stream
- Destination string
- This is the destination to where the data is delivered. The only options are
s3(Deprecated, useextended_s3instead),extended_s3,redshift,elasticsearch,splunk,http_endpointandopensearch. - Destination
Id string - Elasticsearch
Configuration FirehoseDelivery Stream Elasticsearch Configuration - Configuration options if elasticsearch is the destination. More details are given below.
- Extended
S3Configuration FirehoseDelivery Stream Extended S3Configuration - Enhanced configuration options for the s3 destination. More details are given below.
- Http
Endpoint FirehoseConfiguration Delivery Stream Http Endpoint Configuration - Configuration options if http_endpoint is the destination. requires the user to also specify a
s3_configurationblock. More details are given below. - Kinesis
Source FirehoseConfiguration Delivery Stream Kinesis Source Configuration - Allows the ability to specify the kinesis stream that is used as the source of the firehose delivery stream.
- Name string
- A name to identify the stream. This is unique to the AWS account and region the Stream is created in. When using for WAF logging, name must be prefixed with
aws-waf-logs-. See AWS Documentation for more details. - Opensearch
Configuration FirehoseDelivery Stream Opensearch Configuration - Configuration options if opensearch is the destination. More details are given below.
- Redshift
Configuration FirehoseDelivery Stream Redshift Configuration - Configuration options if redshift is the destination.
Using
redshift_configurationrequires the user to also specify as3_configurationblock. More details are given below. - S3Configuration
Firehose
Delivery Stream S3Configuration - Required for non-S3 destinations. For S3 destination, use
extended_s3_configurationinstead. Configuration options for the s3 destination (or the intermediate bucket if the destination is redshift). More details are given below. - Server
Side FirehoseEncryption Delivery Stream Server Side Encryption - Encrypt at rest options. Server-side encryption should not be enabled when a kinesis stream is configured as the source of the firehose delivery stream.
- Splunk
Configuration FirehoseDelivery Stream Splunk Configuration - Configuration options if splunk is the destination. More details are given below.
- Dictionary<string, string>
- A map of tags to assign to the resource. If configured with a provider
default_tagsconfiguration block present, tags with matching keys will overwrite those defined at the provider-level. - Dictionary<string, string>
- A map of tags assigned to the resource, including those inherited from the provider
default_tagsconfiguration block. - Version
Id string - Specifies the table version for the output data schema. Defaults to
LATEST.
- Arn string
- The Amazon Resource Name (ARN) specifying the Stream
- Destination string
- This is the destination to where the data is delivered. The only options are
s3(Deprecated, useextended_s3instead),extended_s3,redshift,elasticsearch,splunk,http_endpointandopensearch. - Destination
Id string - Elasticsearch
Configuration FirehoseDelivery Stream Elasticsearch Configuration Args - Configuration options if elasticsearch is the destination. More details are given below.
- Extended
S3Configuration FirehoseDelivery Stream Extended S3Configuration Args - Enhanced configuration options for the s3 destination. More details are given below.
- Http
Endpoint FirehoseConfiguration Delivery Stream Http Endpoint Configuration Args - Configuration options if http_endpoint is the destination. requires the user to also specify a
s3_configurationblock. More details are given below. - Kinesis
Source FirehoseConfiguration Delivery Stream Kinesis Source Configuration Args - Allows the ability to specify the kinesis stream that is used as the source of the firehose delivery stream.
- Name string
- A name to identify the stream. This is unique to the AWS account and region the Stream is created in. When using for WAF logging, name must be prefixed with
aws-waf-logs-. See AWS Documentation for more details. - Opensearch
Configuration FirehoseDelivery Stream Opensearch Configuration Args - Configuration options if opensearch is the destination. More details are given below.
- Redshift
Configuration FirehoseDelivery Stream Redshift Configuration Args - Configuration options if redshift is the destination.
Using
redshift_configurationrequires the user to also specify as3_configurationblock. More details are given below. - S3Configuration
Firehose
Delivery Stream S3Configuration Args - Required for non-S3 destinations. For S3 destination, use
extended_s3_configurationinstead. Configuration options for the s3 destination (or the intermediate bucket if the destination is redshift). More details are given below. - Server
Side FirehoseEncryption Delivery Stream Server Side Encryption Args - Encrypt at rest options. Server-side encryption should not be enabled when a kinesis stream is configured as the source of the firehose delivery stream.
- Splunk
Configuration FirehoseDelivery Stream Splunk Configuration Args - Configuration options if splunk is the destination. More details are given below.
- map[string]string
- A map of tags to assign to the resource. If configured with a provider
default_tagsconfiguration block present, tags with matching keys will overwrite those defined at the provider-level. - map[string]string
- A map of tags assigned to the resource, including those inherited from the provider
default_tagsconfiguration block. - Version
Id string - Specifies the table version for the output data schema. Defaults to
LATEST.
- arn String
- The Amazon Resource Name (ARN) specifying the Stream
- destination String
- This is the destination to where the data is delivered. The only options are
s3(Deprecated, useextended_s3instead),extended_s3,redshift,elasticsearch,splunk,http_endpointandopensearch. - destination
Id String - elasticsearch
Configuration FirehoseDelivery Stream Elasticsearch Configuration - Configuration options if elasticsearch is the destination. More details are given below.
- extended
S3Configuration FirehoseDelivery Stream Extended S3Configuration - Enhanced configuration options for the s3 destination. More details are given below.
- http
Endpoint FirehoseConfiguration Delivery Stream Http Endpoint Configuration - Configuration options if http_endpoint is the destination. requires the user to also specify a
s3_configurationblock. More details are given below. - kinesis
Source FirehoseConfiguration Delivery Stream Kinesis Source Configuration - Allows the ability to specify the kinesis stream that is used as the source of the firehose delivery stream.
- name String
- A name to identify the stream. This is unique to the AWS account and region the Stream is created in. When using for WAF logging, name must be prefixed with
aws-waf-logs-. See AWS Documentation for more details. - opensearch
Configuration FirehoseDelivery Stream Opensearch Configuration - Configuration options if opensearch is the destination. More details are given below.
- redshift
Configuration FirehoseDelivery Stream Redshift Configuration - Configuration options if redshift is the destination.
Using
redshift_configurationrequires the user to also specify as3_configurationblock. More details are given below. - s3Configuration
Firehose
Delivery Stream S3Configuration - Required for non-S3 destinations. For S3 destination, use
extended_s3_configurationinstead. Configuration options for the s3 destination (or the intermediate bucket if the destination is redshift). More details are given below. - server
Side FirehoseEncryption Delivery Stream Server Side Encryption - Encrypt at rest options. Server-side encryption should not be enabled when a kinesis stream is configured as the source of the firehose delivery stream.
- splunk
Configuration FirehoseDelivery Stream Splunk Configuration - Configuration options if splunk is the destination. More details are given below.
- Map<String,String>
- A map of tags to assign to the resource. If configured with a provider
default_tagsconfiguration block present, tags with matching keys will overwrite those defined at the provider-level. - Map<String,String>
- A map of tags assigned to the resource, including those inherited from the provider
default_tagsconfiguration block. - version
Id String - Specifies the table version for the output data schema. Defaults to
LATEST.
- arn string
- The Amazon Resource Name (ARN) specifying the Stream
- destination string
- This is the destination to where the data is delivered. The only options are
s3(Deprecated, useextended_s3instead),extended_s3,redshift,elasticsearch,splunk,http_endpointandopensearch. - destination
Id string - elasticsearch
Configuration FirehoseDelivery Stream Elasticsearch Configuration - Configuration options if elasticsearch is the destination. More details are given below.
- extended
S3Configuration FirehoseDelivery Stream Extended S3Configuration - Enhanced configuration options for the s3 destination. More details are given below.
- http
Endpoint FirehoseConfiguration Delivery Stream Http Endpoint Configuration - Configuration options if http_endpoint is the destination. requires the user to also specify a
s3_configurationblock. More details are given below. - kinesis
Source FirehoseConfiguration Delivery Stream Kinesis Source Configuration - Allows the ability to specify the kinesis stream that is used as the source of the firehose delivery stream.
- name string
- A name to identify the stream. This is unique to the AWS account and region the Stream is created in. When using for WAF logging, name must be prefixed with
aws-waf-logs-. See AWS Documentation for more details. - opensearch
Configuration FirehoseDelivery Stream Opensearch Configuration - Configuration options if opensearch is the destination. More details are given below.
- redshift
Configuration FirehoseDelivery Stream Redshift Configuration - Configuration options if redshift is the destination.
Using
redshift_configurationrequires the user to also specify as3_configurationblock. More details are given below. - s3Configuration
Firehose
Delivery Stream S3Configuration - Required for non-S3 destinations. For S3 destination, use
extended_s3_configurationinstead. Configuration options for the s3 destination (or the intermediate bucket if the destination is redshift). More details are given below. - server
Side FirehoseEncryption Delivery Stream Server Side Encryption - Encrypt at rest options. Server-side encryption should not be enabled when a kinesis stream is configured as the source of the firehose delivery stream.
- splunk
Configuration FirehoseDelivery Stream Splunk Configuration - Configuration options if splunk is the destination. More details are given below.
- {[key: string]: string}
- A map of tags to assign to the resource. If configured with a provider
default_tagsconfiguration block present, tags with matching keys will overwrite those defined at the provider-level. - {[key: string]: string}
- A map of tags assigned to the resource, including those inherited from the provider
default_tagsconfiguration block. - version
Id string - Specifies the table version for the output data schema. Defaults to
LATEST.
- arn str
- The Amazon Resource Name (ARN) specifying the Stream
- destination str
- This is the destination to where the data is delivered. The only options are
s3(Deprecated, useextended_s3instead),extended_s3,redshift,elasticsearch,splunk,http_endpointandopensearch. - destination_
id str - elasticsearch_
configuration FirehoseDelivery Stream Elasticsearch Configuration Args - Configuration options if elasticsearch is the destination. More details are given below.
- extended_
s3_ Firehoseconfiguration Delivery Stream Extended S3Configuration Args - Enhanced configuration options for the s3 destination. More details are given below.
- http_
endpoint_ Firehoseconfiguration Delivery Stream Http Endpoint Configuration Args - Configuration options if http_endpoint is the destination. requires the user to also specify a
s3_configurationblock. More details are given below. - kinesis_
source_ Firehoseconfiguration Delivery Stream Kinesis Source Configuration Args - Allows the ability to specify the kinesis stream that is used as the source of the firehose delivery stream.
- name str
- A name to identify the stream. This is unique to the AWS account and region the Stream is created in. When using for WAF logging, name must be prefixed with
aws-waf-logs-. See AWS Documentation for more details. - opensearch_
configuration FirehoseDelivery Stream Opensearch Configuration Args - Configuration options if opensearch is the destination. More details are given below.
- redshift_
configuration FirehoseDelivery Stream Redshift Configuration Args - Configuration options if redshift is the destination.
Using
redshift_configurationrequires the user to also specify as3_configurationblock. More details are given below. - s3_
configuration FirehoseDelivery Stream S3Configuration Args - Required for non-S3 destinations. For S3 destination, use
extended_s3_configurationinstead. Configuration options for the s3 destination (or the intermediate bucket if the destination is redshift). More details are given below. - server_
side_ Firehoseencryption Delivery Stream Server Side Encryption Args - Encrypt at rest options. Server-side encryption should not be enabled when a kinesis stream is configured as the source of the firehose delivery stream.
- splunk_
configuration FirehoseDelivery Stream Splunk Configuration Args - Configuration options if splunk is the destination. More details are given below.
- Mapping[str, str]
- A map of tags to assign to the resource. If configured with a provider
default_tagsconfiguration block present, tags with matching keys will overwrite those defined at the provider-level. - Mapping[str, str]
- A map of tags assigned to the resource, including those inherited from the provider
default_tagsconfiguration block. - version_
id str - Specifies the table version for the output data schema. Defaults to
LATEST.
- arn String
- The Amazon Resource Name (ARN) specifying the Stream
- destination String
- This is the destination to where the data is delivered. The only options are
s3(Deprecated, useextended_s3instead),extended_s3,redshift,elasticsearch,splunk,http_endpointandopensearch. - destination
Id String - elasticsearch
Configuration Property Map - Configuration options if elasticsearch is the destination. More details are given below.
- extended
S3Configuration Property Map - Enhanced configuration options for the s3 destination. More details are given below.
- http
Endpoint Property MapConfiguration - Configuration options if http_endpoint is the destination. requires the user to also specify a
s3_configurationblock. More details are given below. - kinesis
Source Property MapConfiguration - Allows the ability to specify the kinesis stream that is used as the source of the firehose delivery stream.
- name String
- A name to identify the stream. This is unique to the AWS account and region the Stream is created in. When using for WAF logging, name must be prefixed with
aws-waf-logs-. See AWS Documentation for more details. - opensearch
Configuration Property Map - Configuration options if opensearch is the destination. More details are given below.
- redshift
Configuration Property Map - Configuration options if redshift is the destination.
Using
redshift_configurationrequires the user to also specify as3_configurationblock. More details are given below. - s3Configuration Property Map
- Required for non-S3 destinations. For S3 destination, use
extended_s3_configurationinstead. Configuration options for the s3 destination (or the intermediate bucket if the destination is redshift). More details are given below. - server
Side Property MapEncryption - Encrypt at rest options. Server-side encryption should not be enabled when a kinesis stream is configured as the source of the firehose delivery stream.
- splunk
Configuration Property Map - Configuration options if splunk is the destination. More details are given below.
- Map<String>
- A map of tags to assign to the resource. If configured with a provider
default_tagsconfiguration block present, tags with matching keys will overwrite those defined at the provider-level. - Map<String>
- A map of tags assigned to the resource, including those inherited from the provider
default_tagsconfiguration block. - version
Id String - Specifies the table version for the output data schema. Defaults to
LATEST.
Supporting Types
FirehoseDeliveryStreamElasticsearchConfiguration, FirehoseDeliveryStreamElasticsearchConfigurationArgs
- Index
Name string - The Elasticsearch index name.
- Role
Arn string - The ARN of the IAM role to be assumed by Firehose for calling the Amazon ES Configuration API and for indexing documents. The IAM role must have permission for
DescribeElasticsearchDomain,DescribeElasticsearchDomains, andDescribeElasticsearchDomainConfig. The pattern needs to bearn:.*. - Buffering
Interval int - Buffer incoming data for the specified period of time, in seconds between 60 to 900, before delivering it to the destination. The default value is 300s.
- Buffering
Size int - Buffer incoming data to the specified size, in MBs between 1 to 100, before delivering it to the destination. The default value is 5MB.
- Cloudwatch
Logging FirehoseOptions Delivery Stream Elasticsearch Configuration Cloudwatch Logging Options - The CloudWatch Logging Options for the delivery stream. More details are given below
- Cluster
Endpoint string - The endpoint to use when communicating with the cluster. Conflicts with
domain_arn. - Domain
Arn string - The ARN of the Amazon ES domain. The pattern needs to be
arn:.*. Conflicts withcluster_endpoint. - Index
Rotation stringPeriod - The Elasticsearch index rotation period. Index rotation appends a timestamp to the IndexName to facilitate expiration of old data. Valid values are
NoRotation,OneHour,OneDay,OneWeek, andOneMonth. The default value isOneDay. - Processing
Configuration FirehoseDelivery Stream Elasticsearch Configuration Processing Configuration - The data processing configuration. More details are given below.
- Retry
Duration int - After an initial failure to deliver to Amazon Elasticsearch, the total amount of time, in seconds between 0 to 7200, during which Firehose re-attempts delivery (including the first attempt). After this time has elapsed, the failed documents are written to Amazon S3. The default value is 300s. There will be no retry if the value is 0.
- S3Backup
Mode string - Defines how documents should be delivered to Amazon S3. Valid values are
FailedDocumentsOnlyandAllDocuments. Default value isFailedDocumentsOnly. - Type
Name string - The Elasticsearch type name with maximum length of 100 characters.
- Vpc
Config FirehoseDelivery Stream Elasticsearch Configuration Vpc Config - The VPC configuration for the delivery stream to connect to Elastic Search associated with the VPC. More details are given below
- Index
Name string - The Elasticsearch index name.
- Role
Arn string - The ARN of the IAM role to be assumed by Firehose for calling the Amazon ES Configuration API and for indexing documents. The IAM role must have permission for
DescribeElasticsearchDomain,DescribeElasticsearchDomains, andDescribeElasticsearchDomainConfig. The pattern needs to bearn:.*. - Buffering
Interval int - Buffer incoming data for the specified period of time, in seconds between 60 to 900, before delivering it to the destination. The default value is 300s.
- Buffering
Size int - Buffer incoming data to the specified size, in MBs between 1 to 100, before delivering it to the destination. The default value is 5MB.
- Cloudwatch
Logging FirehoseOptions Delivery Stream Elasticsearch Configuration Cloudwatch Logging Options - The CloudWatch Logging Options for the delivery stream. More details are given below
- Cluster
Endpoint string - The endpoint to use when communicating with the cluster. Conflicts with
domain_arn. - Domain
Arn string - The ARN of the Amazon ES domain. The pattern needs to be
arn:.*. Conflicts withcluster_endpoint. - Index
Rotation stringPeriod - The Elasticsearch index rotation period. Index rotation appends a timestamp to the IndexName to facilitate expiration of old data. Valid values are
NoRotation,OneHour,OneDay,OneWeek, andOneMonth. The default value isOneDay. - Processing
Configuration FirehoseDelivery Stream Elasticsearch Configuration Processing Configuration - The data processing configuration. More details are given below.
- Retry
Duration int - After an initial failure to deliver to Amazon Elasticsearch, the total amount of time, in seconds between 0 to 7200, during which Firehose re-attempts delivery (including the first attempt). After this time has elapsed, the failed documents are written to Amazon S3. The default value is 300s. There will be no retry if the value is 0.
- S3Backup
Mode string - Defines how documents should be delivered to Amazon S3. Valid values are
FailedDocumentsOnlyandAllDocuments. Default value isFailedDocumentsOnly. - Type
Name string - The Elasticsearch type name with maximum length of 100 characters.
- Vpc
Config FirehoseDelivery Stream Elasticsearch Configuration Vpc Config - The VPC configuration for the delivery stream to connect to Elastic Search associated with the VPC. More details are given below
- index
Name String - The Elasticsearch index name.
- role
Arn String - The ARN of the IAM role to be assumed by Firehose for calling the Amazon ES Configuration API and for indexing documents. The IAM role must have permission for
DescribeElasticsearchDomain,DescribeElasticsearchDomains, andDescribeElasticsearchDomainConfig. The pattern needs to bearn:.*. - buffering
Interval Integer - Buffer incoming data for the specified period of time, in seconds between 60 to 900, before delivering it to the destination. The default value is 300s.
- buffering
Size Integer - Buffer incoming data to the specified size, in MBs between 1 to 100, before delivering it to the destination. The default value is 5MB.
- cloudwatch
Logging FirehoseOptions Delivery Stream Elasticsearch Configuration Cloudwatch Logging Options - The CloudWatch Logging Options for the delivery stream. More details are given below
- cluster
Endpoint String - The endpoint to use when communicating with the cluster. Conflicts with
domain_arn. - domain
Arn String - The ARN of the Amazon ES domain. The pattern needs to be
arn:.*. Conflicts withcluster_endpoint. - index
Rotation StringPeriod - The Elasticsearch index rotation period. Index rotation appends a timestamp to the IndexName to facilitate expiration of old data. Valid values are
NoRotation,OneHour,OneDay,OneWeek, andOneMonth. The default value isOneDay. - processing
Configuration FirehoseDelivery Stream Elasticsearch Configuration Processing Configuration - The data processing configuration. More details are given below.
- retry
Duration Integer - After an initial failure to deliver to Amazon Elasticsearch, the total amount of time, in seconds between 0 to 7200, during which Firehose re-attempts delivery (including the first attempt). After this time has elapsed, the failed documents are written to Amazon S3. The default value is 300s. There will be no retry if the value is 0.
- s3Backup
Mode String - Defines how documents should be delivered to Amazon S3. Valid values are
FailedDocumentsOnlyandAllDocuments. Default value isFailedDocumentsOnly. - type
Name String - The Elasticsearch type name with maximum length of 100 characters.
- vpc
Config FirehoseDelivery Stream Elasticsearch Configuration Vpc Config - The VPC configuration for the delivery stream to connect to Elastic Search associated with the VPC. More details are given below
- index
Name string - The Elasticsearch index name.
- role
Arn string - The ARN of the IAM role to be assumed by Firehose for calling the Amazon ES Configuration API and for indexing documents. The IAM role must have permission for
DescribeElasticsearchDomain,DescribeElasticsearchDomains, andDescribeElasticsearchDomainConfig. The pattern needs to bearn:.*. - buffering
Interval number - Buffer incoming data for the specified period of time, in seconds between 60 to 900, before delivering it to the destination. The default value is 300s.
- buffering
Size number - Buffer incoming data to the specified size, in MBs between 1 to 100, before delivering it to the destination. The default value is 5MB.
- cloudwatch
Logging FirehoseOptions Delivery Stream Elasticsearch Configuration Cloudwatch Logging Options - The CloudWatch Logging Options for the delivery stream. More details are given below
- cluster
Endpoint string - The endpoint to use when communicating with the cluster. Conflicts with
domain_arn. - domain
Arn string - The ARN of the Amazon ES domain. The pattern needs to be
arn:.*. Conflicts withcluster_endpoint. - index
Rotation stringPeriod - The Elasticsearch index rotation period. Index rotation appends a timestamp to the IndexName to facilitate expiration of old data. Valid values are
NoRotation,OneHour,OneDay,OneWeek, andOneMonth. The default value isOneDay. - processing
Configuration FirehoseDelivery Stream Elasticsearch Configuration Processing Configuration - The data processing configuration. More details are given below.
- retry
Duration number - After an initial failure to deliver to Amazon Elasticsearch, the total amount of time, in seconds between 0 to 7200, during which Firehose re-attempts delivery (including the first attempt). After this time has elapsed, the failed documents are written to Amazon S3. The default value is 300s. There will be no retry if the value is 0.
- s3Backup
Mode string - Defines how documents should be delivered to Amazon S3. Valid values are
FailedDocumentsOnlyandAllDocuments. Default value isFailedDocumentsOnly. - type
Name string - The Elasticsearch type name with maximum length of 100 characters.
- vpc
Config FirehoseDelivery Stream Elasticsearch Configuration Vpc Config - The VPC configuration for the delivery stream to connect to Elastic Search associated with the VPC. More details are given below
- index_
name str - The Elasticsearch index name.
- role_
arn str - The ARN of the IAM role to be assumed by Firehose for calling the Amazon ES Configuration API and for indexing documents. The IAM role must have permission for
DescribeElasticsearchDomain,DescribeElasticsearchDomains, andDescribeElasticsearchDomainConfig. The pattern needs to bearn:.*. - buffering_
interval int - Buffer incoming data for the specified period of time, in seconds between 60 to 900, before delivering it to the destination. The default value is 300s.
- buffering_
size int - Buffer incoming data to the specified size, in MBs between 1 to 100, before delivering it to the destination. The default value is 5MB.
- cloudwatch_
logging_ Firehoseoptions Delivery Stream Elasticsearch Configuration Cloudwatch Logging Options - The CloudWatch Logging Options for the delivery stream. More details are given below
- cluster_
endpoint str - The endpoint to use when communicating with the cluster. Conflicts with
domain_arn. - domain_
arn str - The ARN of the Amazon ES domain. The pattern needs to be
arn:.*. Conflicts withcluster_endpoint. - index_
rotation_ strperiod - The Elasticsearch index rotation period. Index rotation appends a timestamp to the IndexName to facilitate expiration of old data. Valid values are
NoRotation,OneHour,OneDay,OneWeek, andOneMonth. The default value isOneDay. - processing_
configuration FirehoseDelivery Stream Elasticsearch Configuration Processing Configuration - The data processing configuration. More details are given below.
- retry_
duration int - After an initial failure to deliver to Amazon Elasticsearch, the total amount of time, in seconds between 0 to 7200, during which Firehose re-attempts delivery (including the first attempt). After this time has elapsed, the failed documents are written to Amazon S3. The default value is 300s. There will be no retry if the value is 0.
- s3_
backup_ strmode - Defines how documents should be delivered to Amazon S3. Valid values are
FailedDocumentsOnlyandAllDocuments. Default value isFailedDocumentsOnly. - type_
name str - The Elasticsearch type name with maximum length of 100 characters.
- vpc_
config FirehoseDelivery Stream Elasticsearch Configuration Vpc Config - The VPC configuration for the delivery stream to connect to Elastic Search associated with the VPC. More details are given below
- index
Name String - The Elasticsearch index name.
- role
Arn String - The ARN of the IAM role to be assumed by Firehose for calling the Amazon ES Configuration API and for indexing documents. The IAM role must have permission for
DescribeElasticsearchDomain,DescribeElasticsearchDomains, andDescribeElasticsearchDomainConfig. The pattern needs to bearn:.*. - buffering
Interval Number - Buffer incoming data for the specified period of time, in seconds between 60 to 900, before delivering it to the destination. The default value is 300s.
- buffering
Size Number - Buffer incoming data to the specified size, in MBs between 1 to 100, before delivering it to the destination. The default value is 5MB.
- cloudwatch
Logging Property MapOptions - The CloudWatch Logging Options for the delivery stream. More details are given below
- cluster
Endpoint String - The endpoint to use when communicating with the cluster. Conflicts with
domain_arn. - domain
Arn String - The ARN of the Amazon ES domain. The pattern needs to be
arn:.*. Conflicts withcluster_endpoint. - index
Rotation StringPeriod - The Elasticsearch index rotation period. Index rotation appends a timestamp to the IndexName to facilitate expiration of old data. Valid values are
NoRotation,OneHour,OneDay,OneWeek, andOneMonth. The default value isOneDay. - processing
Configuration Property Map - The data processing configuration. More details are given below.
- retry
Duration Number - After an initial failure to deliver to Amazon Elasticsearch, the total amount of time, in seconds between 0 to 7200, during which Firehose re-attempts delivery (including the first attempt). After this time has elapsed, the failed documents are written to Amazon S3. The default value is 300s. There will be no retry if the value is 0.
- s3Backup
Mode String - Defines how documents should be delivered to Amazon S3. Valid values are
FailedDocumentsOnlyandAllDocuments. Default value isFailedDocumentsOnly. - type
Name String - The Elasticsearch type name with maximum length of 100 characters.
- vpc
Config Property Map - The VPC configuration for the delivery stream to connect to Elastic Search associated with the VPC. More details are given below
FirehoseDeliveryStreamElasticsearchConfigurationCloudwatchLoggingOptions, FirehoseDeliveryStreamElasticsearchConfigurationCloudwatchLoggingOptionsArgs
- Enabled bool
- Enables or disables the logging. Defaults to
false. - Log
Group stringName - The CloudWatch group name for logging. This value is required if
enabledis true. - Log
Stream stringName - The CloudWatch log stream name for logging. This value is required if
enabledis true.
- Enabled bool
- Enables or disables the logging. Defaults to
false. - Log
Group stringName - The CloudWatch group name for logging. This value is required if
enabledis true. - Log
Stream stringName - The CloudWatch log stream name for logging. This value is required if
enabledis true.
- enabled Boolean
- Enables or disables the logging. Defaults to
false. - log
Group StringName - The CloudWatch group name for logging. This value is required if
enabledis true. - log
Stream StringName - The CloudWatch log stream name for logging. This value is required if
enabledis true.
- enabled boolean
- Enables or disables the logging. Defaults to
false. - log
Group stringName - The CloudWatch group name for logging. This value is required if
enabledis true. - log
Stream stringName - The CloudWatch log stream name for logging. This value is required if
enabledis true.
- enabled bool
- Enables or disables the logging. Defaults to
false. - log_
group_ strname - The CloudWatch group name for logging. This value is required if
enabledis true. - log_
stream_ strname - The CloudWatch log stream name for logging. This value is required if
enabledis true.
- enabled Boolean
- Enables or disables the logging. Defaults to
false. - log
Group StringName - The CloudWatch group name for logging. This value is required if
enabledis true. - log
Stream StringName - The CloudWatch log stream name for logging. This value is required if
enabledis true.
FirehoseDeliveryStreamElasticsearchConfigurationProcessingConfiguration, FirehoseDeliveryStreamElasticsearchConfigurationProcessingConfigurationArgs
- Enabled bool
- Enables or disables data processing.
- Processors
List<Firehose
Delivery Stream Elasticsearch Configuration Processing Configuration Processor> - Array of data processors. More details are given below
- Enabled bool
- Enables or disables data processing.
- Processors
[]Firehose
Delivery Stream Elasticsearch Configuration Processing Configuration Processor - Array of data processors. More details are given below
- enabled Boolean
- Enables or disables data processing.
- processors
List<Firehose
Delivery Stream Elasticsearch Configuration Processing Configuration Processor> - Array of data processors. More details are given below
- enabled boolean
- Enables or disables data processing.
- processors
Firehose
Delivery Stream Elasticsearch Configuration Processing Configuration Processor[] - Array of data processors. More details are given below
- enabled bool
- Enables or disables data processing.
- processors
Sequence[Firehose
Delivery Stream Elasticsearch Configuration Processing Configuration Processor] - Array of data processors. More details are given below
- enabled Boolean
- Enables or disables data processing.
- processors List<Property Map>
- Array of data processors. More details are given below
FirehoseDeliveryStreamElasticsearchConfigurationProcessingConfigurationProcessor, FirehoseDeliveryStreamElasticsearchConfigurationProcessingConfigurationProcessorArgs
- Type string
- The type of processor. Valid Values:
RecordDeAggregation,Lambda,MetadataExtraction,AppendDelimiterToRecord. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - Parameters
List<Firehose
Delivery Stream Elasticsearch Configuration Processing Configuration Processor Parameter> - Array of processor parameters. More details are given below
- Type string
- The type of processor. Valid Values:
RecordDeAggregation,Lambda,MetadataExtraction,AppendDelimiterToRecord. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - Parameters
[]Firehose
Delivery Stream Elasticsearch Configuration Processing Configuration Processor Parameter - Array of processor parameters. More details are given below
- type String
- The type of processor. Valid Values:
RecordDeAggregation,Lambda,MetadataExtraction,AppendDelimiterToRecord. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - parameters
List<Firehose
Delivery Stream Elasticsearch Configuration Processing Configuration Processor Parameter> - Array of processor parameters. More details are given below
- type string
- The type of processor. Valid Values:
RecordDeAggregation,Lambda,MetadataExtraction,AppendDelimiterToRecord. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - parameters
Firehose
Delivery Stream Elasticsearch Configuration Processing Configuration Processor Parameter[] - Array of processor parameters. More details are given below
- type str
- The type of processor. Valid Values:
RecordDeAggregation,Lambda,MetadataExtraction,AppendDelimiterToRecord. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - parameters
Sequence[Firehose
Delivery Stream Elasticsearch Configuration Processing Configuration Processor Parameter] - Array of processor parameters. More details are given below
- type String
- The type of processor. Valid Values:
RecordDeAggregation,Lambda,MetadataExtraction,AppendDelimiterToRecord. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - parameters List<Property Map>
- Array of processor parameters. More details are given below
FirehoseDeliveryStreamElasticsearchConfigurationProcessingConfigurationProcessorParameter, FirehoseDeliveryStreamElasticsearchConfigurationProcessingConfigurationProcessorParameterArgs
- Parameter
Name string - Parameter name. Valid Values:
LambdaArn,NumberOfRetries,MetadataExtractionQuery,JsonParsingEngine,RoleArn,BufferSizeInMBs,BufferIntervalInSeconds,SubRecordType,Delimiter. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - Parameter
Value string Parameter value. Must be between 1 and 512 length (inclusive). When providing a Lambda ARN, you should specify the resource version as well.
NOTE: Parameters with default values, including
NumberOfRetries(default: 3),RoleArn(default: firehose role ARN),BufferSizeInMBs(default: 3), andBufferIntervalInSeconds(default: 60), are not stored in state. To prevent perpetual differences, it is therefore recommended to only include parameters with non-default values.
- Parameter
Name string - Parameter name. Valid Values:
LambdaArn,NumberOfRetries,MetadataExtractionQuery,JsonParsingEngine,RoleArn,BufferSizeInMBs,BufferIntervalInSeconds,SubRecordType,Delimiter. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - Parameter
Value string Parameter value. Must be between 1 and 512 length (inclusive). When providing a Lambda ARN, you should specify the resource version as well.
NOTE: Parameters with default values, including
NumberOfRetries(default: 3),RoleArn(default: firehose role ARN),BufferSizeInMBs(default: 3), andBufferIntervalInSeconds(default: 60), are not stored in state. To prevent perpetual differences, it is therefore recommended to only include parameters with non-default values.
- parameter
Name String - Parameter name. Valid Values:
LambdaArn,NumberOfRetries,MetadataExtractionQuery,JsonParsingEngine,RoleArn,BufferSizeInMBs,BufferIntervalInSeconds,SubRecordType,Delimiter. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - parameter
Value String Parameter value. Must be between 1 and 512 length (inclusive). When providing a Lambda ARN, you should specify the resource version as well.
NOTE: Parameters with default values, including
NumberOfRetries(default: 3),RoleArn(default: firehose role ARN),BufferSizeInMBs(default: 3), andBufferIntervalInSeconds(default: 60), are not stored in state. To prevent perpetual differences, it is therefore recommended to only include parameters with non-default values.
- parameter
Name string - Parameter name. Valid Values:
LambdaArn,NumberOfRetries,MetadataExtractionQuery,JsonParsingEngine,RoleArn,BufferSizeInMBs,BufferIntervalInSeconds,SubRecordType,Delimiter. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - parameter
Value string Parameter value. Must be between 1 and 512 length (inclusive). When providing a Lambda ARN, you should specify the resource version as well.
NOTE: Parameters with default values, including
NumberOfRetries(default: 3),RoleArn(default: firehose role ARN),BufferSizeInMBs(default: 3), andBufferIntervalInSeconds(default: 60), are not stored in state. To prevent perpetual differences, it is therefore recommended to only include parameters with non-default values.
- parameter_
name str - Parameter name. Valid Values:
LambdaArn,NumberOfRetries,MetadataExtractionQuery,JsonParsingEngine,RoleArn,BufferSizeInMBs,BufferIntervalInSeconds,SubRecordType,Delimiter. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - parameter_
value str Parameter value. Must be between 1 and 512 length (inclusive). When providing a Lambda ARN, you should specify the resource version as well.
NOTE: Parameters with default values, including
NumberOfRetries(default: 3),RoleArn(default: firehose role ARN),BufferSizeInMBs(default: 3), andBufferIntervalInSeconds(default: 60), are not stored in state. To prevent perpetual differences, it is therefore recommended to only include parameters with non-default values.
- parameter
Name String - Parameter name. Valid Values:
LambdaArn,NumberOfRetries,MetadataExtractionQuery,JsonParsingEngine,RoleArn,BufferSizeInMBs,BufferIntervalInSeconds,SubRecordType,Delimiter. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - parameter
Value String Parameter value. Must be between 1 and 512 length (inclusive). When providing a Lambda ARN, you should specify the resource version as well.
NOTE: Parameters with default values, including
NumberOfRetries(default: 3),RoleArn(default: firehose role ARN),BufferSizeInMBs(default: 3), andBufferIntervalInSeconds(default: 60), are not stored in state. To prevent perpetual differences, it is therefore recommended to only include parameters with non-default values.
FirehoseDeliveryStreamElasticsearchConfigurationVpcConfig, FirehoseDeliveryStreamElasticsearchConfigurationVpcConfigArgs
- Role
Arn string - The ARN of the IAM role to be assumed by Firehose for calling the Amazon EC2 configuration API and for creating network interfaces. Make sure role has necessary IAM permissions
- Security
Group List<string>Ids - A list of security group IDs to associate with Kinesis Firehose.
- Subnet
Ids List<string> - A list of subnet IDs to associate with Kinesis Firehose.
- Vpc
Id string
- Role
Arn string - The ARN of the IAM role to be assumed by Firehose for calling the Amazon EC2 configuration API and for creating network interfaces. Make sure role has necessary IAM permissions
- Security
Group []stringIds - A list of security group IDs to associate with Kinesis Firehose.
- Subnet
Ids []string - A list of subnet IDs to associate with Kinesis Firehose.
- Vpc
Id string
- role
Arn String - The ARN of the IAM role to be assumed by Firehose for calling the Amazon EC2 configuration API and for creating network interfaces. Make sure role has necessary IAM permissions
- security
Group List<String>Ids - A list of security group IDs to associate with Kinesis Firehose.
- subnet
Ids List<String> - A list of subnet IDs to associate with Kinesis Firehose.
- vpc
Id String
- role
Arn string - The ARN of the IAM role to be assumed by Firehose for calling the Amazon EC2 configuration API and for creating network interfaces. Make sure role has necessary IAM permissions
- security
Group string[]Ids - A list of security group IDs to associate with Kinesis Firehose.
- subnet
Ids string[] - A list of subnet IDs to associate with Kinesis Firehose.
- vpc
Id string
- role_
arn str - The ARN of the IAM role to be assumed by Firehose for calling the Amazon EC2 configuration API and for creating network interfaces. Make sure role has necessary IAM permissions
- security_
group_ Sequence[str]ids - A list of security group IDs to associate with Kinesis Firehose.
- subnet_
ids Sequence[str] - A list of subnet IDs to associate with Kinesis Firehose.
- vpc_
id str
- role
Arn String - The ARN of the IAM role to be assumed by Firehose for calling the Amazon EC2 configuration API and for creating network interfaces. Make sure role has necessary IAM permissions
- security
Group List<String>Ids - A list of security group IDs to associate with Kinesis Firehose.
- subnet
Ids List<String> - A list of subnet IDs to associate with Kinesis Firehose.
- vpc
Id String
FirehoseDeliveryStreamExtendedS3Configuration, FirehoseDeliveryStreamExtendedS3ConfigurationArgs
- Bucket
Arn string - The ARN of the S3 bucket
- Role
Arn string - The ARN of the role that provides access to the source Kinesis stream.
- Buffer
Interval int - Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination. The default value is 300.
- Buffer
Size int - Buffer incoming data to the specified size, in MBs, before delivering it to the destination. The default value is 5. We recommend setting SizeInMBs to a value greater than the amount of data you typically ingest into the delivery stream in 10 seconds. For example, if you typically ingest data at 1 MB/sec set SizeInMBs to be 10 MB or higher.
- Cloudwatch
Logging FirehoseOptions Delivery Stream Extended S3Configuration Cloudwatch Logging Options The CloudWatch Logging Options for the delivery stream. More details are given below
The
extended_s3_configurationobject supports the same fields froms3_configurationas well as the following:- Compression
Format string - The compression format. If no value is specified, the default is
UNCOMPRESSED. Other supported values areGZIP,ZIP,Snappy, &HADOOP_SNAPPY. - Data
Format FirehoseConversion Configuration Delivery Stream Extended S3Configuration Data Format Conversion Configuration - Nested argument for the serializer, deserializer, and schema for converting data from the JSON format to the Parquet or ORC format before writing it to Amazon S3. More details given below.
- Dynamic
Partitioning FirehoseConfiguration Delivery Stream Extended S3Configuration Dynamic Partitioning Configuration - The configuration for dynamic partitioning. See Dynamic Partitioning Configuration below for more details. Required when using dynamic partitioning.
- Error
Output stringPrefix - Prefix added to failed records before writing them to S3. Not currently supported for
redshiftdestination. This prefix appears immediately following the bucket name. For information about how to specify this prefix, see Custom Prefixes for Amazon S3 Objects. - Kms
Key stringArn - Specifies the KMS key ARN the stream will use to encrypt data. If not set, no encryption will be used.
- Prefix string
- The "YYYY/MM/DD/HH" time format prefix is automatically used for delivered S3 files. You can specify an extra prefix to be added in front of the time format prefix. Note that if the prefix ends with a slash, it appears as a folder in the S3 bucket
- Processing
Configuration FirehoseDelivery Stream Extended S3Configuration Processing Configuration - The data processing configuration. More details are given below.
- S3Backup
Configuration FirehoseDelivery Stream Extended S3Configuration S3Backup Configuration - The configuration for backup in Amazon S3. Required if
s3_backup_modeisEnabled. Supports the same fields ass3_configurationobject. - S3Backup
Mode string - The Amazon S3 backup mode. Valid values are
DisabledandEnabled. Default value isDisabled.
- Bucket
Arn string - The ARN of the S3 bucket
- Role
Arn string - The ARN of the role that provides access to the source Kinesis stream.
- Buffer
Interval int - Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination. The default value is 300.
- Buffer
Size int - Buffer incoming data to the specified size, in MBs, before delivering it to the destination. The default value is 5. We recommend setting SizeInMBs to a value greater than the amount of data you typically ingest into the delivery stream in 10 seconds. For example, if you typically ingest data at 1 MB/sec set SizeInMBs to be 10 MB or higher.
- Cloudwatch
Logging FirehoseOptions Delivery Stream Extended S3Configuration Cloudwatch Logging Options The CloudWatch Logging Options for the delivery stream. More details are given below
The
extended_s3_configurationobject supports the same fields froms3_configurationas well as the following:- Compression
Format string - The compression format. If no value is specified, the default is
UNCOMPRESSED. Other supported values areGZIP,ZIP,Snappy, &HADOOP_SNAPPY. - Data
Format FirehoseConversion Configuration Delivery Stream Extended S3Configuration Data Format Conversion Configuration - Nested argument for the serializer, deserializer, and schema for converting data from the JSON format to the Parquet or ORC format before writing it to Amazon S3. More details given below.
- Dynamic
Partitioning FirehoseConfiguration Delivery Stream Extended S3Configuration Dynamic Partitioning Configuration - The configuration for dynamic partitioning. See Dynamic Partitioning Configuration below for more details. Required when using dynamic partitioning.
- Error
Output stringPrefix - Prefix added to failed records before writing them to S3. Not currently supported for
redshiftdestination. This prefix appears immediately following the bucket name. For information about how to specify this prefix, see Custom Prefixes for Amazon S3 Objects. - Kms
Key stringArn - Specifies the KMS key ARN the stream will use to encrypt data. If not set, no encryption will be used.
- Prefix string
- The "YYYY/MM/DD/HH" time format prefix is automatically used for delivered S3 files. You can specify an extra prefix to be added in front of the time format prefix. Note that if the prefix ends with a slash, it appears as a folder in the S3 bucket
- Processing
Configuration FirehoseDelivery Stream Extended S3Configuration Processing Configuration - The data processing configuration. More details are given below.
- S3Backup
Configuration FirehoseDelivery Stream Extended S3Configuration S3Backup Configuration - The configuration for backup in Amazon S3. Required if
s3_backup_modeisEnabled. Supports the same fields ass3_configurationobject. - S3Backup
Mode string - The Amazon S3 backup mode. Valid values are
DisabledandEnabled. Default value isDisabled.
- bucket
Arn String - The ARN of the S3 bucket
- role
Arn String - The ARN of the role that provides access to the source Kinesis stream.
- buffer
Interval Integer - Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination. The default value is 300.
- buffer
Size Integer - Buffer incoming data to the specified size, in MBs, before delivering it to the destination. The default value is 5. We recommend setting SizeInMBs to a value greater than the amount of data you typically ingest into the delivery stream in 10 seconds. For example, if you typically ingest data at 1 MB/sec set SizeInMBs to be 10 MB or higher.
- cloudwatch
Logging FirehoseOptions Delivery Stream Extended S3Configuration Cloudwatch Logging Options The CloudWatch Logging Options for the delivery stream. More details are given below
The
extended_s3_configurationobject supports the same fields froms3_configurationas well as the following:- compression
Format String - The compression format. If no value is specified, the default is
UNCOMPRESSED. Other supported values areGZIP,ZIP,Snappy, &HADOOP_SNAPPY. - data
Format FirehoseConversion Configuration Delivery Stream Extended S3Configuration Data Format Conversion Configuration - Nested argument for the serializer, deserializer, and schema for converting data from the JSON format to the Parquet or ORC format before writing it to Amazon S3. More details given below.
- dynamic
Partitioning FirehoseConfiguration Delivery Stream Extended S3Configuration Dynamic Partitioning Configuration - The configuration for dynamic partitioning. See Dynamic Partitioning Configuration below for more details. Required when using dynamic partitioning.
- error
Output StringPrefix - Prefix added to failed records before writing them to S3. Not currently supported for
redshiftdestination. This prefix appears immediately following the bucket name. For information about how to specify this prefix, see Custom Prefixes for Amazon S3 Objects. - kms
Key StringArn - Specifies the KMS key ARN the stream will use to encrypt data. If not set, no encryption will be used.
- prefix String
- The "YYYY/MM/DD/HH" time format prefix is automatically used for delivered S3 files. You can specify an extra prefix to be added in front of the time format prefix. Note that if the prefix ends with a slash, it appears as a folder in the S3 bucket
- processing
Configuration FirehoseDelivery Stream Extended S3Configuration Processing Configuration - The data processing configuration. More details are given below.
- s3Backup
Configuration FirehoseDelivery Stream Extended S3Configuration S3Backup Configuration - The configuration for backup in Amazon S3. Required if
s3_backup_modeisEnabled. Supports the same fields ass3_configurationobject. - s3Backup
Mode String - The Amazon S3 backup mode. Valid values are
DisabledandEnabled. Default value isDisabled.
- bucket
Arn string - The ARN of the S3 bucket
- role
Arn string - The ARN of the role that provides access to the source Kinesis stream.
- buffer
Interval number - Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination. The default value is 300.
- buffer
Size number - Buffer incoming data to the specified size, in MBs, before delivering it to the destination. The default value is 5. We recommend setting SizeInMBs to a value greater than the amount of data you typically ingest into the delivery stream in 10 seconds. For example, if you typically ingest data at 1 MB/sec set SizeInMBs to be 10 MB or higher.
- cloudwatch
Logging FirehoseOptions Delivery Stream Extended S3Configuration Cloudwatch Logging Options The CloudWatch Logging Options for the delivery stream. More details are given below
The
extended_s3_configurationobject supports the same fields froms3_configurationas well as the following:- compression
Format string - The compression format. If no value is specified, the default is
UNCOMPRESSED. Other supported values areGZIP,ZIP,Snappy, &HADOOP_SNAPPY. - data
Format FirehoseConversion Configuration Delivery Stream Extended S3Configuration Data Format Conversion Configuration - Nested argument for the serializer, deserializer, and schema for converting data from the JSON format to the Parquet or ORC format before writing it to Amazon S3. More details given below.
- dynamic
Partitioning FirehoseConfiguration Delivery Stream Extended S3Configuration Dynamic Partitioning Configuration - The configuration for dynamic partitioning. See Dynamic Partitioning Configuration below for more details. Required when using dynamic partitioning.
- error
Output stringPrefix - Prefix added to failed records before writing them to S3. Not currently supported for
redshiftdestination. This prefix appears immediately following the bucket name. For information about how to specify this prefix, see Custom Prefixes for Amazon S3 Objects. - kms
Key stringArn - Specifies the KMS key ARN the stream will use to encrypt data. If not set, no encryption will be used.
- prefix string
- The "YYYY/MM/DD/HH" time format prefix is automatically used for delivered S3 files. You can specify an extra prefix to be added in front of the time format prefix. Note that if the prefix ends with a slash, it appears as a folder in the S3 bucket
- processing
Configuration FirehoseDelivery Stream Extended S3Configuration Processing Configuration - The data processing configuration. More details are given below.
- s3Backup
Configuration FirehoseDelivery Stream Extended S3Configuration S3Backup Configuration - The configuration for backup in Amazon S3. Required if
s3_backup_modeisEnabled. Supports the same fields ass3_configurationobject. - s3Backup
Mode string - The Amazon S3 backup mode. Valid values are
DisabledandEnabled. Default value isDisabled.
- bucket_
arn str - The ARN of the S3 bucket
- role_
arn str - The ARN of the role that provides access to the source Kinesis stream.
- buffer_
interval int - Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination. The default value is 300.
- buffer_
size int - Buffer incoming data to the specified size, in MBs, before delivering it to the destination. The default value is 5. We recommend setting SizeInMBs to a value greater than the amount of data you typically ingest into the delivery stream in 10 seconds. For example, if you typically ingest data at 1 MB/sec set SizeInMBs to be 10 MB or higher.
- cloudwatch_
logging_ Firehoseoptions Delivery Stream Extended S3Configuration Cloudwatch Logging Options The CloudWatch Logging Options for the delivery stream. More details are given below
The
extended_s3_configurationobject supports the same fields froms3_configurationas well as the following:- compression_
format str - The compression format. If no value is specified, the default is
UNCOMPRESSED. Other supported values areGZIP,ZIP,Snappy, &HADOOP_SNAPPY. - data_
format_ Firehoseconversion_ configuration Delivery Stream Extended S3Configuration Data Format Conversion Configuration - Nested argument for the serializer, deserializer, and schema for converting data from the JSON format to the Parquet or ORC format before writing it to Amazon S3. More details given below.
- dynamic_
partitioning_ Firehoseconfiguration Delivery Stream Extended S3Configuration Dynamic Partitioning Configuration - The configuration for dynamic partitioning. See Dynamic Partitioning Configuration below for more details. Required when using dynamic partitioning.
- error_
output_ strprefix - Prefix added to failed records before writing them to S3. Not currently supported for
redshiftdestination. This prefix appears immediately following the bucket name. For information about how to specify this prefix, see Custom Prefixes for Amazon S3 Objects. - kms_
key_ strarn - Specifies the KMS key ARN the stream will use to encrypt data. If not set, no encryption will be used.
- prefix str
- The "YYYY/MM/DD/HH" time format prefix is automatically used for delivered S3 files. You can specify an extra prefix to be added in front of the time format prefix. Note that if the prefix ends with a slash, it appears as a folder in the S3 bucket
- processing_
configuration FirehoseDelivery Stream Extended S3Configuration Processing Configuration - The data processing configuration. More details are given below.
- s3_
backup_ Firehoseconfiguration Delivery Stream Extended S3Configuration S3Backup Configuration - The configuration for backup in Amazon S3. Required if
s3_backup_modeisEnabled. Supports the same fields ass3_configurationobject. - s3_
backup_ strmode - The Amazon S3 backup mode. Valid values are
DisabledandEnabled. Default value isDisabled.
- bucket
Arn String - The ARN of the S3 bucket
- role
Arn String - The ARN of the role that provides access to the source Kinesis stream.
- buffer
Interval Number - Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination. The default value is 300.
- buffer
Size Number - Buffer incoming data to the specified size, in MBs, before delivering it to the destination. The default value is 5. We recommend setting SizeInMBs to a value greater than the amount of data you typically ingest into the delivery stream in 10 seconds. For example, if you typically ingest data at 1 MB/sec set SizeInMBs to be 10 MB or higher.
- cloudwatch
Logging Property MapOptions The CloudWatch Logging Options for the delivery stream. More details are given below
The
extended_s3_configurationobject supports the same fields froms3_configurationas well as the following:- compression
Format String - The compression format. If no value is specified, the default is
UNCOMPRESSED. Other supported values areGZIP,ZIP,Snappy, &HADOOP_SNAPPY. - data
Format Property MapConversion Configuration - Nested argument for the serializer, deserializer, and schema for converting data from the JSON format to the Parquet or ORC format before writing it to Amazon S3. More details given below.
- dynamic
Partitioning Property MapConfiguration - The configuration for dynamic partitioning. See Dynamic Partitioning Configuration below for more details. Required when using dynamic partitioning.
- error
Output StringPrefix - Prefix added to failed records before writing them to S3. Not currently supported for
redshiftdestination. This prefix appears immediately following the bucket name. For information about how to specify this prefix, see Custom Prefixes for Amazon S3 Objects. - kms
Key StringArn - Specifies the KMS key ARN the stream will use to encrypt data. If not set, no encryption will be used.
- prefix String
- The "YYYY/MM/DD/HH" time format prefix is automatically used for delivered S3 files. You can specify an extra prefix to be added in front of the time format prefix. Note that if the prefix ends with a slash, it appears as a folder in the S3 bucket
- processing
Configuration Property Map - The data processing configuration. More details are given below.
- s3Backup
Configuration Property Map - The configuration for backup in Amazon S3. Required if
s3_backup_modeisEnabled. Supports the same fields ass3_configurationobject. - s3Backup
Mode String - The Amazon S3 backup mode. Valid values are
DisabledandEnabled. Default value isDisabled.
FirehoseDeliveryStreamExtendedS3ConfigurationCloudwatchLoggingOptions, FirehoseDeliveryStreamExtendedS3ConfigurationCloudwatchLoggingOptionsArgs
- Enabled bool
- Enables or disables the logging. Defaults to
false. - Log
Group stringName - The CloudWatch group name for logging. This value is required if
enabledis true. - Log
Stream stringName - The CloudWatch log stream name for logging. This value is required if
enabledis true.
- Enabled bool
- Enables or disables the logging. Defaults to
false. - Log
Group stringName - The CloudWatch group name for logging. This value is required if
enabledis true. - Log
Stream stringName - The CloudWatch log stream name for logging. This value is required if
enabledis true.
- enabled Boolean
- Enables or disables the logging. Defaults to
false. - log
Group StringName - The CloudWatch group name for logging. This value is required if
enabledis true. - log
Stream StringName - The CloudWatch log stream name for logging. This value is required if
enabledis true.
- enabled boolean
- Enables or disables the logging. Defaults to
false. - log
Group stringName - The CloudWatch group name for logging. This value is required if
enabledis true. - log
Stream stringName - The CloudWatch log stream name for logging. This value is required if
enabledis true.
- enabled bool
- Enables or disables the logging. Defaults to
false. - log_
group_ strname - The CloudWatch group name for logging. This value is required if
enabledis true. - log_
stream_ strname - The CloudWatch log stream name for logging. This value is required if
enabledis true.
- enabled Boolean
- Enables or disables the logging. Defaults to
false. - log
Group StringName - The CloudWatch group name for logging. This value is required if
enabledis true. - log
Stream StringName - The CloudWatch log stream name for logging. This value is required if
enabledis true.
FirehoseDeliveryStreamExtendedS3ConfigurationDataFormatConversionConfiguration, FirehoseDeliveryStreamExtendedS3ConfigurationDataFormatConversionConfigurationArgs
- Input
Format FirehoseConfiguration Delivery Stream Extended S3Configuration Data Format Conversion Configuration Input Format Configuration - Nested argument that specifies the deserializer that you want Kinesis Data Firehose to use to convert the format of your data from JSON. More details below.
- Output
Format FirehoseConfiguration Delivery Stream Extended S3Configuration Data Format Conversion Configuration Output Format Configuration - Nested argument that specifies the serializer that you want Kinesis Data Firehose to use to convert the format of your data to the Parquet or ORC format. More details below.
- Schema
Configuration FirehoseDelivery Stream Extended S3Configuration Data Format Conversion Configuration Schema Configuration - Nested argument that specifies the AWS Glue Data Catalog table that contains the column information. More details below.
- Enabled bool
- Defaults to
true. Set it tofalseif you want to disable format conversion while preserving the configuration details.
- Input
Format FirehoseConfiguration Delivery Stream Extended S3Configuration Data Format Conversion Configuration Input Format Configuration - Nested argument that specifies the deserializer that you want Kinesis Data Firehose to use to convert the format of your data from JSON. More details below.
- Output
Format FirehoseConfiguration Delivery Stream Extended S3Configuration Data Format Conversion Configuration Output Format Configuration - Nested argument that specifies the serializer that you want Kinesis Data Firehose to use to convert the format of your data to the Parquet or ORC format. More details below.
- Schema
Configuration FirehoseDelivery Stream Extended S3Configuration Data Format Conversion Configuration Schema Configuration - Nested argument that specifies the AWS Glue Data Catalog table that contains the column information. More details below.
- Enabled bool
- Defaults to
true. Set it tofalseif you want to disable format conversion while preserving the configuration details.
- input
Format FirehoseConfiguration Delivery Stream Extended S3Configuration Data Format Conversion Configuration Input Format Configuration - Nested argument that specifies the deserializer that you want Kinesis Data Firehose to use to convert the format of your data from JSON. More details below.
- output
Format FirehoseConfiguration Delivery Stream Extended S3Configuration Data Format Conversion Configuration Output Format Configuration - Nested argument that specifies the serializer that you want Kinesis Data Firehose to use to convert the format of your data to the Parquet or ORC format. More details below.
- schema
Configuration FirehoseDelivery Stream Extended S3Configuration Data Format Conversion Configuration Schema Configuration - Nested argument that specifies the AWS Glue Data Catalog table that contains the column information. More details below.
- enabled Boolean
- Defaults to
true. Set it tofalseif you want to disable format conversion while preserving the configuration details.
- input
Format FirehoseConfiguration Delivery Stream Extended S3Configuration Data Format Conversion Configuration Input Format Configuration - Nested argument that specifies the deserializer that you want Kinesis Data Firehose to use to convert the format of your data from JSON. More details below.
- output
Format FirehoseConfiguration Delivery Stream Extended S3Configuration Data Format Conversion Configuration Output Format Configuration - Nested argument that specifies the serializer that you want Kinesis Data Firehose to use to convert the format of your data to the Parquet or ORC format. More details below.
- schema
Configuration FirehoseDelivery Stream Extended S3Configuration Data Format Conversion Configuration Schema Configuration - Nested argument that specifies the AWS Glue Data Catalog table that contains the column information. More details below.
- enabled boolean
- Defaults to
true. Set it tofalseif you want to disable format conversion while preserving the configuration details.
- input_
format_ Firehoseconfiguration Delivery Stream Extended S3Configuration Data Format Conversion Configuration Input Format Configuration - Nested argument that specifies the deserializer that you want Kinesis Data Firehose to use to convert the format of your data from JSON. More details below.
- output_
format_ Firehoseconfiguration Delivery Stream Extended S3Configuration Data Format Conversion Configuration Output Format Configuration - Nested argument that specifies the serializer that you want Kinesis Data Firehose to use to convert the format of your data to the Parquet or ORC format. More details below.
- schema_
configuration FirehoseDelivery Stream Extended S3Configuration Data Format Conversion Configuration Schema Configuration - Nested argument that specifies the AWS Glue Data Catalog table that contains the column information. More details below.
- enabled bool
- Defaults to
true. Set it tofalseif you want to disable format conversion while preserving the configuration details.
- input
Format Property MapConfiguration - Nested argument that specifies the deserializer that you want Kinesis Data Firehose to use to convert the format of your data from JSON. More details below.
- output
Format Property MapConfiguration - Nested argument that specifies the serializer that you want Kinesis Data Firehose to use to convert the format of your data to the Parquet or ORC format. More details below.
- schema
Configuration Property Map - Nested argument that specifies the AWS Glue Data Catalog table that contains the column information. More details below.
- enabled Boolean
- Defaults to
true. Set it tofalseif you want to disable format conversion while preserving the configuration details.
FirehoseDeliveryStreamExtendedS3ConfigurationDataFormatConversionConfigurationInputFormatConfiguration, FirehoseDeliveryStreamExtendedS3ConfigurationDataFormatConversionConfigurationInputFormatConfigurationArgs
- Deserializer
Firehose
Delivery Stream Extended S3Configuration Data Format Conversion Configuration Input Format Configuration Deserializer - Nested argument that specifies which deserializer to use. You can choose either the Apache Hive JSON SerDe or the OpenX JSON SerDe. More details below.
- Deserializer
Firehose
Delivery Stream Extended S3Configuration Data Format Conversion Configuration Input Format Configuration Deserializer - Nested argument that specifies which deserializer to use. You can choose either the Apache Hive JSON SerDe or the OpenX JSON SerDe. More details below.
- deserializer
Firehose
Delivery Stream Extended S3Configuration Data Format Conversion Configuration Input Format Configuration Deserializer - Nested argument that specifies which deserializer to use. You can choose either the Apache Hive JSON SerDe or the OpenX JSON SerDe. More details below.
- deserializer
Firehose
Delivery Stream Extended S3Configuration Data Format Conversion Configuration Input Format Configuration Deserializer - Nested argument that specifies which deserializer to use. You can choose either the Apache Hive JSON SerDe or the OpenX JSON SerDe. More details below.
- deserializer
Firehose
Delivery Stream Extended S3Configuration Data Format Conversion Configuration Input Format Configuration Deserializer - Nested argument that specifies which deserializer to use. You can choose either the Apache Hive JSON SerDe or the OpenX JSON SerDe. More details below.
- deserializer Property Map
- Nested argument that specifies which deserializer to use. You can choose either the Apache Hive JSON SerDe or the OpenX JSON SerDe. More details below.
FirehoseDeliveryStreamExtendedS3ConfigurationDataFormatConversionConfigurationInputFormatConfigurationDeserializer, FirehoseDeliveryStreamExtendedS3ConfigurationDataFormatConversionConfigurationInputFormatConfigurationDeserializerArgs
- Hive
Json FirehoseSer De Delivery Stream Extended S3Configuration Data Format Conversion Configuration Input Format Configuration Deserializer Hive Json Ser De - Nested argument that specifies the native Hive / HCatalog JsonSerDe. More details below.
- Open
XJson FirehoseSer De Delivery Stream Extended S3Configuration Data Format Conversion Configuration Input Format Configuration Deserializer Open XJson Ser De - Nested argument that specifies the OpenX SerDe. More details below.
- Hive
Json FirehoseSer De Delivery Stream Extended S3Configuration Data Format Conversion Configuration Input Format Configuration Deserializer Hive Json Ser De - Nested argument that specifies the native Hive / HCatalog JsonSerDe. More details below.
- Open
XJson FirehoseSer De Delivery Stream Extended S3Configuration Data Format Conversion Configuration Input Format Configuration Deserializer Open XJson Ser De - Nested argument that specifies the OpenX SerDe. More details below.
- hive
Json FirehoseSer De Delivery Stream Extended S3Configuration Data Format Conversion Configuration Input Format Configuration Deserializer Hive Json Ser De - Nested argument that specifies the native Hive / HCatalog JsonSerDe. More details below.
- open
XJson FirehoseSer De Delivery Stream Extended S3Configuration Data Format Conversion Configuration Input Format Configuration Deserializer Open XJson Ser De - Nested argument that specifies the OpenX SerDe. More details below.
- hive
Json FirehoseSer De Delivery Stream Extended S3Configuration Data Format Conversion Configuration Input Format Configuration Deserializer Hive Json Ser De - Nested argument that specifies the native Hive / HCatalog JsonSerDe. More details below.
- open
XJson FirehoseSer De Delivery Stream Extended S3Configuration Data Format Conversion Configuration Input Format Configuration Deserializer Open XJson Ser De - Nested argument that specifies the OpenX SerDe. More details below.
- hive_
json_ Firehoseser_ de Delivery Stream Extended S3Configuration Data Format Conversion Configuration Input Format Configuration Deserializer Hive Json Ser De - Nested argument that specifies the native Hive / HCatalog JsonSerDe. More details below.
- open_
x_ Firehosejson_ ser_ de Delivery Stream Extended S3Configuration Data Format Conversion Configuration Input Format Configuration Deserializer Open XJson Ser De - Nested argument that specifies the OpenX SerDe. More details below.
- hive
Json Property MapSer De - Nested argument that specifies the native Hive / HCatalog JsonSerDe. More details below.
- open
XJson Property MapSer De - Nested argument that specifies the OpenX SerDe. More details below.
FirehoseDeliveryStreamExtendedS3ConfigurationDataFormatConversionConfigurationInputFormatConfigurationDeserializerHiveJsonSerDe, FirehoseDeliveryStreamExtendedS3ConfigurationDataFormatConversionConfigurationInputFormatConfigurationDeserializerHiveJsonSerDeArgs
- Timestamp
Formats List<string> - A list of how you want Kinesis Data Firehose to parse the date and time stamps that may be present in your input data JSON. To specify these format strings, follow the pattern syntax of JodaTime's DateTimeFormat format strings. For more information, see Class DateTimeFormat. You can also use the special value millis to parse time stamps in epoch milliseconds. If you don't specify a format, Kinesis Data Firehose uses java.sql.Timestamp::valueOf by default.
- Timestamp
Formats []string - A list of how you want Kinesis Data Firehose to parse the date and time stamps that may be present in your input data JSON. To specify these format strings, follow the pattern syntax of JodaTime's DateTimeFormat format strings. For more information, see Class DateTimeFormat. You can also use the special value millis to parse time stamps in epoch milliseconds. If you don't specify a format, Kinesis Data Firehose uses java.sql.Timestamp::valueOf by default.
- timestamp
Formats List<String> - A list of how you want Kinesis Data Firehose to parse the date and time stamps that may be present in your input data JSON. To specify these format strings, follow the pattern syntax of JodaTime's DateTimeFormat format strings. For more information, see Class DateTimeFormat. You can also use the special value millis to parse time stamps in epoch milliseconds. If you don't specify a format, Kinesis Data Firehose uses java.sql.Timestamp::valueOf by default.
- timestamp
Formats string[] - A list of how you want Kinesis Data Firehose to parse the date and time stamps that may be present in your input data JSON. To specify these format strings, follow the pattern syntax of JodaTime's DateTimeFormat format strings. For more information, see Class DateTimeFormat. You can also use the special value millis to parse time stamps in epoch milliseconds. If you don't specify a format, Kinesis Data Firehose uses java.sql.Timestamp::valueOf by default.
- timestamp_
formats Sequence[str] - A list of how you want Kinesis Data Firehose to parse the date and time stamps that may be present in your input data JSON. To specify these format strings, follow the pattern syntax of JodaTime's DateTimeFormat format strings. For more information, see Class DateTimeFormat. You can also use the special value millis to parse time stamps in epoch milliseconds. If you don't specify a format, Kinesis Data Firehose uses java.sql.Timestamp::valueOf by default.
- timestamp
Formats List<String> - A list of how you want Kinesis Data Firehose to parse the date and time stamps that may be present in your input data JSON. To specify these format strings, follow the pattern syntax of JodaTime's DateTimeFormat format strings. For more information, see Class DateTimeFormat. You can also use the special value millis to parse time stamps in epoch milliseconds. If you don't specify a format, Kinesis Data Firehose uses java.sql.Timestamp::valueOf by default.
FirehoseDeliveryStreamExtendedS3ConfigurationDataFormatConversionConfigurationInputFormatConfigurationDeserializerOpenXJsonSerDe, FirehoseDeliveryStreamExtendedS3ConfigurationDataFormatConversionConfigurationInputFormatConfigurationDeserializerOpenXJsonSerDeArgs
- Case
Insensitive bool - When set to true, which is the default, Kinesis Data Firehose converts JSON keys to lowercase before deserializing them.
- Column
To Dictionary<string, string>Json Key Mappings - A map of column names to JSON keys that aren't identical to the column names. This is useful when the JSON contains keys that are Hive keywords. For example, timestamp is a Hive keyword. If you have a JSON key named timestamp, set this parameter to
{ ts = "timestamp" }to map this key to a column named ts. - Convert
Dots boolIn Json Keys To Underscores - When set to
true, specifies that the names of the keys include dots and that you want Kinesis Data Firehose to replace them with underscores. This is useful because Apache Hive does not allow dots in column names. For example, if the JSON contains a key whose name is "a.b", you can define the column name to be "a_b" when using this option. Defaults tofalse.
- Case
Insensitive bool - When set to true, which is the default, Kinesis Data Firehose converts JSON keys to lowercase before deserializing them.
- Column
To map[string]stringJson Key Mappings - A map of column names to JSON keys that aren't identical to the column names. This is useful when the JSON contains keys that are Hive keywords. For example, timestamp is a Hive keyword. If you have a JSON key named timestamp, set this parameter to
{ ts = "timestamp" }to map this key to a column named ts. - Convert
Dots boolIn Json Keys To Underscores - When set to
true, specifies that the names of the keys include dots and that you want Kinesis Data Firehose to replace them with underscores. This is useful because Apache Hive does not allow dots in column names. For example, if the JSON contains a key whose name is "a.b", you can define the column name to be "a_b" when using this option. Defaults tofalse.
- case
Insensitive Boolean - When set to true, which is the default, Kinesis Data Firehose converts JSON keys to lowercase before deserializing them.
- column
To Map<String,String>Json Key Mappings - A map of column names to JSON keys that aren't identical to the column names. This is useful when the JSON contains keys that are Hive keywords. For example, timestamp is a Hive keyword. If you have a JSON key named timestamp, set this parameter to
{ ts = "timestamp" }to map this key to a column named ts. - convert
Dots BooleanIn Json Keys To Underscores - When set to
true, specifies that the names of the keys include dots and that you want Kinesis Data Firehose to replace them with underscores. This is useful because Apache Hive does not allow dots in column names. For example, if the JSON contains a key whose name is "a.b", you can define the column name to be "a_b" when using this option. Defaults tofalse.
- case
Insensitive boolean - When set to true, which is the default, Kinesis Data Firehose converts JSON keys to lowercase before deserializing them.
- column
To {[key: string]: string}Json Key Mappings - A map of column names to JSON keys that aren't identical to the column names. This is useful when the JSON contains keys that are Hive keywords. For example, timestamp is a Hive keyword. If you have a JSON key named timestamp, set this parameter to
{ ts = "timestamp" }to map this key to a column named ts. - convert
Dots booleanIn Json Keys To Underscores - When set to
true, specifies that the names of the keys include dots and that you want Kinesis Data Firehose to replace them with underscores. This is useful because Apache Hive does not allow dots in column names. For example, if the JSON contains a key whose name is "a.b", you can define the column name to be "a_b" when using this option. Defaults tofalse.
- case_
insensitive bool - When set to true, which is the default, Kinesis Data Firehose converts JSON keys to lowercase before deserializing them.
- column_
to_ Mapping[str, str]json_ key_ mappings - A map of column names to JSON keys that aren't identical to the column names. This is useful when the JSON contains keys that are Hive keywords. For example, timestamp is a Hive keyword. If you have a JSON key named timestamp, set this parameter to
{ ts = "timestamp" }to map this key to a column named ts. - convert_
dots_ boolin_ json_ keys_ to_ underscores - When set to
true, specifies that the names of the keys include dots and that you want Kinesis Data Firehose to replace them with underscores. This is useful because Apache Hive does not allow dots in column names. For example, if the JSON contains a key whose name is "a.b", you can define the column name to be "a_b" when using this option. Defaults tofalse.
- case
Insensitive Boolean - When set to true, which is the default, Kinesis Data Firehose converts JSON keys to lowercase before deserializing them.
- column
To Map<String>Json Key Mappings - A map of column names to JSON keys that aren't identical to the column names. This is useful when the JSON contains keys that are Hive keywords. For example, timestamp is a Hive keyword. If you have a JSON key named timestamp, set this parameter to
{ ts = "timestamp" }to map this key to a column named ts. - convert
Dots BooleanIn Json Keys To Underscores - When set to
true, specifies that the names of the keys include dots and that you want Kinesis Data Firehose to replace them with underscores. This is useful because Apache Hive does not allow dots in column names. For example, if the JSON contains a key whose name is "a.b", you can define the column name to be "a_b" when using this option. Defaults tofalse.
FirehoseDeliveryStreamExtendedS3ConfigurationDataFormatConversionConfigurationOutputFormatConfiguration, FirehoseDeliveryStreamExtendedS3ConfigurationDataFormatConversionConfigurationOutputFormatConfigurationArgs
- Serializer
Firehose
Delivery Stream Extended S3Configuration Data Format Conversion Configuration Output Format Configuration Serializer - Nested argument that specifies which serializer to use. You can choose either the ORC SerDe or the Parquet SerDe. More details below.
- Serializer
Firehose
Delivery Stream Extended S3Configuration Data Format Conversion Configuration Output Format Configuration Serializer - Nested argument that specifies which serializer to use. You can choose either the ORC SerDe or the Parquet SerDe. More details below.
- serializer
Firehose
Delivery Stream Extended S3Configuration Data Format Conversion Configuration Output Format Configuration Serializer - Nested argument that specifies which serializer to use. You can choose either the ORC SerDe or the Parquet SerDe. More details below.
- serializer
Firehose
Delivery Stream Extended S3Configuration Data Format Conversion Configuration Output Format Configuration Serializer - Nested argument that specifies which serializer to use. You can choose either the ORC SerDe or the Parquet SerDe. More details below.
- serializer
Firehose
Delivery Stream Extended S3Configuration Data Format Conversion Configuration Output Format Configuration Serializer - Nested argument that specifies which serializer to use. You can choose either the ORC SerDe or the Parquet SerDe. More details below.
- serializer Property Map
- Nested argument that specifies which serializer to use. You can choose either the ORC SerDe or the Parquet SerDe. More details below.
FirehoseDeliveryStreamExtendedS3ConfigurationDataFormatConversionConfigurationOutputFormatConfigurationSerializer, FirehoseDeliveryStreamExtendedS3ConfigurationDataFormatConversionConfigurationOutputFormatConfigurationSerializerArgs
- Orc
Ser FirehoseDe Delivery Stream Extended S3Configuration Data Format Conversion Configuration Output Format Configuration Serializer Orc Ser De - Nested argument that specifies converting data to the ORC format before storing it in Amazon S3. For more information, see Apache ORC. More details below.
- Parquet
Ser FirehoseDe Delivery Stream Extended S3Configuration Data Format Conversion Configuration Output Format Configuration Serializer Parquet Ser De - Nested argument that specifies converting data to the Parquet format before storing it in Amazon S3. For more information, see Apache Parquet. More details below.
- Orc
Ser FirehoseDe Delivery Stream Extended S3Configuration Data Format Conversion Configuration Output Format Configuration Serializer Orc Ser De - Nested argument that specifies converting data to the ORC format before storing it in Amazon S3. For more information, see Apache ORC. More details below.
- Parquet
Ser FirehoseDe Delivery Stream Extended S3Configuration Data Format Conversion Configuration Output Format Configuration Serializer Parquet Ser De - Nested argument that specifies converting data to the Parquet format before storing it in Amazon S3. For more information, see Apache Parquet. More details below.
- orc
Ser FirehoseDe Delivery Stream Extended S3Configuration Data Format Conversion Configuration Output Format Configuration Serializer Orc Ser De - Nested argument that specifies converting data to the ORC format before storing it in Amazon S3. For more information, see Apache ORC. More details below.
- parquet
Ser FirehoseDe Delivery Stream Extended S3Configuration Data Format Conversion Configuration Output Format Configuration Serializer Parquet Ser De - Nested argument that specifies converting data to the Parquet format before storing it in Amazon S3. For more information, see Apache Parquet. More details below.
- orc
Ser FirehoseDe Delivery Stream Extended S3Configuration Data Format Conversion Configuration Output Format Configuration Serializer Orc Ser De - Nested argument that specifies converting data to the ORC format before storing it in Amazon S3. For more information, see Apache ORC. More details below.
- parquet
Ser FirehoseDe Delivery Stream Extended S3Configuration Data Format Conversion Configuration Output Format Configuration Serializer Parquet Ser De - Nested argument that specifies converting data to the Parquet format before storing it in Amazon S3. For more information, see Apache Parquet. More details below.
- orc_
ser_ Firehosede Delivery Stream Extended S3Configuration Data Format Conversion Configuration Output Format Configuration Serializer Orc Ser De - Nested argument that specifies converting data to the ORC format before storing it in Amazon S3. For more information, see Apache ORC. More details below.
- parquet_
ser_ Firehosede Delivery Stream Extended S3Configuration Data Format Conversion Configuration Output Format Configuration Serializer Parquet Ser De - Nested argument that specifies converting data to the Parquet format before storing it in Amazon S3. For more information, see Apache Parquet. More details below.
- orc
Ser Property MapDe - Nested argument that specifies converting data to the ORC format before storing it in Amazon S3. For more information, see Apache ORC. More details below.
- parquet
Ser Property MapDe - Nested argument that specifies converting data to the Parquet format before storing it in Amazon S3. For more information, see Apache Parquet. More details below.
FirehoseDeliveryStreamExtendedS3ConfigurationDataFormatConversionConfigurationOutputFormatConfigurationSerializerOrcSerDe, FirehoseDeliveryStreamExtendedS3ConfigurationDataFormatConversionConfigurationOutputFormatConfigurationSerializerOrcSerDeArgs
- Block
Size intBytes - The Hadoop Distributed File System (HDFS) block size. This is useful if you intend to copy the data from Amazon S3 to HDFS before querying. The default is 256 MiB and the minimum is 64 MiB. Kinesis Data Firehose uses this value for padding calculations.
- Bloom
Filter List<string>Columns - A list of column names for which you want Kinesis Data Firehose to create bloom filters.
- Bloom
Filter doubleFalse Positive Probability - The Bloom filter false positive probability (FPP). The lower the FPP, the bigger the Bloom filter. The default value is
0.05, the minimum is0, and the maximum is1. - Compression string
- The compression code to use over data blocks. The default is
SNAPPY. - Dictionary
Key doubleThreshold - A float that represents the fraction of the total number of non-null rows. To turn off dictionary encoding, set this fraction to a number that is less than the number of distinct keys in a dictionary. To always use dictionary encoding, set this threshold to
1. - Enable
Padding bool - Set this to
trueto indicate that you want stripes to be padded to the HDFS block boundaries. This is useful if you intend to copy the data from Amazon S3 to HDFS before querying. The default isfalse. - Format
Version string - The version of the file to write. The possible values are
V0_11andV0_12. The default isV0_12. - Padding
Tolerance double - A float between 0 and 1 that defines the tolerance for block padding as a decimal fraction of stripe size. The default value is
0.05, which means 5 percent of stripe size. For the default values of 64 MiB ORC stripes and 256 MiB HDFS blocks, the default block padding tolerance of 5 percent reserves a maximum of 3.2 MiB for padding within the 256 MiB block. In such a case, if the available size within the block is more than 3.2 MiB, a new, smaller stripe is inserted to fit within that space. This ensures that no stripe crosses block boundaries and causes remote reads within a node-local task. Kinesis Data Firehose ignores this parameter whenenable_paddingisfalse. - Row
Index intStride - The number of rows between index entries. The default is
10000and the minimum is1000. - Stripe
Size intBytes - The number of bytes in each stripe. The default is 64 MiB and the minimum is 8 MiB.
- Block
Size intBytes - The Hadoop Distributed File System (HDFS) block size. This is useful if you intend to copy the data from Amazon S3 to HDFS before querying. The default is 256 MiB and the minimum is 64 MiB. Kinesis Data Firehose uses this value for padding calculations.
- Bloom
Filter []stringColumns - A list of column names for which you want Kinesis Data Firehose to create bloom filters.
- Bloom
Filter float64False Positive Probability - The Bloom filter false positive probability (FPP). The lower the FPP, the bigger the Bloom filter. The default value is
0.05, the minimum is0, and the maximum is1. - Compression string
- The compression code to use over data blocks. The default is
SNAPPY. - Dictionary
Key float64Threshold - A float that represents the fraction of the total number of non-null rows. To turn off dictionary encoding, set this fraction to a number that is less than the number of distinct keys in a dictionary. To always use dictionary encoding, set this threshold to
1. - Enable
Padding bool - Set this to
trueto indicate that you want stripes to be padded to the HDFS block boundaries. This is useful if you intend to copy the data from Amazon S3 to HDFS before querying. The default isfalse. - Format
Version string - The version of the file to write. The possible values are
V0_11andV0_12. The default isV0_12. - Padding
Tolerance float64 - A float between 0 and 1 that defines the tolerance for block padding as a decimal fraction of stripe size. The default value is
0.05, which means 5 percent of stripe size. For the default values of 64 MiB ORC stripes and 256 MiB HDFS blocks, the default block padding tolerance of 5 percent reserves a maximum of 3.2 MiB for padding within the 256 MiB block. In such a case, if the available size within the block is more than 3.2 MiB, a new, smaller stripe is inserted to fit within that space. This ensures that no stripe crosses block boundaries and causes remote reads within a node-local task. Kinesis Data Firehose ignores this parameter whenenable_paddingisfalse. - Row
Index intStride - The number of rows between index entries. The default is
10000and the minimum is1000. - Stripe
Size intBytes - The number of bytes in each stripe. The default is 64 MiB and the minimum is 8 MiB.
- block
Size IntegerBytes - The Hadoop Distributed File System (HDFS) block size. This is useful if you intend to copy the data from Amazon S3 to HDFS before querying. The default is 256 MiB and the minimum is 64 MiB. Kinesis Data Firehose uses this value for padding calculations.
- bloom
Filter List<String>Columns - A list of column names for which you want Kinesis Data Firehose to create bloom filters.
- bloom
Filter DoubleFalse Positive Probability - The Bloom filter false positive probability (FPP). The lower the FPP, the bigger the Bloom filter. The default value is
0.05, the minimum is0, and the maximum is1. - compression String
- The compression code to use over data blocks. The default is
SNAPPY. - dictionary
Key DoubleThreshold - A float that represents the fraction of the total number of non-null rows. To turn off dictionary encoding, set this fraction to a number that is less than the number of distinct keys in a dictionary. To always use dictionary encoding, set this threshold to
1. - enable
Padding Boolean - Set this to
trueto indicate that you want stripes to be padded to the HDFS block boundaries. This is useful if you intend to copy the data from Amazon S3 to HDFS before querying. The default isfalse. - format
Version String - The version of the file to write. The possible values are
V0_11andV0_12. The default isV0_12. - padding
Tolerance Double - A float between 0 and 1 that defines the tolerance for block padding as a decimal fraction of stripe size. The default value is
0.05, which means 5 percent of stripe size. For the default values of 64 MiB ORC stripes and 256 MiB HDFS blocks, the default block padding tolerance of 5 percent reserves a maximum of 3.2 MiB for padding within the 256 MiB block. In such a case, if the available size within the block is more than 3.2 MiB, a new, smaller stripe is inserted to fit within that space. This ensures that no stripe crosses block boundaries and causes remote reads within a node-local task. Kinesis Data Firehose ignores this parameter whenenable_paddingisfalse. - row
Index IntegerStride - The number of rows between index entries. The default is
10000and the minimum is1000. - stripe
Size IntegerBytes - The number of bytes in each stripe. The default is 64 MiB and the minimum is 8 MiB.
- block
Size numberBytes - The Hadoop Distributed File System (HDFS) block size. This is useful if you intend to copy the data from Amazon S3 to HDFS before querying. The default is 256 MiB and the minimum is 64 MiB. Kinesis Data Firehose uses this value for padding calculations.
- bloom
Filter string[]Columns - A list of column names for which you want Kinesis Data Firehose to create bloom filters.
- bloom
Filter numberFalse Positive Probability - The Bloom filter false positive probability (FPP). The lower the FPP, the bigger the Bloom filter. The default value is
0.05, the minimum is0, and the maximum is1. - compression string
- The compression code to use over data blocks. The default is
SNAPPY. - dictionary
Key numberThreshold - A float that represents the fraction of the total number of non-null rows. To turn off dictionary encoding, set this fraction to a number that is less than the number of distinct keys in a dictionary. To always use dictionary encoding, set this threshold to
1. - enable
Padding boolean - Set this to
trueto indicate that you want stripes to be padded to the HDFS block boundaries. This is useful if you intend to copy the data from Amazon S3 to HDFS before querying. The default isfalse. - format
Version string - The version of the file to write. The possible values are
V0_11andV0_12. The default isV0_12. - padding
Tolerance number - A float between 0 and 1 that defines the tolerance for block padding as a decimal fraction of stripe size. The default value is
0.05, which means 5 percent of stripe size. For the default values of 64 MiB ORC stripes and 256 MiB HDFS blocks, the default block padding tolerance of 5 percent reserves a maximum of 3.2 MiB for padding within the 256 MiB block. In such a case, if the available size within the block is more than 3.2 MiB, a new, smaller stripe is inserted to fit within that space. This ensures that no stripe crosses block boundaries and causes remote reads within a node-local task. Kinesis Data Firehose ignores this parameter whenenable_paddingisfalse. - row
Index numberStride - The number of rows between index entries. The default is
10000and the minimum is1000. - stripe
Size numberBytes - The number of bytes in each stripe. The default is 64 MiB and the minimum is 8 MiB.
- block_
size_ intbytes - The Hadoop Distributed File System (HDFS) block size. This is useful if you intend to copy the data from Amazon S3 to HDFS before querying. The default is 256 MiB and the minimum is 64 MiB. Kinesis Data Firehose uses this value for padding calculations.
- bloom_
filter_ Sequence[str]columns - A list of column names for which you want Kinesis Data Firehose to create bloom filters.
- bloom_
filter_ floatfalse_ positive_ probability - The Bloom filter false positive probability (FPP). The lower the FPP, the bigger the Bloom filter. The default value is
0.05, the minimum is0, and the maximum is1. - compression str
- The compression code to use over data blocks. The default is
SNAPPY. - dictionary_
key_ floatthreshold - A float that represents the fraction of the total number of non-null rows. To turn off dictionary encoding, set this fraction to a number that is less than the number of distinct keys in a dictionary. To always use dictionary encoding, set this threshold to
1. - enable_
padding bool - Set this to
trueto indicate that you want stripes to be padded to the HDFS block boundaries. This is useful if you intend to copy the data from Amazon S3 to HDFS before querying. The default isfalse. - format_
version str - The version of the file to write. The possible values are
V0_11andV0_12. The default isV0_12. - padding_
tolerance float - A float between 0 and 1 that defines the tolerance for block padding as a decimal fraction of stripe size. The default value is
0.05, which means 5 percent of stripe size. For the default values of 64 MiB ORC stripes and 256 MiB HDFS blocks, the default block padding tolerance of 5 percent reserves a maximum of 3.2 MiB for padding within the 256 MiB block. In such a case, if the available size within the block is more than 3.2 MiB, a new, smaller stripe is inserted to fit within that space. This ensures that no stripe crosses block boundaries and causes remote reads within a node-local task. Kinesis Data Firehose ignores this parameter whenenable_paddingisfalse. - row_
index_ intstride - The number of rows between index entries. The default is
10000and the minimum is1000. - stripe_
size_ intbytes - The number of bytes in each stripe. The default is 64 MiB and the minimum is 8 MiB.
- block
Size NumberBytes - The Hadoop Distributed File System (HDFS) block size. This is useful if you intend to copy the data from Amazon S3 to HDFS before querying. The default is 256 MiB and the minimum is 64 MiB. Kinesis Data Firehose uses this value for padding calculations.
- bloom
Filter List<String>Columns - A list of column names for which you want Kinesis Data Firehose to create bloom filters.
- bloom
Filter NumberFalse Positive Probability - The Bloom filter false positive probability (FPP). The lower the FPP, the bigger the Bloom filter. The default value is
0.05, the minimum is0, and the maximum is1. - compression String
- The compression code to use over data blocks. The default is
SNAPPY. - dictionary
Key NumberThreshold - A float that represents the fraction of the total number of non-null rows. To turn off dictionary encoding, set this fraction to a number that is less than the number of distinct keys in a dictionary. To always use dictionary encoding, set this threshold to
1. - enable
Padding Boolean - Set this to
trueto indicate that you want stripes to be padded to the HDFS block boundaries. This is useful if you intend to copy the data from Amazon S3 to HDFS before querying. The default isfalse. - format
Version String - The version of the file to write. The possible values are
V0_11andV0_12. The default isV0_12. - padding
Tolerance Number - A float between 0 and 1 that defines the tolerance for block padding as a decimal fraction of stripe size. The default value is
0.05, which means 5 percent of stripe size. For the default values of 64 MiB ORC stripes and 256 MiB HDFS blocks, the default block padding tolerance of 5 percent reserves a maximum of 3.2 MiB for padding within the 256 MiB block. In such a case, if the available size within the block is more than 3.2 MiB, a new, smaller stripe is inserted to fit within that space. This ensures that no stripe crosses block boundaries and causes remote reads within a node-local task. Kinesis Data Firehose ignores this parameter whenenable_paddingisfalse. - row
Index NumberStride - The number of rows between index entries. The default is
10000and the minimum is1000. - stripe
Size NumberBytes - The number of bytes in each stripe. The default is 64 MiB and the minimum is 8 MiB.
FirehoseDeliveryStreamExtendedS3ConfigurationDataFormatConversionConfigurationOutputFormatConfigurationSerializerParquetSerDe, FirehoseDeliveryStreamExtendedS3ConfigurationDataFormatConversionConfigurationOutputFormatConfigurationSerializerParquetSerDeArgs
- Block
Size intBytes - The Hadoop Distributed File System (HDFS) block size. This is useful if you intend to copy the data from Amazon S3 to HDFS before querying. The default is 256 MiB and the minimum is 64 MiB. Kinesis Data Firehose uses this value for padding calculations.
- Compression string
- The compression code to use over data blocks. The possible values are
UNCOMPRESSED,SNAPPY, andGZIP, with the default beingSNAPPY. UseSNAPPYfor higher decompression speed. UseGZIPif the compression ratio is more important than speed. - Enable
Dictionary boolCompression - Indicates whether to enable dictionary compression.
- Max
Padding intBytes - The maximum amount of padding to apply. This is useful if you intend to copy the data from Amazon S3 to HDFS before querying. The default is
0. - Page
Size intBytes - The Parquet page size. Column chunks are divided into pages. A page is conceptually an indivisible unit (in terms of compression and encoding). The minimum value is 64 KiB and the default is 1 MiB.
- Writer
Version string - Indicates the version of row format to output. The possible values are
V1andV2. The default isV1.
- Block
Size intBytes - The Hadoop Distributed File System (HDFS) block size. This is useful if you intend to copy the data from Amazon S3 to HDFS before querying. The default is 256 MiB and the minimum is 64 MiB. Kinesis Data Firehose uses this value for padding calculations.
- Compression string
- The compression code to use over data blocks. The possible values are
UNCOMPRESSED,SNAPPY, andGZIP, with the default beingSNAPPY. UseSNAPPYfor higher decompression speed. UseGZIPif the compression ratio is more important than speed. - Enable
Dictionary boolCompression - Indicates whether to enable dictionary compression.
- Max
Padding intBytes - The maximum amount of padding to apply. This is useful if you intend to copy the data from Amazon S3 to HDFS before querying. The default is
0. - Page
Size intBytes - The Parquet page size. Column chunks are divided into pages. A page is conceptually an indivisible unit (in terms of compression and encoding). The minimum value is 64 KiB and the default is 1 MiB.
- Writer
Version string - Indicates the version of row format to output. The possible values are
V1andV2. The default isV1.
- block
Size IntegerBytes - The Hadoop Distributed File System (HDFS) block size. This is useful if you intend to copy the data from Amazon S3 to HDFS before querying. The default is 256 MiB and the minimum is 64 MiB. Kinesis Data Firehose uses this value for padding calculations.
- compression String
- The compression code to use over data blocks. The possible values are
UNCOMPRESSED,SNAPPY, andGZIP, with the default beingSNAPPY. UseSNAPPYfor higher decompression speed. UseGZIPif the compression ratio is more important than speed. - enable
Dictionary BooleanCompression - Indicates whether to enable dictionary compression.
- max
Padding IntegerBytes - The maximum amount of padding to apply. This is useful if you intend to copy the data from Amazon S3 to HDFS before querying. The default is
0. - page
Size IntegerBytes - The Parquet page size. Column chunks are divided into pages. A page is conceptually an indivisible unit (in terms of compression and encoding). The minimum value is 64 KiB and the default is 1 MiB.
- writer
Version String - Indicates the version of row format to output. The possible values are
V1andV2. The default isV1.
- block
Size numberBytes - The Hadoop Distributed File System (HDFS) block size. This is useful if you intend to copy the data from Amazon S3 to HDFS before querying. The default is 256 MiB and the minimum is 64 MiB. Kinesis Data Firehose uses this value for padding calculations.
- compression string
- The compression code to use over data blocks. The possible values are
UNCOMPRESSED,SNAPPY, andGZIP, with the default beingSNAPPY. UseSNAPPYfor higher decompression speed. UseGZIPif the compression ratio is more important than speed. - enable
Dictionary booleanCompression - Indicates whether to enable dictionary compression.
- max
Padding numberBytes - The maximum amount of padding to apply. This is useful if you intend to copy the data from Amazon S3 to HDFS before querying. The default is
0. - page
Size numberBytes - The Parquet page size. Column chunks are divided into pages. A page is conceptually an indivisible unit (in terms of compression and encoding). The minimum value is 64 KiB and the default is 1 MiB.
- writer
Version string - Indicates the version of row format to output. The possible values are
V1andV2. The default isV1.
- block_
size_ intbytes - The Hadoop Distributed File System (HDFS) block size. This is useful if you intend to copy the data from Amazon S3 to HDFS before querying. The default is 256 MiB and the minimum is 64 MiB. Kinesis Data Firehose uses this value for padding calculations.
- compression str
- The compression code to use over data blocks. The possible values are
UNCOMPRESSED,SNAPPY, andGZIP, with the default beingSNAPPY. UseSNAPPYfor higher decompression speed. UseGZIPif the compression ratio is more important than speed. - enable_
dictionary_ boolcompression - Indicates whether to enable dictionary compression.
- max_
padding_ intbytes - The maximum amount of padding to apply. This is useful if you intend to copy the data from Amazon S3 to HDFS before querying. The default is
0. - page_
size_ intbytes - The Parquet page size. Column chunks are divided into pages. A page is conceptually an indivisible unit (in terms of compression and encoding). The minimum value is 64 KiB and the default is 1 MiB.
- writer_
version str - Indicates the version of row format to output. The possible values are
V1andV2. The default isV1.
- block
Size NumberBytes - The Hadoop Distributed File System (HDFS) block size. This is useful if you intend to copy the data from Amazon S3 to HDFS before querying. The default is 256 MiB and the minimum is 64 MiB. Kinesis Data Firehose uses this value for padding calculations.
- compression String
- The compression code to use over data blocks. The possible values are
UNCOMPRESSED,SNAPPY, andGZIP, with the default beingSNAPPY. UseSNAPPYfor higher decompression speed. UseGZIPif the compression ratio is more important than speed. - enable
Dictionary BooleanCompression - Indicates whether to enable dictionary compression.
- max
Padding NumberBytes - The maximum amount of padding to apply. This is useful if you intend to copy the data from Amazon S3 to HDFS before querying. The default is
0. - page
Size NumberBytes - The Parquet page size. Column chunks are divided into pages. A page is conceptually an indivisible unit (in terms of compression and encoding). The minimum value is 64 KiB and the default is 1 MiB.
- writer
Version String - Indicates the version of row format to output. The possible values are
V1andV2. The default isV1.
FirehoseDeliveryStreamExtendedS3ConfigurationDataFormatConversionConfigurationSchemaConfiguration, FirehoseDeliveryStreamExtendedS3ConfigurationDataFormatConversionConfigurationSchemaConfigurationArgs
- Database
Name string - Specifies the name of the AWS Glue database that contains the schema for the output data.
- Role
Arn string - The role that Kinesis Data Firehose can use to access AWS Glue. This role must be in the same account you use for Kinesis Data Firehose. Cross-account roles aren't allowed.
- Table
Name string - Specifies the AWS Glue table that contains the column information that constitutes your data schema.
- Catalog
Id string - The ID of the AWS Glue Data Catalog. If you don't supply this, the AWS account ID is used by default.
- Region string
- If you don't specify an AWS Region, the default is the current region.
- Version
Id string - Specifies the table version for the output data schema. Defaults to
LATEST.
- Database
Name string - Specifies the name of the AWS Glue database that contains the schema for the output data.
- Role
Arn string - The role that Kinesis Data Firehose can use to access AWS Glue. This role must be in the same account you use for Kinesis Data Firehose. Cross-account roles aren't allowed.
- Table
Name string - Specifies the AWS Glue table that contains the column information that constitutes your data schema.
- Catalog
Id string - The ID of the AWS Glue Data Catalog. If you don't supply this, the AWS account ID is used by default.
- Region string
- If you don't specify an AWS Region, the default is the current region.
- Version
Id string - Specifies the table version for the output data schema. Defaults to
LATEST.
- database
Name String - Specifies the name of the AWS Glue database that contains the schema for the output data.
- role
Arn String - The role that Kinesis Data Firehose can use to access AWS Glue. This role must be in the same account you use for Kinesis Data Firehose. Cross-account roles aren't allowed.
- table
Name String - Specifies the AWS Glue table that contains the column information that constitutes your data schema.
- catalog
Id String - The ID of the AWS Glue Data Catalog. If you don't supply this, the AWS account ID is used by default.
- region String
- If you don't specify an AWS Region, the default is the current region.
- version
Id String - Specifies the table version for the output data schema. Defaults to
LATEST.
- database
Name string - Specifies the name of the AWS Glue database that contains the schema for the output data.
- role
Arn string - The role that Kinesis Data Firehose can use to access AWS Glue. This role must be in the same account you use for Kinesis Data Firehose. Cross-account roles aren't allowed.
- table
Name string - Specifies the AWS Glue table that contains the column information that constitutes your data schema.
- catalog
Id string - The ID of the AWS Glue Data Catalog. If you don't supply this, the AWS account ID is used by default.
- region string
- If you don't specify an AWS Region, the default is the current region.
- version
Id string - Specifies the table version for the output data schema. Defaults to
LATEST.
- database_
name str - Specifies the name of the AWS Glue database that contains the schema for the output data.
- role_
arn str - The role that Kinesis Data Firehose can use to access AWS Glue. This role must be in the same account you use for Kinesis Data Firehose. Cross-account roles aren't allowed.
- table_
name str - Specifies the AWS Glue table that contains the column information that constitutes your data schema.
- catalog_
id str - The ID of the AWS Glue Data Catalog. If you don't supply this, the AWS account ID is used by default.
- region str
- If you don't specify an AWS Region, the default is the current region.
- version_
id str - Specifies the table version for the output data schema. Defaults to
LATEST.
- database
Name String - Specifies the name of the AWS Glue database that contains the schema for the output data.
- role
Arn String - The role that Kinesis Data Firehose can use to access AWS Glue. This role must be in the same account you use for Kinesis Data Firehose. Cross-account roles aren't allowed.
- table
Name String - Specifies the AWS Glue table that contains the column information that constitutes your data schema.
- catalog
Id String - The ID of the AWS Glue Data Catalog. If you don't supply this, the AWS account ID is used by default.
- region String
- If you don't specify an AWS Region, the default is the current region.
- version
Id String - Specifies the table version for the output data schema. Defaults to
LATEST.
FirehoseDeliveryStreamExtendedS3ConfigurationDynamicPartitioningConfiguration, FirehoseDeliveryStreamExtendedS3ConfigurationDynamicPartitioningConfigurationArgs
- Enabled bool
- Enables or disables dynamic partitioning. Defaults to
false. - Retry
Duration int Total amount of seconds Firehose spends on retries. Valid values between 0 and 7200. Default is 300.
NOTE: You can enable dynamic partitioning only when you create a new delivery stream. Once you enable dynamic partitioning on a delivery stream, it cannot be disabled on this delivery stream. Therefore, the provider will recreate the resource whenever dynamic partitioning is enabled or disabled.
- Enabled bool
- Enables or disables dynamic partitioning. Defaults to
false. - Retry
Duration int Total amount of seconds Firehose spends on retries. Valid values between 0 and 7200. Default is 300.
NOTE: You can enable dynamic partitioning only when you create a new delivery stream. Once you enable dynamic partitioning on a delivery stream, it cannot be disabled on this delivery stream. Therefore, the provider will recreate the resource whenever dynamic partitioning is enabled or disabled.
- enabled Boolean
- Enables or disables dynamic partitioning. Defaults to
false. - retry
Duration Integer Total amount of seconds Firehose spends on retries. Valid values between 0 and 7200. Default is 300.
NOTE: You can enable dynamic partitioning only when you create a new delivery stream. Once you enable dynamic partitioning on a delivery stream, it cannot be disabled on this delivery stream. Therefore, the provider will recreate the resource whenever dynamic partitioning is enabled or disabled.
- enabled boolean
- Enables or disables dynamic partitioning. Defaults to
false. - retry
Duration number Total amount of seconds Firehose spends on retries. Valid values between 0 and 7200. Default is 300.
NOTE: You can enable dynamic partitioning only when you create a new delivery stream. Once you enable dynamic partitioning on a delivery stream, it cannot be disabled on this delivery stream. Therefore, the provider will recreate the resource whenever dynamic partitioning is enabled or disabled.
- enabled bool
- Enables or disables dynamic partitioning. Defaults to
false. - retry_
duration int Total amount of seconds Firehose spends on retries. Valid values between 0 and 7200. Default is 300.
NOTE: You can enable dynamic partitioning only when you create a new delivery stream. Once you enable dynamic partitioning on a delivery stream, it cannot be disabled on this delivery stream. Therefore, the provider will recreate the resource whenever dynamic partitioning is enabled or disabled.
- enabled Boolean
- Enables or disables dynamic partitioning. Defaults to
false. - retry
Duration Number Total amount of seconds Firehose spends on retries. Valid values between 0 and 7200. Default is 300.
NOTE: You can enable dynamic partitioning only when you create a new delivery stream. Once you enable dynamic partitioning on a delivery stream, it cannot be disabled on this delivery stream. Therefore, the provider will recreate the resource whenever dynamic partitioning is enabled or disabled.
FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfiguration, FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationArgs
- Enabled bool
- Enables or disables data processing.
- Processors
List<Firehose
Delivery Stream Extended S3Configuration Processing Configuration Processor> - Array of data processors. More details are given below
- Enabled bool
- Enables or disables data processing.
- Processors
[]Firehose
Delivery Stream Extended S3Configuration Processing Configuration Processor - Array of data processors. More details are given below
- enabled Boolean
- Enables or disables data processing.
- processors
List<Firehose
Delivery Stream Extended S3Configuration Processing Configuration Processor> - Array of data processors. More details are given below
- enabled boolean
- Enables or disables data processing.
- processors
Firehose
Delivery Stream Extended S3Configuration Processing Configuration Processor[] - Array of data processors. More details are given below
- enabled bool
- Enables or disables data processing.
- processors
Sequence[Firehose
Delivery Stream Extended S3Configuration Processing Configuration Processor] - Array of data processors. More details are given below
- enabled Boolean
- Enables or disables data processing.
- processors List<Property Map>
- Array of data processors. More details are given below
FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationProcessor, FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationProcessorArgs
- Type string
- The type of processor. Valid Values:
RecordDeAggregation,Lambda,MetadataExtraction,AppendDelimiterToRecord. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - Parameters
List<Firehose
Delivery Stream Extended S3Configuration Processing Configuration Processor Parameter> - Array of processor parameters. More details are given below
- Type string
- The type of processor. Valid Values:
RecordDeAggregation,Lambda,MetadataExtraction,AppendDelimiterToRecord. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - Parameters
[]Firehose
Delivery Stream Extended S3Configuration Processing Configuration Processor Parameter - Array of processor parameters. More details are given below
- type String
- The type of processor. Valid Values:
RecordDeAggregation,Lambda,MetadataExtraction,AppendDelimiterToRecord. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - parameters
List<Firehose
Delivery Stream Extended S3Configuration Processing Configuration Processor Parameter> - Array of processor parameters. More details are given below
- type string
- The type of processor. Valid Values:
RecordDeAggregation,Lambda,MetadataExtraction,AppendDelimiterToRecord. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - parameters
Firehose
Delivery Stream Extended S3Configuration Processing Configuration Processor Parameter[] - Array of processor parameters. More details are given below
- type str
- The type of processor. Valid Values:
RecordDeAggregation,Lambda,MetadataExtraction,AppendDelimiterToRecord. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - parameters
Sequence[Firehose
Delivery Stream Extended S3Configuration Processing Configuration Processor Parameter] - Array of processor parameters. More details are given below
- type String
- The type of processor. Valid Values:
RecordDeAggregation,Lambda,MetadataExtraction,AppendDelimiterToRecord. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - parameters List<Property Map>
- Array of processor parameters. More details are given below
FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationProcessorParameter, FirehoseDeliveryStreamExtendedS3ConfigurationProcessingConfigurationProcessorParameterArgs
- Parameter
Name string - Parameter name. Valid Values:
LambdaArn,NumberOfRetries,MetadataExtractionQuery,JsonParsingEngine,RoleArn,BufferSizeInMBs,BufferIntervalInSeconds,SubRecordType,Delimiter. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - Parameter
Value string Parameter value. Must be between 1 and 512 length (inclusive). When providing a Lambda ARN, you should specify the resource version as well.
NOTE: Parameters with default values, including
NumberOfRetries(default: 3),RoleArn(default: firehose role ARN),BufferSizeInMBs(default: 3), andBufferIntervalInSeconds(default: 60), are not stored in state. To prevent perpetual differences, it is therefore recommended to only include parameters with non-default values.
- Parameter
Name string - Parameter name. Valid Values:
LambdaArn,NumberOfRetries,MetadataExtractionQuery,JsonParsingEngine,RoleArn,BufferSizeInMBs,BufferIntervalInSeconds,SubRecordType,Delimiter. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - Parameter
Value string Parameter value. Must be between 1 and 512 length (inclusive). When providing a Lambda ARN, you should specify the resource version as well.
NOTE: Parameters with default values, including
NumberOfRetries(default: 3),RoleArn(default: firehose role ARN),BufferSizeInMBs(default: 3), andBufferIntervalInSeconds(default: 60), are not stored in state. To prevent perpetual differences, it is therefore recommended to only include parameters with non-default values.
- parameter
Name String - Parameter name. Valid Values:
LambdaArn,NumberOfRetries,MetadataExtractionQuery,JsonParsingEngine,RoleArn,BufferSizeInMBs,BufferIntervalInSeconds,SubRecordType,Delimiter. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - parameter
Value String Parameter value. Must be between 1 and 512 length (inclusive). When providing a Lambda ARN, you should specify the resource version as well.
NOTE: Parameters with default values, including
NumberOfRetries(default: 3),RoleArn(default: firehose role ARN),BufferSizeInMBs(default: 3), andBufferIntervalInSeconds(default: 60), are not stored in state. To prevent perpetual differences, it is therefore recommended to only include parameters with non-default values.
- parameter
Name string - Parameter name. Valid Values:
LambdaArn,NumberOfRetries,MetadataExtractionQuery,JsonParsingEngine,RoleArn,BufferSizeInMBs,BufferIntervalInSeconds,SubRecordType,Delimiter. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - parameter
Value string Parameter value. Must be between 1 and 512 length (inclusive). When providing a Lambda ARN, you should specify the resource version as well.
NOTE: Parameters with default values, including
NumberOfRetries(default: 3),RoleArn(default: firehose role ARN),BufferSizeInMBs(default: 3), andBufferIntervalInSeconds(default: 60), are not stored in state. To prevent perpetual differences, it is therefore recommended to only include parameters with non-default values.
- parameter_
name str - Parameter name. Valid Values:
LambdaArn,NumberOfRetries,MetadataExtractionQuery,JsonParsingEngine,RoleArn,BufferSizeInMBs,BufferIntervalInSeconds,SubRecordType,Delimiter. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - parameter_
value str Parameter value. Must be between 1 and 512 length (inclusive). When providing a Lambda ARN, you should specify the resource version as well.
NOTE: Parameters with default values, including
NumberOfRetries(default: 3),RoleArn(default: firehose role ARN),BufferSizeInMBs(default: 3), andBufferIntervalInSeconds(default: 60), are not stored in state. To prevent perpetual differences, it is therefore recommended to only include parameters with non-default values.
- parameter
Name String - Parameter name. Valid Values:
LambdaArn,NumberOfRetries,MetadataExtractionQuery,JsonParsingEngine,RoleArn,BufferSizeInMBs,BufferIntervalInSeconds,SubRecordType,Delimiter. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - parameter
Value String Parameter value. Must be between 1 and 512 length (inclusive). When providing a Lambda ARN, you should specify the resource version as well.
NOTE: Parameters with default values, including
NumberOfRetries(default: 3),RoleArn(default: firehose role ARN),BufferSizeInMBs(default: 3), andBufferIntervalInSeconds(default: 60), are not stored in state. To prevent perpetual differences, it is therefore recommended to only include parameters with non-default values.
FirehoseDeliveryStreamExtendedS3ConfigurationS3BackupConfiguration, FirehoseDeliveryStreamExtendedS3ConfigurationS3BackupConfigurationArgs
- Bucket
Arn string - The ARN of the S3 bucket
- Role
Arn string - The ARN of the role that provides access to the source Kinesis stream.
- Buffer
Interval int - Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination. The default value is 300.
- Buffer
Size int - Buffer incoming data to the specified size, in MBs, before delivering it to the destination. The default value is 5. We recommend setting SizeInMBs to a value greater than the amount of data you typically ingest into the delivery stream in 10 seconds. For example, if you typically ingest data at 1 MB/sec set SizeInMBs to be 10 MB or higher.
- Cloudwatch
Logging FirehoseOptions Delivery Stream Extended S3Configuration S3Backup Configuration Cloudwatch Logging Options The CloudWatch Logging Options for the delivery stream. More details are given below
The
extended_s3_configurationobject supports the same fields froms3_configurationas well as the following:- Compression
Format string - The compression format. If no value is specified, the default is
UNCOMPRESSED. Other supported values areGZIP,ZIP,Snappy, &HADOOP_SNAPPY. - Error
Output stringPrefix - Prefix added to failed records before writing them to S3. Not currently supported for
redshiftdestination. This prefix appears immediately following the bucket name. For information about how to specify this prefix, see Custom Prefixes for Amazon S3 Objects. - Kms
Key stringArn - Specifies the KMS key ARN the stream will use to encrypt data. If not set, no encryption will be used.
- Prefix string
- The "YYYY/MM/DD/HH" time format prefix is automatically used for delivered S3 files. You can specify an extra prefix to be added in front of the time format prefix. Note that if the prefix ends with a slash, it appears as a folder in the S3 bucket
- Bucket
Arn string - The ARN of the S3 bucket
- Role
Arn string - The ARN of the role that provides access to the source Kinesis stream.
- Buffer
Interval int - Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination. The default value is 300.
- Buffer
Size int - Buffer incoming data to the specified size, in MBs, before delivering it to the destination. The default value is 5. We recommend setting SizeInMBs to a value greater than the amount of data you typically ingest into the delivery stream in 10 seconds. For example, if you typically ingest data at 1 MB/sec set SizeInMBs to be 10 MB or higher.
- Cloudwatch
Logging FirehoseOptions Delivery Stream Extended S3Configuration S3Backup Configuration Cloudwatch Logging Options The CloudWatch Logging Options for the delivery stream. More details are given below
The
extended_s3_configurationobject supports the same fields froms3_configurationas well as the following:- Compression
Format string - The compression format. If no value is specified, the default is
UNCOMPRESSED. Other supported values areGZIP,ZIP,Snappy, &HADOOP_SNAPPY. - Error
Output stringPrefix - Prefix added to failed records before writing them to S3. Not currently supported for
redshiftdestination. This prefix appears immediately following the bucket name. For information about how to specify this prefix, see Custom Prefixes for Amazon S3 Objects. - Kms
Key stringArn - Specifies the KMS key ARN the stream will use to encrypt data. If not set, no encryption will be used.
- Prefix string
- The "YYYY/MM/DD/HH" time format prefix is automatically used for delivered S3 files. You can specify an extra prefix to be added in front of the time format prefix. Note that if the prefix ends with a slash, it appears as a folder in the S3 bucket
- bucket
Arn String - The ARN of the S3 bucket
- role
Arn String - The ARN of the role that provides access to the source Kinesis stream.
- buffer
Interval Integer - Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination. The default value is 300.
- buffer
Size Integer - Buffer incoming data to the specified size, in MBs, before delivering it to the destination. The default value is 5. We recommend setting SizeInMBs to a value greater than the amount of data you typically ingest into the delivery stream in 10 seconds. For example, if you typically ingest data at 1 MB/sec set SizeInMBs to be 10 MB or higher.
- cloudwatch
Logging FirehoseOptions Delivery Stream Extended S3Configuration S3Backup Configuration Cloudwatch Logging Options The CloudWatch Logging Options for the delivery stream. More details are given below
The
extended_s3_configurationobject supports the same fields froms3_configurationas well as the following:- compression
Format String - The compression format. If no value is specified, the default is
UNCOMPRESSED. Other supported values areGZIP,ZIP,Snappy, &HADOOP_SNAPPY. - error
Output StringPrefix - Prefix added to failed records before writing them to S3. Not currently supported for
redshiftdestination. This prefix appears immediately following the bucket name. For information about how to specify this prefix, see Custom Prefixes for Amazon S3 Objects. - kms
Key StringArn - Specifies the KMS key ARN the stream will use to encrypt data. If not set, no encryption will be used.
- prefix String
- The "YYYY/MM/DD/HH" time format prefix is automatically used for delivered S3 files. You can specify an extra prefix to be added in front of the time format prefix. Note that if the prefix ends with a slash, it appears as a folder in the S3 bucket
- bucket
Arn string - The ARN of the S3 bucket
- role
Arn string - The ARN of the role that provides access to the source Kinesis stream.
- buffer
Interval number - Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination. The default value is 300.
- buffer
Size number - Buffer incoming data to the specified size, in MBs, before delivering it to the destination. The default value is 5. We recommend setting SizeInMBs to a value greater than the amount of data you typically ingest into the delivery stream in 10 seconds. For example, if you typically ingest data at 1 MB/sec set SizeInMBs to be 10 MB or higher.
- cloudwatch
Logging FirehoseOptions Delivery Stream Extended S3Configuration S3Backup Configuration Cloudwatch Logging Options The CloudWatch Logging Options for the delivery stream. More details are given below
The
extended_s3_configurationobject supports the same fields froms3_configurationas well as the following:- compression
Format string - The compression format. If no value is specified, the default is
UNCOMPRESSED. Other supported values areGZIP,ZIP,Snappy, &HADOOP_SNAPPY. - error
Output stringPrefix - Prefix added to failed records before writing them to S3. Not currently supported for
redshiftdestination. This prefix appears immediately following the bucket name. For information about how to specify this prefix, see Custom Prefixes for Amazon S3 Objects. - kms
Key stringArn - Specifies the KMS key ARN the stream will use to encrypt data. If not set, no encryption will be used.
- prefix string
- The "YYYY/MM/DD/HH" time format prefix is automatically used for delivered S3 files. You can specify an extra prefix to be added in front of the time format prefix. Note that if the prefix ends with a slash, it appears as a folder in the S3 bucket
- bucket_
arn str - The ARN of the S3 bucket
- role_
arn str - The ARN of the role that provides access to the source Kinesis stream.
- buffer_
interval int - Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination. The default value is 300.
- buffer_
size int - Buffer incoming data to the specified size, in MBs, before delivering it to the destination. The default value is 5. We recommend setting SizeInMBs to a value greater than the amount of data you typically ingest into the delivery stream in 10 seconds. For example, if you typically ingest data at 1 MB/sec set SizeInMBs to be 10 MB or higher.
- cloudwatch_
logging_ Firehoseoptions Delivery Stream Extended S3Configuration S3Backup Configuration Cloudwatch Logging Options The CloudWatch Logging Options for the delivery stream. More details are given below
The
extended_s3_configurationobject supports the same fields froms3_configurationas well as the following:- compression_
format str - The compression format. If no value is specified, the default is
UNCOMPRESSED. Other supported values areGZIP,ZIP,Snappy, &HADOOP_SNAPPY. - error_
output_ strprefix - Prefix added to failed records before writing them to S3. Not currently supported for
redshiftdestination. This prefix appears immediately following the bucket name. For information about how to specify this prefix, see Custom Prefixes for Amazon S3 Objects. - kms_
key_ strarn - Specifies the KMS key ARN the stream will use to encrypt data. If not set, no encryption will be used.
- prefix str
- The "YYYY/MM/DD/HH" time format prefix is automatically used for delivered S3 files. You can specify an extra prefix to be added in front of the time format prefix. Note that if the prefix ends with a slash, it appears as a folder in the S3 bucket
- bucket
Arn String - The ARN of the S3 bucket
- role
Arn String - The ARN of the role that provides access to the source Kinesis stream.
- buffer
Interval Number - Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination. The default value is 300.
- buffer
Size Number - Buffer incoming data to the specified size, in MBs, before delivering it to the destination. The default value is 5. We recommend setting SizeInMBs to a value greater than the amount of data you typically ingest into the delivery stream in 10 seconds. For example, if you typically ingest data at 1 MB/sec set SizeInMBs to be 10 MB or higher.
- cloudwatch
Logging Property MapOptions The CloudWatch Logging Options for the delivery stream. More details are given below
The
extended_s3_configurationobject supports the same fields froms3_configurationas well as the following:- compression
Format String - The compression format. If no value is specified, the default is
UNCOMPRESSED. Other supported values areGZIP,ZIP,Snappy, &HADOOP_SNAPPY. - error
Output StringPrefix - Prefix added to failed records before writing them to S3. Not currently supported for
redshiftdestination. This prefix appears immediately following the bucket name. For information about how to specify this prefix, see Custom Prefixes for Amazon S3 Objects. - kms
Key StringArn - Specifies the KMS key ARN the stream will use to encrypt data. If not set, no encryption will be used.
- prefix String
- The "YYYY/MM/DD/HH" time format prefix is automatically used for delivered S3 files. You can specify an extra prefix to be added in front of the time format prefix. Note that if the prefix ends with a slash, it appears as a folder in the S3 bucket
FirehoseDeliveryStreamExtendedS3ConfigurationS3BackupConfigurationCloudwatchLoggingOptions, FirehoseDeliveryStreamExtendedS3ConfigurationS3BackupConfigurationCloudwatchLoggingOptionsArgs
- Enabled bool
- Enables or disables the logging. Defaults to
false. - Log
Group stringName - The CloudWatch group name for logging. This value is required if
enabledis true. - Log
Stream stringName - The CloudWatch log stream name for logging. This value is required if
enabledis true.
- Enabled bool
- Enables or disables the logging. Defaults to
false. - Log
Group stringName - The CloudWatch group name for logging. This value is required if
enabledis true. - Log
Stream stringName - The CloudWatch log stream name for logging. This value is required if
enabledis true.
- enabled Boolean
- Enables or disables the logging. Defaults to
false. - log
Group StringName - The CloudWatch group name for logging. This value is required if
enabledis true. - log
Stream StringName - The CloudWatch log stream name for logging. This value is required if
enabledis true.
- enabled boolean
- Enables or disables the logging. Defaults to
false. - log
Group stringName - The CloudWatch group name for logging. This value is required if
enabledis true. - log
Stream stringName - The CloudWatch log stream name for logging. This value is required if
enabledis true.
- enabled bool
- Enables or disables the logging. Defaults to
false. - log_
group_ strname - The CloudWatch group name for logging. This value is required if
enabledis true. - log_
stream_ strname - The CloudWatch log stream name for logging. This value is required if
enabledis true.
- enabled Boolean
- Enables or disables the logging. Defaults to
false. - log
Group StringName - The CloudWatch group name for logging. This value is required if
enabledis true. - log
Stream StringName - The CloudWatch log stream name for logging. This value is required if
enabledis true.
FirehoseDeliveryStreamHttpEndpointConfiguration, FirehoseDeliveryStreamHttpEndpointConfigurationArgs
- Url string
- The HTTP endpoint URL to which Kinesis Firehose sends your data.
- Access
Key string - The access key required for Kinesis Firehose to authenticate with the HTTP endpoint selected as the destination.
- Buffering
Interval int - Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination. The default value is 300 (5 minutes).
- Buffering
Size int - Buffer incoming data to the specified size, in MBs, before delivering it to the destination. The default value is 5.
- Cloudwatch
Logging FirehoseOptions Delivery Stream Http Endpoint Configuration Cloudwatch Logging Options - The CloudWatch Logging Options for the delivery stream. More details are given below.
- Name string
- The HTTP endpoint name.
- Processing
Configuration FirehoseDelivery Stream Http Endpoint Configuration Processing Configuration - The data processing configuration. More details are given below.
- Request
Configuration FirehoseDelivery Stream Http Endpoint Configuration Request Configuration - The request configuration. More details are given below.
- Retry
Duration int - Total amount of seconds Firehose spends on retries. This duration starts after the initial attempt fails, It does not include the time periods during which Firehose waits for acknowledgment from the specified destination after each attempt. Valid values between
0and7200. Default is300. - Role
Arn string - Kinesis Data Firehose uses this IAM role for all the permissions that the delivery stream needs. The pattern needs to be
arn:.*. - S3Backup
Mode string - Defines how documents should be delivered to Amazon S3. Valid values are
FailedDataOnlyandAllData. Default value isFailedDataOnly.
- Url string
- The HTTP endpoint URL to which Kinesis Firehose sends your data.
- Access
Key string - The access key required for Kinesis Firehose to authenticate with the HTTP endpoint selected as the destination.
- Buffering
Interval int - Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination. The default value is 300 (5 minutes).
- Buffering
Size int - Buffer incoming data to the specified size, in MBs, before delivering it to the destination. The default value is 5.
- Cloudwatch
Logging FirehoseOptions Delivery Stream Http Endpoint Configuration Cloudwatch Logging Options - The CloudWatch Logging Options for the delivery stream. More details are given below.
- Name string
- The HTTP endpoint name.
- Processing
Configuration FirehoseDelivery Stream Http Endpoint Configuration Processing Configuration - The data processing configuration. More details are given below.
- Request
Configuration FirehoseDelivery Stream Http Endpoint Configuration Request Configuration - The request configuration. More details are given below.
- Retry
Duration int - Total amount of seconds Firehose spends on retries. This duration starts after the initial attempt fails, It does not include the time periods during which Firehose waits for acknowledgment from the specified destination after each attempt. Valid values between
0and7200. Default is300. - Role
Arn string - Kinesis Data Firehose uses this IAM role for all the permissions that the delivery stream needs. The pattern needs to be
arn:.*. - S3Backup
Mode string - Defines how documents should be delivered to Amazon S3. Valid values are
FailedDataOnlyandAllData. Default value isFailedDataOnly.
- url String
- The HTTP endpoint URL to which Kinesis Firehose sends your data.
- access
Key String - The access key required for Kinesis Firehose to authenticate with the HTTP endpoint selected as the destination.
- buffering
Interval Integer - Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination. The default value is 300 (5 minutes).
- buffering
Size Integer - Buffer incoming data to the specified size, in MBs, before delivering it to the destination. The default value is 5.
- cloudwatch
Logging FirehoseOptions Delivery Stream Http Endpoint Configuration Cloudwatch Logging Options - The CloudWatch Logging Options for the delivery stream. More details are given below.
- name String
- The HTTP endpoint name.
- processing
Configuration FirehoseDelivery Stream Http Endpoint Configuration Processing Configuration - The data processing configuration. More details are given below.
- request
Configuration FirehoseDelivery Stream Http Endpoint Configuration Request Configuration - The request configuration. More details are given below.
- retry
Duration Integer - Total amount of seconds Firehose spends on retries. This duration starts after the initial attempt fails, It does not include the time periods during which Firehose waits for acknowledgment from the specified destination after each attempt. Valid values between
0and7200. Default is300. - role
Arn String - Kinesis Data Firehose uses this IAM role for all the permissions that the delivery stream needs. The pattern needs to be
arn:.*. - s3Backup
Mode String - Defines how documents should be delivered to Amazon S3. Valid values are
FailedDataOnlyandAllData. Default value isFailedDataOnly.
- url string
- The HTTP endpoint URL to which Kinesis Firehose sends your data.
- access
Key string - The access key required for Kinesis Firehose to authenticate with the HTTP endpoint selected as the destination.
- buffering
Interval number - Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination. The default value is 300 (5 minutes).
- buffering
Size number - Buffer incoming data to the specified size, in MBs, before delivering it to the destination. The default value is 5.
- cloudwatch
Logging FirehoseOptions Delivery Stream Http Endpoint Configuration Cloudwatch Logging Options - The CloudWatch Logging Options for the delivery stream. More details are given below.
- name string
- The HTTP endpoint name.
- processing
Configuration FirehoseDelivery Stream Http Endpoint Configuration Processing Configuration - The data processing configuration. More details are given below.
- request
Configuration FirehoseDelivery Stream Http Endpoint Configuration Request Configuration - The request configuration. More details are given below.
- retry
Duration number - Total amount of seconds Firehose spends on retries. This duration starts after the initial attempt fails, It does not include the time periods during which Firehose waits for acknowledgment from the specified destination after each attempt. Valid values between
0and7200. Default is300. - role
Arn string - Kinesis Data Firehose uses this IAM role for all the permissions that the delivery stream needs. The pattern needs to be
arn:.*. - s3Backup
Mode string - Defines how documents should be delivered to Amazon S3. Valid values are
FailedDataOnlyandAllData. Default value isFailedDataOnly.
- url str
- The HTTP endpoint URL to which Kinesis Firehose sends your data.
- access_
key str - The access key required for Kinesis Firehose to authenticate with the HTTP endpoint selected as the destination.
- buffering_
interval int - Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination. The default value is 300 (5 minutes).
- buffering_
size int - Buffer incoming data to the specified size, in MBs, before delivering it to the destination. The default value is 5.
- cloudwatch_
logging_ Firehoseoptions Delivery Stream Http Endpoint Configuration Cloudwatch Logging Options - The CloudWatch Logging Options for the delivery stream. More details are given below.
- name str
- The HTTP endpoint name.
- processing_
configuration FirehoseDelivery Stream Http Endpoint Configuration Processing Configuration - The data processing configuration. More details are given below.
- request_
configuration FirehoseDelivery Stream Http Endpoint Configuration Request Configuration - The request configuration. More details are given below.
- retry_
duration int - Total amount of seconds Firehose spends on retries. This duration starts after the initial attempt fails, It does not include the time periods during which Firehose waits for acknowledgment from the specified destination after each attempt. Valid values between
0and7200. Default is300. - role_
arn str - Kinesis Data Firehose uses this IAM role for all the permissions that the delivery stream needs. The pattern needs to be
arn:.*. - s3_
backup_ strmode - Defines how documents should be delivered to Amazon S3. Valid values are
FailedDataOnlyandAllData. Default value isFailedDataOnly.
- url String
- The HTTP endpoint URL to which Kinesis Firehose sends your data.
- access
Key String - The access key required for Kinesis Firehose to authenticate with the HTTP endpoint selected as the destination.
- buffering
Interval Number - Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination. The default value is 300 (5 minutes).
- buffering
Size Number - Buffer incoming data to the specified size, in MBs, before delivering it to the destination. The default value is 5.
- cloudwatch
Logging Property MapOptions - The CloudWatch Logging Options for the delivery stream. More details are given below.
- name String
- The HTTP endpoint name.
- processing
Configuration Property Map - The data processing configuration. More details are given below.
- request
Configuration Property Map - The request configuration. More details are given below.
- retry
Duration Number - Total amount of seconds Firehose spends on retries. This duration starts after the initial attempt fails, It does not include the time periods during which Firehose waits for acknowledgment from the specified destination after each attempt. Valid values between
0and7200. Default is300. - role
Arn String - Kinesis Data Firehose uses this IAM role for all the permissions that the delivery stream needs. The pattern needs to be
arn:.*. - s3Backup
Mode String - Defines how documents should be delivered to Amazon S3. Valid values are
FailedDataOnlyandAllData. Default value isFailedDataOnly.
FirehoseDeliveryStreamHttpEndpointConfigurationCloudwatchLoggingOptions, FirehoseDeliveryStreamHttpEndpointConfigurationCloudwatchLoggingOptionsArgs
- Enabled bool
- Enables or disables the logging. Defaults to
false. - Log
Group stringName - The CloudWatch group name for logging. This value is required if
enabledis true. - Log
Stream stringName - The CloudWatch log stream name for logging. This value is required if
enabledis true.
- Enabled bool
- Enables or disables the logging. Defaults to
false. - Log
Group stringName - The CloudWatch group name for logging. This value is required if
enabledis true. - Log
Stream stringName - The CloudWatch log stream name for logging. This value is required if
enabledis true.
- enabled Boolean
- Enables or disables the logging. Defaults to
false. - log
Group StringName - The CloudWatch group name for logging. This value is required if
enabledis true. - log
Stream StringName - The CloudWatch log stream name for logging. This value is required if
enabledis true.
- enabled boolean
- Enables or disables the logging. Defaults to
false. - log
Group stringName - The CloudWatch group name for logging. This value is required if
enabledis true. - log
Stream stringName - The CloudWatch log stream name for logging. This value is required if
enabledis true.
- enabled bool
- Enables or disables the logging. Defaults to
false. - log_
group_ strname - The CloudWatch group name for logging. This value is required if
enabledis true. - log_
stream_ strname - The CloudWatch log stream name for logging. This value is required if
enabledis true.
- enabled Boolean
- Enables or disables the logging. Defaults to
false. - log
Group StringName - The CloudWatch group name for logging. This value is required if
enabledis true. - log
Stream StringName - The CloudWatch log stream name for logging. This value is required if
enabledis true.
FirehoseDeliveryStreamHttpEndpointConfigurationProcessingConfiguration, FirehoseDeliveryStreamHttpEndpointConfigurationProcessingConfigurationArgs
- Enabled bool
- Enables or disables data processing.
- Processors
List<Firehose
Delivery Stream Http Endpoint Configuration Processing Configuration Processor> - Array of data processors. More details are given below
- Enabled bool
- Enables or disables data processing.
- Processors
[]Firehose
Delivery Stream Http Endpoint Configuration Processing Configuration Processor - Array of data processors. More details are given below
- enabled Boolean
- Enables or disables data processing.
- processors
List<Firehose
Delivery Stream Http Endpoint Configuration Processing Configuration Processor> - Array of data processors. More details are given below
- enabled boolean
- Enables or disables data processing.
- processors
Firehose
Delivery Stream Http Endpoint Configuration Processing Configuration Processor[] - Array of data processors. More details are given below
- enabled bool
- Enables or disables data processing.
- processors
Sequence[Firehose
Delivery Stream Http Endpoint Configuration Processing Configuration Processor] - Array of data processors. More details are given below
- enabled Boolean
- Enables or disables data processing.
- processors List<Property Map>
- Array of data processors. More details are given below
FirehoseDeliveryStreamHttpEndpointConfigurationProcessingConfigurationProcessor, FirehoseDeliveryStreamHttpEndpointConfigurationProcessingConfigurationProcessorArgs
- Type string
- The type of processor. Valid Values:
RecordDeAggregation,Lambda,MetadataExtraction,AppendDelimiterToRecord. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - Parameters
List<Firehose
Delivery Stream Http Endpoint Configuration Processing Configuration Processor Parameter> - Array of processor parameters. More details are given below
- Type string
- The type of processor. Valid Values:
RecordDeAggregation,Lambda,MetadataExtraction,AppendDelimiterToRecord. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - Parameters
[]Firehose
Delivery Stream Http Endpoint Configuration Processing Configuration Processor Parameter - Array of processor parameters. More details are given below
- type String
- The type of processor. Valid Values:
RecordDeAggregation,Lambda,MetadataExtraction,AppendDelimiterToRecord. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - parameters
List<Firehose
Delivery Stream Http Endpoint Configuration Processing Configuration Processor Parameter> - Array of processor parameters. More details are given below
- type string
- The type of processor. Valid Values:
RecordDeAggregation,Lambda,MetadataExtraction,AppendDelimiterToRecord. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - parameters
Firehose
Delivery Stream Http Endpoint Configuration Processing Configuration Processor Parameter[] - Array of processor parameters. More details are given below
- type str
- The type of processor. Valid Values:
RecordDeAggregation,Lambda,MetadataExtraction,AppendDelimiterToRecord. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - parameters
Sequence[Firehose
Delivery Stream Http Endpoint Configuration Processing Configuration Processor Parameter] - Array of processor parameters. More details are given below
- type String
- The type of processor. Valid Values:
RecordDeAggregation,Lambda,MetadataExtraction,AppendDelimiterToRecord. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - parameters List<Property Map>
- Array of processor parameters. More details are given below
FirehoseDeliveryStreamHttpEndpointConfigurationProcessingConfigurationProcessorParameter, FirehoseDeliveryStreamHttpEndpointConfigurationProcessingConfigurationProcessorParameterArgs
- Parameter
Name string - Parameter name. Valid Values:
LambdaArn,NumberOfRetries,MetadataExtractionQuery,JsonParsingEngine,RoleArn,BufferSizeInMBs,BufferIntervalInSeconds,SubRecordType,Delimiter. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - Parameter
Value string Parameter value. Must be between 1 and 512 length (inclusive). When providing a Lambda ARN, you should specify the resource version as well.
NOTE: Parameters with default values, including
NumberOfRetries(default: 3),RoleArn(default: firehose role ARN),BufferSizeInMBs(default: 3), andBufferIntervalInSeconds(default: 60), are not stored in state. To prevent perpetual differences, it is therefore recommended to only include parameters with non-default values.
- Parameter
Name string - Parameter name. Valid Values:
LambdaArn,NumberOfRetries,MetadataExtractionQuery,JsonParsingEngine,RoleArn,BufferSizeInMBs,BufferIntervalInSeconds,SubRecordType,Delimiter. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - Parameter
Value string Parameter value. Must be between 1 and 512 length (inclusive). When providing a Lambda ARN, you should specify the resource version as well.
NOTE: Parameters with default values, including
NumberOfRetries(default: 3),RoleArn(default: firehose role ARN),BufferSizeInMBs(default: 3), andBufferIntervalInSeconds(default: 60), are not stored in state. To prevent perpetual differences, it is therefore recommended to only include parameters with non-default values.
- parameter
Name String - Parameter name. Valid Values:
LambdaArn,NumberOfRetries,MetadataExtractionQuery,JsonParsingEngine,RoleArn,BufferSizeInMBs,BufferIntervalInSeconds,SubRecordType,Delimiter. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - parameter
Value String Parameter value. Must be between 1 and 512 length (inclusive). When providing a Lambda ARN, you should specify the resource version as well.
NOTE: Parameters with default values, including
NumberOfRetries(default: 3),RoleArn(default: firehose role ARN),BufferSizeInMBs(default: 3), andBufferIntervalInSeconds(default: 60), are not stored in state. To prevent perpetual differences, it is therefore recommended to only include parameters with non-default values.
- parameter
Name string - Parameter name. Valid Values:
LambdaArn,NumberOfRetries,MetadataExtractionQuery,JsonParsingEngine,RoleArn,BufferSizeInMBs,BufferIntervalInSeconds,SubRecordType,Delimiter. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - parameter
Value string Parameter value. Must be between 1 and 512 length (inclusive). When providing a Lambda ARN, you should specify the resource version as well.
NOTE: Parameters with default values, including
NumberOfRetries(default: 3),RoleArn(default: firehose role ARN),BufferSizeInMBs(default: 3), andBufferIntervalInSeconds(default: 60), are not stored in state. To prevent perpetual differences, it is therefore recommended to only include parameters with non-default values.
- parameter_
name str - Parameter name. Valid Values:
LambdaArn,NumberOfRetries,MetadataExtractionQuery,JsonParsingEngine,RoleArn,BufferSizeInMBs,BufferIntervalInSeconds,SubRecordType,Delimiter. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - parameter_
value str Parameter value. Must be between 1 and 512 length (inclusive). When providing a Lambda ARN, you should specify the resource version as well.
NOTE: Parameters with default values, including
NumberOfRetries(default: 3),RoleArn(default: firehose role ARN),BufferSizeInMBs(default: 3), andBufferIntervalInSeconds(default: 60), are not stored in state. To prevent perpetual differences, it is therefore recommended to only include parameters with non-default values.
- parameter
Name String - Parameter name. Valid Values:
LambdaArn,NumberOfRetries,MetadataExtractionQuery,JsonParsingEngine,RoleArn,BufferSizeInMBs,BufferIntervalInSeconds,SubRecordType,Delimiter. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - parameter
Value String Parameter value. Must be between 1 and 512 length (inclusive). When providing a Lambda ARN, you should specify the resource version as well.
NOTE: Parameters with default values, including
NumberOfRetries(default: 3),RoleArn(default: firehose role ARN),BufferSizeInMBs(default: 3), andBufferIntervalInSeconds(default: 60), are not stored in state. To prevent perpetual differences, it is therefore recommended to only include parameters with non-default values.
FirehoseDeliveryStreamHttpEndpointConfigurationRequestConfiguration, FirehoseDeliveryStreamHttpEndpointConfigurationRequestConfigurationArgs
- Common
Attributes List<FirehoseDelivery Stream Http Endpoint Configuration Request Configuration Common Attribute> - Describes the metadata sent to the HTTP endpoint destination. More details are given below
- Content
Encoding string - Kinesis Data Firehose uses the content encoding to compress the body of a request before sending the request to the destination. Valid values are
NONEandGZIP. Default value isNONE.
- Common
Attributes []FirehoseDelivery Stream Http Endpoint Configuration Request Configuration Common Attribute - Describes the metadata sent to the HTTP endpoint destination. More details are given below
- Content
Encoding string - Kinesis Data Firehose uses the content encoding to compress the body of a request before sending the request to the destination. Valid values are
NONEandGZIP. Default value isNONE.
- common
Attributes List<FirehoseDelivery Stream Http Endpoint Configuration Request Configuration Common Attribute> - Describes the metadata sent to the HTTP endpoint destination. More details are given below
- content
Encoding String - Kinesis Data Firehose uses the content encoding to compress the body of a request before sending the request to the destination. Valid values are
NONEandGZIP. Default value isNONE.
- common
Attributes FirehoseDelivery Stream Http Endpoint Configuration Request Configuration Common Attribute[] - Describes the metadata sent to the HTTP endpoint destination. More details are given below
- content
Encoding string - Kinesis Data Firehose uses the content encoding to compress the body of a request before sending the request to the destination. Valid values are
NONEandGZIP. Default value isNONE.
- common_
attributes Sequence[FirehoseDelivery Stream Http Endpoint Configuration Request Configuration Common Attribute] - Describes the metadata sent to the HTTP endpoint destination. More details are given below
- content_
encoding str - Kinesis Data Firehose uses the content encoding to compress the body of a request before sending the request to the destination. Valid values are
NONEandGZIP. Default value isNONE.
- common
Attributes List<Property Map> - Describes the metadata sent to the HTTP endpoint destination. More details are given below
- content
Encoding String - Kinesis Data Firehose uses the content encoding to compress the body of a request before sending the request to the destination. Valid values are
NONEandGZIP. Default value isNONE.
FirehoseDeliveryStreamHttpEndpointConfigurationRequestConfigurationCommonAttribute, FirehoseDeliveryStreamHttpEndpointConfigurationRequestConfigurationCommonAttributeArgs
FirehoseDeliveryStreamKinesisSourceConfiguration, FirehoseDeliveryStreamKinesisSourceConfigurationArgs
- Kinesis
Stream stringArn - The kinesis stream used as the source of the firehose delivery stream.
- Role
Arn string - The ARN of the role that provides access to the source Kinesis stream.
- Kinesis
Stream stringArn - The kinesis stream used as the source of the firehose delivery stream.
- Role
Arn string - The ARN of the role that provides access to the source Kinesis stream.
- kinesis
Stream StringArn - The kinesis stream used as the source of the firehose delivery stream.
- role
Arn String - The ARN of the role that provides access to the source Kinesis stream.
- kinesis
Stream stringArn - The kinesis stream used as the source of the firehose delivery stream.
- role
Arn string - The ARN of the role that provides access to the source Kinesis stream.
- kinesis_
stream_ strarn - The kinesis stream used as the source of the firehose delivery stream.
- role_
arn str - The ARN of the role that provides access to the source Kinesis stream.
- kinesis
Stream StringArn - The kinesis stream used as the source of the firehose delivery stream.
- role
Arn String - The ARN of the role that provides access to the source Kinesis stream.
FirehoseDeliveryStreamOpensearchConfiguration, FirehoseDeliveryStreamOpensearchConfigurationArgs
- Index
Name string - The Opensearch index name.
- Role
Arn string - The ARN of the IAM role to be assumed by Firehose for calling the Amazon ES Configuration API and for indexing documents. The IAM role must have permission for
DescribeDomain,DescribeDomains, andDescribeDomainConfig. The pattern needs to bearn:.*. - Buffering
Interval int - Buffer incoming data for the specified period of time, in seconds between 60 to 900, before delivering it to the destination. The default value is 300s.
- Buffering
Size int - Buffer incoming data to the specified size, in MBs between 1 to 100, before delivering it to the destination. The default value is 5MB.
- Cloudwatch
Logging FirehoseOptions Delivery Stream Opensearch Configuration Cloudwatch Logging Options - The CloudWatch Logging Options for the delivery stream. More details are given below
- Cluster
Endpoint string - The endpoint to use when communicating with the cluster. Conflicts with
domain_arn. - Domain
Arn string - The ARN of the Amazon ES domain. The pattern needs to be
arn:.*. Conflicts withcluster_endpoint. - Index
Rotation stringPeriod - The Opensearch index rotation period. Index rotation appends a timestamp to the IndexName to facilitate expiration of old data. Valid values are
NoRotation,OneHour,OneDay,OneWeek, andOneMonth. The default value isOneDay. - Processing
Configuration FirehoseDelivery Stream Opensearch Configuration Processing Configuration - The data processing configuration. More details are given below.
- Retry
Duration int - After an initial failure to deliver to Amazon OpenSearch, the total amount of time, in seconds between 0 to 7200, during which Firehose re-attempts delivery (including the first attempt). After this time has elapsed, the failed documents are written to Amazon S3. The default value is 300s. There will be no retry if the value is 0.
- S3Backup
Mode string - Defines how documents should be delivered to Amazon S3. Valid values are
FailedDocumentsOnlyandAllDocuments. Default value isFailedDocumentsOnly. - Type
Name string - The Elasticsearch type name with maximum length of 100 characters. Types are deprecated in OpenSearch_1.1. TypeName must be empty.
- Vpc
Config FirehoseDelivery Stream Opensearch Configuration Vpc Config - The VPC configuration for the delivery stream to connect to OpenSearch associated with the VPC. More details are given below
- Index
Name string - The Opensearch index name.
- Role
Arn string - The ARN of the IAM role to be assumed by Firehose for calling the Amazon ES Configuration API and for indexing documents. The IAM role must have permission for
DescribeDomain,DescribeDomains, andDescribeDomainConfig. The pattern needs to bearn:.*. - Buffering
Interval int - Buffer incoming data for the specified period of time, in seconds between 60 to 900, before delivering it to the destination. The default value is 300s.
- Buffering
Size int - Buffer incoming data to the specified size, in MBs between 1 to 100, before delivering it to the destination. The default value is 5MB.
- Cloudwatch
Logging FirehoseOptions Delivery Stream Opensearch Configuration Cloudwatch Logging Options - The CloudWatch Logging Options for the delivery stream. More details are given below
- Cluster
Endpoint string - The endpoint to use when communicating with the cluster. Conflicts with
domain_arn. - Domain
Arn string - The ARN of the Amazon ES domain. The pattern needs to be
arn:.*. Conflicts withcluster_endpoint. - Index
Rotation stringPeriod - The Opensearch index rotation period. Index rotation appends a timestamp to the IndexName to facilitate expiration of old data. Valid values are
NoRotation,OneHour,OneDay,OneWeek, andOneMonth. The default value isOneDay. - Processing
Configuration FirehoseDelivery Stream Opensearch Configuration Processing Configuration - The data processing configuration. More details are given below.
- Retry
Duration int - After an initial failure to deliver to Amazon OpenSearch, the total amount of time, in seconds between 0 to 7200, during which Firehose re-attempts delivery (including the first attempt). After this time has elapsed, the failed documents are written to Amazon S3. The default value is 300s. There will be no retry if the value is 0.
- S3Backup
Mode string - Defines how documents should be delivered to Amazon S3. Valid values are
FailedDocumentsOnlyandAllDocuments. Default value isFailedDocumentsOnly. - Type
Name string - The Elasticsearch type name with maximum length of 100 characters. Types are deprecated in OpenSearch_1.1. TypeName must be empty.
- Vpc
Config FirehoseDelivery Stream Opensearch Configuration Vpc Config - The VPC configuration for the delivery stream to connect to OpenSearch associated with the VPC. More details are given below
- index
Name String - The Opensearch index name.
- role
Arn String - The ARN of the IAM role to be assumed by Firehose for calling the Amazon ES Configuration API and for indexing documents. The IAM role must have permission for
DescribeDomain,DescribeDomains, andDescribeDomainConfig. The pattern needs to bearn:.*. - buffering
Interval Integer - Buffer incoming data for the specified period of time, in seconds between 60 to 900, before delivering it to the destination. The default value is 300s.
- buffering
Size Integer - Buffer incoming data to the specified size, in MBs between 1 to 100, before delivering it to the destination. The default value is 5MB.
- cloudwatch
Logging FirehoseOptions Delivery Stream Opensearch Configuration Cloudwatch Logging Options - The CloudWatch Logging Options for the delivery stream. More details are given below
- cluster
Endpoint String - The endpoint to use when communicating with the cluster. Conflicts with
domain_arn. - domain
Arn String - The ARN of the Amazon ES domain. The pattern needs to be
arn:.*. Conflicts withcluster_endpoint. - index
Rotation StringPeriod - The Opensearch index rotation period. Index rotation appends a timestamp to the IndexName to facilitate expiration of old data. Valid values are
NoRotation,OneHour,OneDay,OneWeek, andOneMonth. The default value isOneDay. - processing
Configuration FirehoseDelivery Stream Opensearch Configuration Processing Configuration - The data processing configuration. More details are given below.
- retry
Duration Integer - After an initial failure to deliver to Amazon OpenSearch, the total amount of time, in seconds between 0 to 7200, during which Firehose re-attempts delivery (including the first attempt). After this time has elapsed, the failed documents are written to Amazon S3. The default value is 300s. There will be no retry if the value is 0.
- s3Backup
Mode String - Defines how documents should be delivered to Amazon S3. Valid values are
FailedDocumentsOnlyandAllDocuments. Default value isFailedDocumentsOnly. - type
Name String - The Elasticsearch type name with maximum length of 100 characters. Types are deprecated in OpenSearch_1.1. TypeName must be empty.
- vpc
Config FirehoseDelivery Stream Opensearch Configuration Vpc Config - The VPC configuration for the delivery stream to connect to OpenSearch associated with the VPC. More details are given below
- index
Name string - The Opensearch index name.
- role
Arn string - The ARN of the IAM role to be assumed by Firehose for calling the Amazon ES Configuration API and for indexing documents. The IAM role must have permission for
DescribeDomain,DescribeDomains, andDescribeDomainConfig. The pattern needs to bearn:.*. - buffering
Interval number - Buffer incoming data for the specified period of time, in seconds between 60 to 900, before delivering it to the destination. The default value is 300s.
- buffering
Size number - Buffer incoming data to the specified size, in MBs between 1 to 100, before delivering it to the destination. The default value is 5MB.
- cloudwatch
Logging FirehoseOptions Delivery Stream Opensearch Configuration Cloudwatch Logging Options - The CloudWatch Logging Options for the delivery stream. More details are given below
- cluster
Endpoint string - The endpoint to use when communicating with the cluster. Conflicts with
domain_arn. - domain
Arn string - The ARN of the Amazon ES domain. The pattern needs to be
arn:.*. Conflicts withcluster_endpoint. - index
Rotation stringPeriod - The Opensearch index rotation period. Index rotation appends a timestamp to the IndexName to facilitate expiration of old data. Valid values are
NoRotation,OneHour,OneDay,OneWeek, andOneMonth. The default value isOneDay. - processing
Configuration FirehoseDelivery Stream Opensearch Configuration Processing Configuration - The data processing configuration. More details are given below.
- retry
Duration number - After an initial failure to deliver to Amazon OpenSearch, the total amount of time, in seconds between 0 to 7200, during which Firehose re-attempts delivery (including the first attempt). After this time has elapsed, the failed documents are written to Amazon S3. The default value is 300s. There will be no retry if the value is 0.
- s3Backup
Mode string - Defines how documents should be delivered to Amazon S3. Valid values are
FailedDocumentsOnlyandAllDocuments. Default value isFailedDocumentsOnly. - type
Name string - The Elasticsearch type name with maximum length of 100 characters. Types are deprecated in OpenSearch_1.1. TypeName must be empty.
- vpc
Config FirehoseDelivery Stream Opensearch Configuration Vpc Config - The VPC configuration for the delivery stream to connect to OpenSearch associated with the VPC. More details are given below
- index_
name str - The Opensearch index name.
- role_
arn str - The ARN of the IAM role to be assumed by Firehose for calling the Amazon ES Configuration API and for indexing documents. The IAM role must have permission for
DescribeDomain,DescribeDomains, andDescribeDomainConfig. The pattern needs to bearn:.*. - buffering_
interval int - Buffer incoming data for the specified period of time, in seconds between 60 to 900, before delivering it to the destination. The default value is 300s.
- buffering_
size int - Buffer incoming data to the specified size, in MBs between 1 to 100, before delivering it to the destination. The default value is 5MB.
- cloudwatch_
logging_ Firehoseoptions Delivery Stream Opensearch Configuration Cloudwatch Logging Options - The CloudWatch Logging Options for the delivery stream. More details are given below
- cluster_
endpoint str - The endpoint to use when communicating with the cluster. Conflicts with
domain_arn. - domain_
arn str - The ARN of the Amazon ES domain. The pattern needs to be
arn:.*. Conflicts withcluster_endpoint. - index_
rotation_ strperiod - The Opensearch index rotation period. Index rotation appends a timestamp to the IndexName to facilitate expiration of old data. Valid values are
NoRotation,OneHour,OneDay,OneWeek, andOneMonth. The default value isOneDay. - processing_
configuration FirehoseDelivery Stream Opensearch Configuration Processing Configuration - The data processing configuration. More details are given below.
- retry_
duration int - After an initial failure to deliver to Amazon OpenSearch, the total amount of time, in seconds between 0 to 7200, during which Firehose re-attempts delivery (including the first attempt). After this time has elapsed, the failed documents are written to Amazon S3. The default value is 300s. There will be no retry if the value is 0.
- s3_
backup_ strmode - Defines how documents should be delivered to Amazon S3. Valid values are
FailedDocumentsOnlyandAllDocuments. Default value isFailedDocumentsOnly. - type_
name str - The Elasticsearch type name with maximum length of 100 characters. Types are deprecated in OpenSearch_1.1. TypeName must be empty.
- vpc_
config FirehoseDelivery Stream Opensearch Configuration Vpc Config - The VPC configuration for the delivery stream to connect to OpenSearch associated with the VPC. More details are given below
- index
Name String - The Opensearch index name.
- role
Arn String - The ARN of the IAM role to be assumed by Firehose for calling the Amazon ES Configuration API and for indexing documents. The IAM role must have permission for
DescribeDomain,DescribeDomains, andDescribeDomainConfig. The pattern needs to bearn:.*. - buffering
Interval Number - Buffer incoming data for the specified period of time, in seconds between 60 to 900, before delivering it to the destination. The default value is 300s.
- buffering
Size Number - Buffer incoming data to the specified size, in MBs between 1 to 100, before delivering it to the destination. The default value is 5MB.
- cloudwatch
Logging Property MapOptions - The CloudWatch Logging Options for the delivery stream. More details are given below
- cluster
Endpoint String - The endpoint to use when communicating with the cluster. Conflicts with
domain_arn. - domain
Arn String - The ARN of the Amazon ES domain. The pattern needs to be
arn:.*. Conflicts withcluster_endpoint. - index
Rotation StringPeriod - The Opensearch index rotation period. Index rotation appends a timestamp to the IndexName to facilitate expiration of old data. Valid values are
NoRotation,OneHour,OneDay,OneWeek, andOneMonth. The default value isOneDay. - processing
Configuration Property Map - The data processing configuration. More details are given below.
- retry
Duration Number - After an initial failure to deliver to Amazon OpenSearch, the total amount of time, in seconds between 0 to 7200, during which Firehose re-attempts delivery (including the first attempt). After this time has elapsed, the failed documents are written to Amazon S3. The default value is 300s. There will be no retry if the value is 0.
- s3Backup
Mode String - Defines how documents should be delivered to Amazon S3. Valid values are
FailedDocumentsOnlyandAllDocuments. Default value isFailedDocumentsOnly. - type
Name String - The Elasticsearch type name with maximum length of 100 characters. Types are deprecated in OpenSearch_1.1. TypeName must be empty.
- vpc
Config Property Map - The VPC configuration for the delivery stream to connect to OpenSearch associated with the VPC. More details are given below
FirehoseDeliveryStreamOpensearchConfigurationCloudwatchLoggingOptions, FirehoseDeliveryStreamOpensearchConfigurationCloudwatchLoggingOptionsArgs
- Enabled bool
- Enables or disables the logging. Defaults to
false. - Log
Group stringName - The CloudWatch group name for logging. This value is required if
enabledis true. - Log
Stream stringName - The CloudWatch log stream name for logging. This value is required if
enabledis true.
- Enabled bool
- Enables or disables the logging. Defaults to
false. - Log
Group stringName - The CloudWatch group name for logging. This value is required if
enabledis true. - Log
Stream stringName - The CloudWatch log stream name for logging. This value is required if
enabledis true.
- enabled Boolean
- Enables or disables the logging. Defaults to
false. - log
Group StringName - The CloudWatch group name for logging. This value is required if
enabledis true. - log
Stream StringName - The CloudWatch log stream name for logging. This value is required if
enabledis true.
- enabled boolean
- Enables or disables the logging. Defaults to
false. - log
Group stringName - The CloudWatch group name for logging. This value is required if
enabledis true. - log
Stream stringName - The CloudWatch log stream name for logging. This value is required if
enabledis true.
- enabled bool
- Enables or disables the logging. Defaults to
false. - log_
group_ strname - The CloudWatch group name for logging. This value is required if
enabledis true. - log_
stream_ strname - The CloudWatch log stream name for logging. This value is required if
enabledis true.
- enabled Boolean
- Enables or disables the logging. Defaults to
false. - log
Group StringName - The CloudWatch group name for logging. This value is required if
enabledis true. - log
Stream StringName - The CloudWatch log stream name for logging. This value is required if
enabledis true.
FirehoseDeliveryStreamOpensearchConfigurationProcessingConfiguration, FirehoseDeliveryStreamOpensearchConfigurationProcessingConfigurationArgs
- Enabled bool
- Enables or disables data processing.
- Processors
List<Firehose
Delivery Stream Opensearch Configuration Processing Configuration Processor> - Array of data processors. More details are given below
- Enabled bool
- Enables or disables data processing.
- Processors
[]Firehose
Delivery Stream Opensearch Configuration Processing Configuration Processor - Array of data processors. More details are given below
- enabled Boolean
- Enables or disables data processing.
- processors
List<Firehose
Delivery Stream Opensearch Configuration Processing Configuration Processor> - Array of data processors. More details are given below
- enabled boolean
- Enables or disables data processing.
- processors
Firehose
Delivery Stream Opensearch Configuration Processing Configuration Processor[] - Array of data processors. More details are given below
- enabled bool
- Enables or disables data processing.
- processors
Sequence[Firehose
Delivery Stream Opensearch Configuration Processing Configuration Processor] - Array of data processors. More details are given below
- enabled Boolean
- Enables or disables data processing.
- processors List<Property Map>
- Array of data processors. More details are given below
FirehoseDeliveryStreamOpensearchConfigurationProcessingConfigurationProcessor, FirehoseDeliveryStreamOpensearchConfigurationProcessingConfigurationProcessorArgs
- Type string
- The type of processor. Valid Values:
RecordDeAggregation,Lambda,MetadataExtraction,AppendDelimiterToRecord. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - Parameters
List<Firehose
Delivery Stream Opensearch Configuration Processing Configuration Processor Parameter> - Array of processor parameters. More details are given below
- Type string
- The type of processor. Valid Values:
RecordDeAggregation,Lambda,MetadataExtraction,AppendDelimiterToRecord. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - Parameters
[]Firehose
Delivery Stream Opensearch Configuration Processing Configuration Processor Parameter - Array of processor parameters. More details are given below
- type String
- The type of processor. Valid Values:
RecordDeAggregation,Lambda,MetadataExtraction,AppendDelimiterToRecord. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - parameters
List<Firehose
Delivery Stream Opensearch Configuration Processing Configuration Processor Parameter> - Array of processor parameters. More details are given below
- type string
- The type of processor. Valid Values:
RecordDeAggregation,Lambda,MetadataExtraction,AppendDelimiterToRecord. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - parameters
Firehose
Delivery Stream Opensearch Configuration Processing Configuration Processor Parameter[] - Array of processor parameters. More details are given below
- type str
- The type of processor. Valid Values:
RecordDeAggregation,Lambda,MetadataExtraction,AppendDelimiterToRecord. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - parameters
Sequence[Firehose
Delivery Stream Opensearch Configuration Processing Configuration Processor Parameter] - Array of processor parameters. More details are given below
- type String
- The type of processor. Valid Values:
RecordDeAggregation,Lambda,MetadataExtraction,AppendDelimiterToRecord. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - parameters List<Property Map>
- Array of processor parameters. More details are given below
FirehoseDeliveryStreamOpensearchConfigurationProcessingConfigurationProcessorParameter, FirehoseDeliveryStreamOpensearchConfigurationProcessingConfigurationProcessorParameterArgs
- Parameter
Name string - Parameter name. Valid Values:
LambdaArn,NumberOfRetries,MetadataExtractionQuery,JsonParsingEngine,RoleArn,BufferSizeInMBs,BufferIntervalInSeconds,SubRecordType,Delimiter. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - Parameter
Value string Parameter value. Must be between 1 and 512 length (inclusive). When providing a Lambda ARN, you should specify the resource version as well.
NOTE: Parameters with default values, including
NumberOfRetries(default: 3),RoleArn(default: firehose role ARN),BufferSizeInMBs(default: 3), andBufferIntervalInSeconds(default: 60), are not stored in state. To prevent perpetual differences, it is therefore recommended to only include parameters with non-default values.
- Parameter
Name string - Parameter name. Valid Values:
LambdaArn,NumberOfRetries,MetadataExtractionQuery,JsonParsingEngine,RoleArn,BufferSizeInMBs,BufferIntervalInSeconds,SubRecordType,Delimiter. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - Parameter
Value string Parameter value. Must be between 1 and 512 length (inclusive). When providing a Lambda ARN, you should specify the resource version as well.
NOTE: Parameters with default values, including
NumberOfRetries(default: 3),RoleArn(default: firehose role ARN),BufferSizeInMBs(default: 3), andBufferIntervalInSeconds(default: 60), are not stored in state. To prevent perpetual differences, it is therefore recommended to only include parameters with non-default values.
- parameter
Name String - Parameter name. Valid Values:
LambdaArn,NumberOfRetries,MetadataExtractionQuery,JsonParsingEngine,RoleArn,BufferSizeInMBs,BufferIntervalInSeconds,SubRecordType,Delimiter. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - parameter
Value String Parameter value. Must be between 1 and 512 length (inclusive). When providing a Lambda ARN, you should specify the resource version as well.
NOTE: Parameters with default values, including
NumberOfRetries(default: 3),RoleArn(default: firehose role ARN),BufferSizeInMBs(default: 3), andBufferIntervalInSeconds(default: 60), are not stored in state. To prevent perpetual differences, it is therefore recommended to only include parameters with non-default values.
- parameter
Name string - Parameter name. Valid Values:
LambdaArn,NumberOfRetries,MetadataExtractionQuery,JsonParsingEngine,RoleArn,BufferSizeInMBs,BufferIntervalInSeconds,SubRecordType,Delimiter. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - parameter
Value string Parameter value. Must be between 1 and 512 length (inclusive). When providing a Lambda ARN, you should specify the resource version as well.
NOTE: Parameters with default values, including
NumberOfRetries(default: 3),RoleArn(default: firehose role ARN),BufferSizeInMBs(default: 3), andBufferIntervalInSeconds(default: 60), are not stored in state. To prevent perpetual differences, it is therefore recommended to only include parameters with non-default values.
- parameter_
name str - Parameter name. Valid Values:
LambdaArn,NumberOfRetries,MetadataExtractionQuery,JsonParsingEngine,RoleArn,BufferSizeInMBs,BufferIntervalInSeconds,SubRecordType,Delimiter. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - parameter_
value str Parameter value. Must be between 1 and 512 length (inclusive). When providing a Lambda ARN, you should specify the resource version as well.
NOTE: Parameters with default values, including
NumberOfRetries(default: 3),RoleArn(default: firehose role ARN),BufferSizeInMBs(default: 3), andBufferIntervalInSeconds(default: 60), are not stored in state. To prevent perpetual differences, it is therefore recommended to only include parameters with non-default values.
- parameter
Name String - Parameter name. Valid Values:
LambdaArn,NumberOfRetries,MetadataExtractionQuery,JsonParsingEngine,RoleArn,BufferSizeInMBs,BufferIntervalInSeconds,SubRecordType,Delimiter. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - parameter
Value String Parameter value. Must be between 1 and 512 length (inclusive). When providing a Lambda ARN, you should specify the resource version as well.
NOTE: Parameters with default values, including
NumberOfRetries(default: 3),RoleArn(default: firehose role ARN),BufferSizeInMBs(default: 3), andBufferIntervalInSeconds(default: 60), are not stored in state. To prevent perpetual differences, it is therefore recommended to only include parameters with non-default values.
FirehoseDeliveryStreamOpensearchConfigurationVpcConfig, FirehoseDeliveryStreamOpensearchConfigurationVpcConfigArgs
- Role
Arn string - The ARN of the IAM role to be assumed by Firehose for calling the Amazon EC2 configuration API and for creating network interfaces. Make sure role has necessary IAM permissions
- Security
Group List<string>Ids - A list of security group IDs to associate with Kinesis Firehose.
- Subnet
Ids List<string> - A list of subnet IDs to associate with Kinesis Firehose.
- Vpc
Id string
- Role
Arn string - The ARN of the IAM role to be assumed by Firehose for calling the Amazon EC2 configuration API and for creating network interfaces. Make sure role has necessary IAM permissions
- Security
Group []stringIds - A list of security group IDs to associate with Kinesis Firehose.
- Subnet
Ids []string - A list of subnet IDs to associate with Kinesis Firehose.
- Vpc
Id string
- role
Arn String - The ARN of the IAM role to be assumed by Firehose for calling the Amazon EC2 configuration API and for creating network interfaces. Make sure role has necessary IAM permissions
- security
Group List<String>Ids - A list of security group IDs to associate with Kinesis Firehose.
- subnet
Ids List<String> - A list of subnet IDs to associate with Kinesis Firehose.
- vpc
Id String
- role
Arn string - The ARN of the IAM role to be assumed by Firehose for calling the Amazon EC2 configuration API and for creating network interfaces. Make sure role has necessary IAM permissions
- security
Group string[]Ids - A list of security group IDs to associate with Kinesis Firehose.
- subnet
Ids string[] - A list of subnet IDs to associate with Kinesis Firehose.
- vpc
Id string
- role_
arn str - The ARN of the IAM role to be assumed by Firehose for calling the Amazon EC2 configuration API and for creating network interfaces. Make sure role has necessary IAM permissions
- security_
group_ Sequence[str]ids - A list of security group IDs to associate with Kinesis Firehose.
- subnet_
ids Sequence[str] - A list of subnet IDs to associate with Kinesis Firehose.
- vpc_
id str
- role
Arn String - The ARN of the IAM role to be assumed by Firehose for calling the Amazon EC2 configuration API and for creating network interfaces. Make sure role has necessary IAM permissions
- security
Group List<String>Ids - A list of security group IDs to associate with Kinesis Firehose.
- subnet
Ids List<String> - A list of subnet IDs to associate with Kinesis Firehose.
- vpc
Id String
FirehoseDeliveryStreamRedshiftConfiguration, FirehoseDeliveryStreamRedshiftConfigurationArgs
- Cluster
Jdbcurl string - The jdbcurl of the redshift cluster.
- Data
Table stringName - The name of the table in the redshift cluster that the s3 bucket will copy to.
- Password string
- The password for the username above.
- Role
Arn string - The arn of the role the stream assumes.
- Username string
- The username that the firehose delivery stream will assume. It is strongly recommended that the username and password provided is used exclusively for Amazon Kinesis Firehose purposes, and that the permissions for the account are restricted for Amazon Redshift INSERT permissions.
- Cloudwatch
Logging FirehoseOptions Delivery Stream Redshift Configuration Cloudwatch Logging Options - The CloudWatch Logging Options for the delivery stream. More details are given below
- Copy
Options string - Copy options for copying the data from the s3 intermediate bucket into redshift, for example to change the default delimiter. For valid values, see the AWS documentation
- Data
Table stringColumns - The data table columns that will be targeted by the copy command.
- Processing
Configuration FirehoseDelivery Stream Redshift Configuration Processing Configuration - The data processing configuration. More details are given below.
- Retry
Duration int - The length of time during which Firehose retries delivery after a failure, starting from the initial request and including the first attempt. The default value is 3600 seconds (60 minutes). Firehose does not retry if the value of DurationInSeconds is 0 (zero) or if the first delivery attempt takes longer than the current value.
- S3Backup
Configuration FirehoseDelivery Stream Redshift Configuration S3Backup Configuration - The configuration for backup in Amazon S3. Required if
s3_backup_modeisEnabled. Supports the same fields ass3_configurationobject. - S3Backup
Mode string - The Amazon S3 backup mode. Valid values are
DisabledandEnabled. Default value isDisabled.
- Cluster
Jdbcurl string - The jdbcurl of the redshift cluster.
- Data
Table stringName - The name of the table in the redshift cluster that the s3 bucket will copy to.
- Password string
- The password for the username above.
- Role
Arn string - The arn of the role the stream assumes.
- Username string
- The username that the firehose delivery stream will assume. It is strongly recommended that the username and password provided is used exclusively for Amazon Kinesis Firehose purposes, and that the permissions for the account are restricted for Amazon Redshift INSERT permissions.
- Cloudwatch
Logging FirehoseOptions Delivery Stream Redshift Configuration Cloudwatch Logging Options - The CloudWatch Logging Options for the delivery stream. More details are given below
- Copy
Options string - Copy options for copying the data from the s3 intermediate bucket into redshift, for example to change the default delimiter. For valid values, see the AWS documentation
- Data
Table stringColumns - The data table columns that will be targeted by the copy command.
- Processing
Configuration FirehoseDelivery Stream Redshift Configuration Processing Configuration - The data processing configuration. More details are given below.
- Retry
Duration int - The length of time during which Firehose retries delivery after a failure, starting from the initial request and including the first attempt. The default value is 3600 seconds (60 minutes). Firehose does not retry if the value of DurationInSeconds is 0 (zero) or if the first delivery attempt takes longer than the current value.
- S3Backup
Configuration FirehoseDelivery Stream Redshift Configuration S3Backup Configuration - The configuration for backup in Amazon S3. Required if
s3_backup_modeisEnabled. Supports the same fields ass3_configurationobject. - S3Backup
Mode string - The Amazon S3 backup mode. Valid values are
DisabledandEnabled. Default value isDisabled.
- cluster
Jdbcurl String - The jdbcurl of the redshift cluster.
- data
Table StringName - The name of the table in the redshift cluster that the s3 bucket will copy to.
- password String
- The password for the username above.
- role
Arn String - The arn of the role the stream assumes.
- username String
- The username that the firehose delivery stream will assume. It is strongly recommended that the username and password provided is used exclusively for Amazon Kinesis Firehose purposes, and that the permissions for the account are restricted for Amazon Redshift INSERT permissions.
- cloudwatch
Logging FirehoseOptions Delivery Stream Redshift Configuration Cloudwatch Logging Options - The CloudWatch Logging Options for the delivery stream. More details are given below
- copy
Options String - Copy options for copying the data from the s3 intermediate bucket into redshift, for example to change the default delimiter. For valid values, see the AWS documentation
- data
Table StringColumns - The data table columns that will be targeted by the copy command.
- processing
Configuration FirehoseDelivery Stream Redshift Configuration Processing Configuration - The data processing configuration. More details are given below.
- retry
Duration Integer - The length of time during which Firehose retries delivery after a failure, starting from the initial request and including the first attempt. The default value is 3600 seconds (60 minutes). Firehose does not retry if the value of DurationInSeconds is 0 (zero) or if the first delivery attempt takes longer than the current value.
- s3Backup
Configuration FirehoseDelivery Stream Redshift Configuration S3Backup Configuration - The configuration for backup in Amazon S3. Required if
s3_backup_modeisEnabled. Supports the same fields ass3_configurationobject. - s3Backup
Mode String - The Amazon S3 backup mode. Valid values are
DisabledandEnabled. Default value isDisabled.
- cluster
Jdbcurl string - The jdbcurl of the redshift cluster.
- data
Table stringName - The name of the table in the redshift cluster that the s3 bucket will copy to.
- password string
- The password for the username above.
- role
Arn string - The arn of the role the stream assumes.
- username string
- The username that the firehose delivery stream will assume. It is strongly recommended that the username and password provided is used exclusively for Amazon Kinesis Firehose purposes, and that the permissions for the account are restricted for Amazon Redshift INSERT permissions.
- cloudwatch
Logging FirehoseOptions Delivery Stream Redshift Configuration Cloudwatch Logging Options - The CloudWatch Logging Options for the delivery stream. More details are given below
- copy
Options string - Copy options for copying the data from the s3 intermediate bucket into redshift, for example to change the default delimiter. For valid values, see the AWS documentation
- data
Table stringColumns - The data table columns that will be targeted by the copy command.
- processing
Configuration FirehoseDelivery Stream Redshift Configuration Processing Configuration - The data processing configuration. More details are given below.
- retry
Duration number - The length of time during which Firehose retries delivery after a failure, starting from the initial request and including the first attempt. The default value is 3600 seconds (60 minutes). Firehose does not retry if the value of DurationInSeconds is 0 (zero) or if the first delivery attempt takes longer than the current value.
- s3Backup
Configuration FirehoseDelivery Stream Redshift Configuration S3Backup Configuration - The configuration for backup in Amazon S3. Required if
s3_backup_modeisEnabled. Supports the same fields ass3_configurationobject. - s3Backup
Mode string - The Amazon S3 backup mode. Valid values are
DisabledandEnabled. Default value isDisabled.
- cluster_
jdbcurl str - The jdbcurl of the redshift cluster.
- data_
table_ strname - The name of the table in the redshift cluster that the s3 bucket will copy to.
- password str
- The password for the username above.
- role_
arn str - The arn of the role the stream assumes.
- username str
- The username that the firehose delivery stream will assume. It is strongly recommended that the username and password provided is used exclusively for Amazon Kinesis Firehose purposes, and that the permissions for the account are restricted for Amazon Redshift INSERT permissions.
- cloudwatch_
logging_ Firehoseoptions Delivery Stream Redshift Configuration Cloudwatch Logging Options - The CloudWatch Logging Options for the delivery stream. More details are given below
- copy_
options str - Copy options for copying the data from the s3 intermediate bucket into redshift, for example to change the default delimiter. For valid values, see the AWS documentation
- data_
table_ strcolumns - The data table columns that will be targeted by the copy command.
- processing_
configuration FirehoseDelivery Stream Redshift Configuration Processing Configuration - The data processing configuration. More details are given below.
- retry_
duration int - The length of time during which Firehose retries delivery after a failure, starting from the initial request and including the first attempt. The default value is 3600 seconds (60 minutes). Firehose does not retry if the value of DurationInSeconds is 0 (zero) or if the first delivery attempt takes longer than the current value.
- s3_
backup_ Firehoseconfiguration Delivery Stream Redshift Configuration S3Backup Configuration - The configuration for backup in Amazon S3. Required if
s3_backup_modeisEnabled. Supports the same fields ass3_configurationobject. - s3_
backup_ strmode - The Amazon S3 backup mode. Valid values are
DisabledandEnabled. Default value isDisabled.
- cluster
Jdbcurl String - The jdbcurl of the redshift cluster.
- data
Table StringName - The name of the table in the redshift cluster that the s3 bucket will copy to.
- password String
- The password for the username above.
- role
Arn String - The arn of the role the stream assumes.
- username String
- The username that the firehose delivery stream will assume. It is strongly recommended that the username and password provided is used exclusively for Amazon Kinesis Firehose purposes, and that the permissions for the account are restricted for Amazon Redshift INSERT permissions.
- cloudwatch
Logging Property MapOptions - The CloudWatch Logging Options for the delivery stream. More details are given below
- copy
Options String - Copy options for copying the data from the s3 intermediate bucket into redshift, for example to change the default delimiter. For valid values, see the AWS documentation
- data
Table StringColumns - The data table columns that will be targeted by the copy command.
- processing
Configuration Property Map - The data processing configuration. More details are given below.
- retry
Duration Number - The length of time during which Firehose retries delivery after a failure, starting from the initial request and including the first attempt. The default value is 3600 seconds (60 minutes). Firehose does not retry if the value of DurationInSeconds is 0 (zero) or if the first delivery attempt takes longer than the current value.
- s3Backup
Configuration Property Map - The configuration for backup in Amazon S3. Required if
s3_backup_modeisEnabled. Supports the same fields ass3_configurationobject. - s3Backup
Mode String - The Amazon S3 backup mode. Valid values are
DisabledandEnabled. Default value isDisabled.
FirehoseDeliveryStreamRedshiftConfigurationCloudwatchLoggingOptions, FirehoseDeliveryStreamRedshiftConfigurationCloudwatchLoggingOptionsArgs
- Enabled bool
- Enables or disables the logging. Defaults to
false. - Log
Group stringName - The CloudWatch group name for logging. This value is required if
enabledis true. - Log
Stream stringName - The CloudWatch log stream name for logging. This value is required if
enabledis true.
- Enabled bool
- Enables or disables the logging. Defaults to
false. - Log
Group stringName - The CloudWatch group name for logging. This value is required if
enabledis true. - Log
Stream stringName - The CloudWatch log stream name for logging. This value is required if
enabledis true.
- enabled Boolean
- Enables or disables the logging. Defaults to
false. - log
Group StringName - The CloudWatch group name for logging. This value is required if
enabledis true. - log
Stream StringName - The CloudWatch log stream name for logging. This value is required if
enabledis true.
- enabled boolean
- Enables or disables the logging. Defaults to
false. - log
Group stringName - The CloudWatch group name for logging. This value is required if
enabledis true. - log
Stream stringName - The CloudWatch log stream name for logging. This value is required if
enabledis true.
- enabled bool
- Enables or disables the logging. Defaults to
false. - log_
group_ strname - The CloudWatch group name for logging. This value is required if
enabledis true. - log_
stream_ strname - The CloudWatch log stream name for logging. This value is required if
enabledis true.
- enabled Boolean
- Enables or disables the logging. Defaults to
false. - log
Group StringName - The CloudWatch group name for logging. This value is required if
enabledis true. - log
Stream StringName - The CloudWatch log stream name for logging. This value is required if
enabledis true.
FirehoseDeliveryStreamRedshiftConfigurationProcessingConfiguration, FirehoseDeliveryStreamRedshiftConfigurationProcessingConfigurationArgs
- Enabled bool
- Enables or disables data processing.
- Processors
List<Firehose
Delivery Stream Redshift Configuration Processing Configuration Processor> - Array of data processors. More details are given below
- Enabled bool
- Enables or disables data processing.
- Processors
[]Firehose
Delivery Stream Redshift Configuration Processing Configuration Processor - Array of data processors. More details are given below
- enabled Boolean
- Enables or disables data processing.
- processors
List<Firehose
Delivery Stream Redshift Configuration Processing Configuration Processor> - Array of data processors. More details are given below
- enabled boolean
- Enables or disables data processing.
- processors
Firehose
Delivery Stream Redshift Configuration Processing Configuration Processor[] - Array of data processors. More details are given below
- enabled bool
- Enables or disables data processing.
- processors
Sequence[Firehose
Delivery Stream Redshift Configuration Processing Configuration Processor] - Array of data processors. More details are given below
- enabled Boolean
- Enables or disables data processing.
- processors List<Property Map>
- Array of data processors. More details are given below
FirehoseDeliveryStreamRedshiftConfigurationProcessingConfigurationProcessor, FirehoseDeliveryStreamRedshiftConfigurationProcessingConfigurationProcessorArgs
- Type string
- The type of processor. Valid Values:
RecordDeAggregation,Lambda,MetadataExtraction,AppendDelimiterToRecord. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - Parameters
List<Firehose
Delivery Stream Redshift Configuration Processing Configuration Processor Parameter> - Array of processor parameters. More details are given below
- Type string
- The type of processor. Valid Values:
RecordDeAggregation,Lambda,MetadataExtraction,AppendDelimiterToRecord. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - Parameters
[]Firehose
Delivery Stream Redshift Configuration Processing Configuration Processor Parameter - Array of processor parameters. More details are given below
- type String
- The type of processor. Valid Values:
RecordDeAggregation,Lambda,MetadataExtraction,AppendDelimiterToRecord. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - parameters
List<Firehose
Delivery Stream Redshift Configuration Processing Configuration Processor Parameter> - Array of processor parameters. More details are given below
- type string
- The type of processor. Valid Values:
RecordDeAggregation,Lambda,MetadataExtraction,AppendDelimiterToRecord. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - parameters
Firehose
Delivery Stream Redshift Configuration Processing Configuration Processor Parameter[] - Array of processor parameters. More details are given below
- type str
- The type of processor. Valid Values:
RecordDeAggregation,Lambda,MetadataExtraction,AppendDelimiterToRecord. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - parameters
Sequence[Firehose
Delivery Stream Redshift Configuration Processing Configuration Processor Parameter] - Array of processor parameters. More details are given below
- type String
- The type of processor. Valid Values:
RecordDeAggregation,Lambda,MetadataExtraction,AppendDelimiterToRecord. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - parameters List<Property Map>
- Array of processor parameters. More details are given below
FirehoseDeliveryStreamRedshiftConfigurationProcessingConfigurationProcessorParameter, FirehoseDeliveryStreamRedshiftConfigurationProcessingConfigurationProcessorParameterArgs
- Parameter
Name string - Parameter name. Valid Values:
LambdaArn,NumberOfRetries,MetadataExtractionQuery,JsonParsingEngine,RoleArn,BufferSizeInMBs,BufferIntervalInSeconds,SubRecordType,Delimiter. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - Parameter
Value string Parameter value. Must be between 1 and 512 length (inclusive). When providing a Lambda ARN, you should specify the resource version as well.
NOTE: Parameters with default values, including
NumberOfRetries(default: 3),RoleArn(default: firehose role ARN),BufferSizeInMBs(default: 3), andBufferIntervalInSeconds(default: 60), are not stored in state. To prevent perpetual differences, it is therefore recommended to only include parameters with non-default values.
- Parameter
Name string - Parameter name. Valid Values:
LambdaArn,NumberOfRetries,MetadataExtractionQuery,JsonParsingEngine,RoleArn,BufferSizeInMBs,BufferIntervalInSeconds,SubRecordType,Delimiter. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - Parameter
Value string Parameter value. Must be between 1 and 512 length (inclusive). When providing a Lambda ARN, you should specify the resource version as well.
NOTE: Parameters with default values, including
NumberOfRetries(default: 3),RoleArn(default: firehose role ARN),BufferSizeInMBs(default: 3), andBufferIntervalInSeconds(default: 60), are not stored in state. To prevent perpetual differences, it is therefore recommended to only include parameters with non-default values.
- parameter
Name String - Parameter name. Valid Values:
LambdaArn,NumberOfRetries,MetadataExtractionQuery,JsonParsingEngine,RoleArn,BufferSizeInMBs,BufferIntervalInSeconds,SubRecordType,Delimiter. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - parameter
Value String Parameter value. Must be between 1 and 512 length (inclusive). When providing a Lambda ARN, you should specify the resource version as well.
NOTE: Parameters with default values, including
NumberOfRetries(default: 3),RoleArn(default: firehose role ARN),BufferSizeInMBs(default: 3), andBufferIntervalInSeconds(default: 60), are not stored in state. To prevent perpetual differences, it is therefore recommended to only include parameters with non-default values.
- parameter
Name string - Parameter name. Valid Values:
LambdaArn,NumberOfRetries,MetadataExtractionQuery,JsonParsingEngine,RoleArn,BufferSizeInMBs,BufferIntervalInSeconds,SubRecordType,Delimiter. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - parameter
Value string Parameter value. Must be between 1 and 512 length (inclusive). When providing a Lambda ARN, you should specify the resource version as well.
NOTE: Parameters with default values, including
NumberOfRetries(default: 3),RoleArn(default: firehose role ARN),BufferSizeInMBs(default: 3), andBufferIntervalInSeconds(default: 60), are not stored in state. To prevent perpetual differences, it is therefore recommended to only include parameters with non-default values.
- parameter_
name str - Parameter name. Valid Values:
LambdaArn,NumberOfRetries,MetadataExtractionQuery,JsonParsingEngine,RoleArn,BufferSizeInMBs,BufferIntervalInSeconds,SubRecordType,Delimiter. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - parameter_
value str Parameter value. Must be between 1 and 512 length (inclusive). When providing a Lambda ARN, you should specify the resource version as well.
NOTE: Parameters with default values, including
NumberOfRetries(default: 3),RoleArn(default: firehose role ARN),BufferSizeInMBs(default: 3), andBufferIntervalInSeconds(default: 60), are not stored in state. To prevent perpetual differences, it is therefore recommended to only include parameters with non-default values.
- parameter
Name String - Parameter name. Valid Values:
LambdaArn,NumberOfRetries,MetadataExtractionQuery,JsonParsingEngine,RoleArn,BufferSizeInMBs,BufferIntervalInSeconds,SubRecordType,Delimiter. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - parameter
Value String Parameter value. Must be between 1 and 512 length (inclusive). When providing a Lambda ARN, you should specify the resource version as well.
NOTE: Parameters with default values, including
NumberOfRetries(default: 3),RoleArn(default: firehose role ARN),BufferSizeInMBs(default: 3), andBufferIntervalInSeconds(default: 60), are not stored in state. To prevent perpetual differences, it is therefore recommended to only include parameters with non-default values.
FirehoseDeliveryStreamRedshiftConfigurationS3BackupConfiguration, FirehoseDeliveryStreamRedshiftConfigurationS3BackupConfigurationArgs
- Bucket
Arn string - The ARN of the S3 bucket
- Role
Arn string - The ARN of the role that provides access to the source Kinesis stream.
- Buffer
Interval int - Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination. The default value is 300.
- Buffer
Size int - Buffer incoming data to the specified size, in MBs, before delivering it to the destination. The default value is 5. We recommend setting SizeInMBs to a value greater than the amount of data you typically ingest into the delivery stream in 10 seconds. For example, if you typically ingest data at 1 MB/sec set SizeInMBs to be 10 MB or higher.
- Cloudwatch
Logging FirehoseOptions Delivery Stream Redshift Configuration S3Backup Configuration Cloudwatch Logging Options The CloudWatch Logging Options for the delivery stream. More details are given below
The
extended_s3_configurationobject supports the same fields froms3_configurationas well as the following:- Compression
Format string - The compression format. If no value is specified, the default is
UNCOMPRESSED. Other supported values areGZIP,ZIP,Snappy, &HADOOP_SNAPPY. - Error
Output stringPrefix - Prefix added to failed records before writing them to S3. Not currently supported for
redshiftdestination. This prefix appears immediately following the bucket name. For information about how to specify this prefix, see Custom Prefixes for Amazon S3 Objects. - Kms
Key stringArn - Specifies the KMS key ARN the stream will use to encrypt data. If not set, no encryption will be used.
- Prefix string
- The "YYYY/MM/DD/HH" time format prefix is automatically used for delivered S3 files. You can specify an extra prefix to be added in front of the time format prefix. Note that if the prefix ends with a slash, it appears as a folder in the S3 bucket
- Bucket
Arn string - The ARN of the S3 bucket
- Role
Arn string - The ARN of the role that provides access to the source Kinesis stream.
- Buffer
Interval int - Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination. The default value is 300.
- Buffer
Size int - Buffer incoming data to the specified size, in MBs, before delivering it to the destination. The default value is 5. We recommend setting SizeInMBs to a value greater than the amount of data you typically ingest into the delivery stream in 10 seconds. For example, if you typically ingest data at 1 MB/sec set SizeInMBs to be 10 MB or higher.
- Cloudwatch
Logging FirehoseOptions Delivery Stream Redshift Configuration S3Backup Configuration Cloudwatch Logging Options The CloudWatch Logging Options for the delivery stream. More details are given below
The
extended_s3_configurationobject supports the same fields froms3_configurationas well as the following:- Compression
Format string - The compression format. If no value is specified, the default is
UNCOMPRESSED. Other supported values areGZIP,ZIP,Snappy, &HADOOP_SNAPPY. - Error
Output stringPrefix - Prefix added to failed records before writing them to S3. Not currently supported for
redshiftdestination. This prefix appears immediately following the bucket name. For information about how to specify this prefix, see Custom Prefixes for Amazon S3 Objects. - Kms
Key stringArn - Specifies the KMS key ARN the stream will use to encrypt data. If not set, no encryption will be used.
- Prefix string
- The "YYYY/MM/DD/HH" time format prefix is automatically used for delivered S3 files. You can specify an extra prefix to be added in front of the time format prefix. Note that if the prefix ends with a slash, it appears as a folder in the S3 bucket
- bucket
Arn String - The ARN of the S3 bucket
- role
Arn String - The ARN of the role that provides access to the source Kinesis stream.
- buffer
Interval Integer - Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination. The default value is 300.
- buffer
Size Integer - Buffer incoming data to the specified size, in MBs, before delivering it to the destination. The default value is 5. We recommend setting SizeInMBs to a value greater than the amount of data you typically ingest into the delivery stream in 10 seconds. For example, if you typically ingest data at 1 MB/sec set SizeInMBs to be 10 MB or higher.
- cloudwatch
Logging FirehoseOptions Delivery Stream Redshift Configuration S3Backup Configuration Cloudwatch Logging Options The CloudWatch Logging Options for the delivery stream. More details are given below
The
extended_s3_configurationobject supports the same fields froms3_configurationas well as the following:- compression
Format String - The compression format. If no value is specified, the default is
UNCOMPRESSED. Other supported values areGZIP,ZIP,Snappy, &HADOOP_SNAPPY. - error
Output StringPrefix - Prefix added to failed records before writing them to S3. Not currently supported for
redshiftdestination. This prefix appears immediately following the bucket name. For information about how to specify this prefix, see Custom Prefixes for Amazon S3 Objects. - kms
Key StringArn - Specifies the KMS key ARN the stream will use to encrypt data. If not set, no encryption will be used.
- prefix String
- The "YYYY/MM/DD/HH" time format prefix is automatically used for delivered S3 files. You can specify an extra prefix to be added in front of the time format prefix. Note that if the prefix ends with a slash, it appears as a folder in the S3 bucket
- bucket
Arn string - The ARN of the S3 bucket
- role
Arn string - The ARN of the role that provides access to the source Kinesis stream.
- buffer
Interval number - Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination. The default value is 300.
- buffer
Size number - Buffer incoming data to the specified size, in MBs, before delivering it to the destination. The default value is 5. We recommend setting SizeInMBs to a value greater than the amount of data you typically ingest into the delivery stream in 10 seconds. For example, if you typically ingest data at 1 MB/sec set SizeInMBs to be 10 MB or higher.
- cloudwatch
Logging FirehoseOptions Delivery Stream Redshift Configuration S3Backup Configuration Cloudwatch Logging Options The CloudWatch Logging Options for the delivery stream. More details are given below
The
extended_s3_configurationobject supports the same fields froms3_configurationas well as the following:- compression
Format string - The compression format. If no value is specified, the default is
UNCOMPRESSED. Other supported values areGZIP,ZIP,Snappy, &HADOOP_SNAPPY. - error
Output stringPrefix - Prefix added to failed records before writing them to S3. Not currently supported for
redshiftdestination. This prefix appears immediately following the bucket name. For information about how to specify this prefix, see Custom Prefixes for Amazon S3 Objects. - kms
Key stringArn - Specifies the KMS key ARN the stream will use to encrypt data. If not set, no encryption will be used.
- prefix string
- The "YYYY/MM/DD/HH" time format prefix is automatically used for delivered S3 files. You can specify an extra prefix to be added in front of the time format prefix. Note that if the prefix ends with a slash, it appears as a folder in the S3 bucket
- bucket_
arn str - The ARN of the S3 bucket
- role_
arn str - The ARN of the role that provides access to the source Kinesis stream.
- buffer_
interval int - Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination. The default value is 300.
- buffer_
size int - Buffer incoming data to the specified size, in MBs, before delivering it to the destination. The default value is 5. We recommend setting SizeInMBs to a value greater than the amount of data you typically ingest into the delivery stream in 10 seconds. For example, if you typically ingest data at 1 MB/sec set SizeInMBs to be 10 MB or higher.
- cloudwatch_
logging_ Firehoseoptions Delivery Stream Redshift Configuration S3Backup Configuration Cloudwatch Logging Options The CloudWatch Logging Options for the delivery stream. More details are given below
The
extended_s3_configurationobject supports the same fields froms3_configurationas well as the following:- compression_
format str - The compression format. If no value is specified, the default is
UNCOMPRESSED. Other supported values areGZIP,ZIP,Snappy, &HADOOP_SNAPPY. - error_
output_ strprefix - Prefix added to failed records before writing them to S3. Not currently supported for
redshiftdestination. This prefix appears immediately following the bucket name. For information about how to specify this prefix, see Custom Prefixes for Amazon S3 Objects. - kms_
key_ strarn - Specifies the KMS key ARN the stream will use to encrypt data. If not set, no encryption will be used.
- prefix str
- The "YYYY/MM/DD/HH" time format prefix is automatically used for delivered S3 files. You can specify an extra prefix to be added in front of the time format prefix. Note that if the prefix ends with a slash, it appears as a folder in the S3 bucket
- bucket
Arn String - The ARN of the S3 bucket
- role
Arn String - The ARN of the role that provides access to the source Kinesis stream.
- buffer
Interval Number - Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination. The default value is 300.
- buffer
Size Number - Buffer incoming data to the specified size, in MBs, before delivering it to the destination. The default value is 5. We recommend setting SizeInMBs to a value greater than the amount of data you typically ingest into the delivery stream in 10 seconds. For example, if you typically ingest data at 1 MB/sec set SizeInMBs to be 10 MB or higher.
- cloudwatch
Logging Property MapOptions The CloudWatch Logging Options for the delivery stream. More details are given below
The
extended_s3_configurationobject supports the same fields froms3_configurationas well as the following:- compression
Format String - The compression format. If no value is specified, the default is
UNCOMPRESSED. Other supported values areGZIP,ZIP,Snappy, &HADOOP_SNAPPY. - error
Output StringPrefix - Prefix added to failed records before writing them to S3. Not currently supported for
redshiftdestination. This prefix appears immediately following the bucket name. For information about how to specify this prefix, see Custom Prefixes for Amazon S3 Objects. - kms
Key StringArn - Specifies the KMS key ARN the stream will use to encrypt data. If not set, no encryption will be used.
- prefix String
- The "YYYY/MM/DD/HH" time format prefix is automatically used for delivered S3 files. You can specify an extra prefix to be added in front of the time format prefix. Note that if the prefix ends with a slash, it appears as a folder in the S3 bucket
FirehoseDeliveryStreamRedshiftConfigurationS3BackupConfigurationCloudwatchLoggingOptions, FirehoseDeliveryStreamRedshiftConfigurationS3BackupConfigurationCloudwatchLoggingOptionsArgs
- Enabled bool
- Enables or disables the logging. Defaults to
false. - Log
Group stringName - The CloudWatch group name for logging. This value is required if
enabledis true. - Log
Stream stringName - The CloudWatch log stream name for logging. This value is required if
enabledis true.
- Enabled bool
- Enables or disables the logging. Defaults to
false. - Log
Group stringName - The CloudWatch group name for logging. This value is required if
enabledis true. - Log
Stream stringName - The CloudWatch log stream name for logging. This value is required if
enabledis true.
- enabled Boolean
- Enables or disables the logging. Defaults to
false. - log
Group StringName - The CloudWatch group name for logging. This value is required if
enabledis true. - log
Stream StringName - The CloudWatch log stream name for logging. This value is required if
enabledis true.
- enabled boolean
- Enables or disables the logging. Defaults to
false. - log
Group stringName - The CloudWatch group name for logging. This value is required if
enabledis true. - log
Stream stringName - The CloudWatch log stream name for logging. This value is required if
enabledis true.
- enabled bool
- Enables or disables the logging. Defaults to
false. - log_
group_ strname - The CloudWatch group name for logging. This value is required if
enabledis true. - log_
stream_ strname - The CloudWatch log stream name for logging. This value is required if
enabledis true.
- enabled Boolean
- Enables or disables the logging. Defaults to
false. - log
Group StringName - The CloudWatch group name for logging. This value is required if
enabledis true. - log
Stream StringName - The CloudWatch log stream name for logging. This value is required if
enabledis true.
FirehoseDeliveryStreamS3Configuration, FirehoseDeliveryStreamS3ConfigurationArgs
- Bucket
Arn string - The ARN of the S3 bucket
- Role
Arn string - The ARN of the role that provides access to the source Kinesis stream.
- Buffer
Interval int - Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination. The default value is 300.
- Buffer
Size int - Buffer incoming data to the specified size, in MBs, before delivering it to the destination. The default value is 5. We recommend setting SizeInMBs to a value greater than the amount of data you typically ingest into the delivery stream in 10 seconds. For example, if you typically ingest data at 1 MB/sec set SizeInMBs to be 10 MB or higher.
- Cloudwatch
Logging FirehoseOptions Delivery Stream S3Configuration Cloudwatch Logging Options The CloudWatch Logging Options for the delivery stream. More details are given below
The
extended_s3_configurationobject supports the same fields froms3_configurationas well as the following:- Compression
Format string - The compression format. If no value is specified, the default is
UNCOMPRESSED. Other supported values areGZIP,ZIP,Snappy, &HADOOP_SNAPPY. - Error
Output stringPrefix - Prefix added to failed records before writing them to S3. Not currently supported for
redshiftdestination. This prefix appears immediately following the bucket name. For information about how to specify this prefix, see Custom Prefixes for Amazon S3 Objects. - Kms
Key stringArn - Specifies the KMS key ARN the stream will use to encrypt data. If not set, no encryption will be used.
- Prefix string
- The "YYYY/MM/DD/HH" time format prefix is automatically used for delivered S3 files. You can specify an extra prefix to be added in front of the time format prefix. Note that if the prefix ends with a slash, it appears as a folder in the S3 bucket
- Bucket
Arn string - The ARN of the S3 bucket
- Role
Arn string - The ARN of the role that provides access to the source Kinesis stream.
- Buffer
Interval int - Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination. The default value is 300.
- Buffer
Size int - Buffer incoming data to the specified size, in MBs, before delivering it to the destination. The default value is 5. We recommend setting SizeInMBs to a value greater than the amount of data you typically ingest into the delivery stream in 10 seconds. For example, if you typically ingest data at 1 MB/sec set SizeInMBs to be 10 MB or higher.
- Cloudwatch
Logging FirehoseOptions Delivery Stream S3Configuration Cloudwatch Logging Options The CloudWatch Logging Options for the delivery stream. More details are given below
The
extended_s3_configurationobject supports the same fields froms3_configurationas well as the following:- Compression
Format string - The compression format. If no value is specified, the default is
UNCOMPRESSED. Other supported values areGZIP,ZIP,Snappy, &HADOOP_SNAPPY. - Error
Output stringPrefix - Prefix added to failed records before writing them to S3. Not currently supported for
redshiftdestination. This prefix appears immediately following the bucket name. For information about how to specify this prefix, see Custom Prefixes for Amazon S3 Objects. - Kms
Key stringArn - Specifies the KMS key ARN the stream will use to encrypt data. If not set, no encryption will be used.
- Prefix string
- The "YYYY/MM/DD/HH" time format prefix is automatically used for delivered S3 files. You can specify an extra prefix to be added in front of the time format prefix. Note that if the prefix ends with a slash, it appears as a folder in the S3 bucket
- bucket
Arn String - The ARN of the S3 bucket
- role
Arn String - The ARN of the role that provides access to the source Kinesis stream.
- buffer
Interval Integer - Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination. The default value is 300.
- buffer
Size Integer - Buffer incoming data to the specified size, in MBs, before delivering it to the destination. The default value is 5. We recommend setting SizeInMBs to a value greater than the amount of data you typically ingest into the delivery stream in 10 seconds. For example, if you typically ingest data at 1 MB/sec set SizeInMBs to be 10 MB or higher.
- cloudwatch
Logging FirehoseOptions Delivery Stream S3Configuration Cloudwatch Logging Options The CloudWatch Logging Options for the delivery stream. More details are given below
The
extended_s3_configurationobject supports the same fields froms3_configurationas well as the following:- compression
Format String - The compression format. If no value is specified, the default is
UNCOMPRESSED. Other supported values areGZIP,ZIP,Snappy, &HADOOP_SNAPPY. - error
Output StringPrefix - Prefix added to failed records before writing them to S3. Not currently supported for
redshiftdestination. This prefix appears immediately following the bucket name. For information about how to specify this prefix, see Custom Prefixes for Amazon S3 Objects. - kms
Key StringArn - Specifies the KMS key ARN the stream will use to encrypt data. If not set, no encryption will be used.
- prefix String
- The "YYYY/MM/DD/HH" time format prefix is automatically used for delivered S3 files. You can specify an extra prefix to be added in front of the time format prefix. Note that if the prefix ends with a slash, it appears as a folder in the S3 bucket
- bucket
Arn string - The ARN of the S3 bucket
- role
Arn string - The ARN of the role that provides access to the source Kinesis stream.
- buffer
Interval number - Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination. The default value is 300.
- buffer
Size number - Buffer incoming data to the specified size, in MBs, before delivering it to the destination. The default value is 5. We recommend setting SizeInMBs to a value greater than the amount of data you typically ingest into the delivery stream in 10 seconds. For example, if you typically ingest data at 1 MB/sec set SizeInMBs to be 10 MB or higher.
- cloudwatch
Logging FirehoseOptions Delivery Stream S3Configuration Cloudwatch Logging Options The CloudWatch Logging Options for the delivery stream. More details are given below
The
extended_s3_configurationobject supports the same fields froms3_configurationas well as the following:- compression
Format string - The compression format. If no value is specified, the default is
UNCOMPRESSED. Other supported values areGZIP,ZIP,Snappy, &HADOOP_SNAPPY. - error
Output stringPrefix - Prefix added to failed records before writing them to S3. Not currently supported for
redshiftdestination. This prefix appears immediately following the bucket name. For information about how to specify this prefix, see Custom Prefixes for Amazon S3 Objects. - kms
Key stringArn - Specifies the KMS key ARN the stream will use to encrypt data. If not set, no encryption will be used.
- prefix string
- The "YYYY/MM/DD/HH" time format prefix is automatically used for delivered S3 files. You can specify an extra prefix to be added in front of the time format prefix. Note that if the prefix ends with a slash, it appears as a folder in the S3 bucket
- bucket_
arn str - The ARN of the S3 bucket
- role_
arn str - The ARN of the role that provides access to the source Kinesis stream.
- buffer_
interval int - Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination. The default value is 300.
- buffer_
size int - Buffer incoming data to the specified size, in MBs, before delivering it to the destination. The default value is 5. We recommend setting SizeInMBs to a value greater than the amount of data you typically ingest into the delivery stream in 10 seconds. For example, if you typically ingest data at 1 MB/sec set SizeInMBs to be 10 MB or higher.
- cloudwatch_
logging_ Firehoseoptions Delivery Stream S3Configuration Cloudwatch Logging Options The CloudWatch Logging Options for the delivery stream. More details are given below
The
extended_s3_configurationobject supports the same fields froms3_configurationas well as the following:- compression_
format str - The compression format. If no value is specified, the default is
UNCOMPRESSED. Other supported values areGZIP,ZIP,Snappy, &HADOOP_SNAPPY. - error_
output_ strprefix - Prefix added to failed records before writing them to S3. Not currently supported for
redshiftdestination. This prefix appears immediately following the bucket name. For information about how to specify this prefix, see Custom Prefixes for Amazon S3 Objects. - kms_
key_ strarn - Specifies the KMS key ARN the stream will use to encrypt data. If not set, no encryption will be used.
- prefix str
- The "YYYY/MM/DD/HH" time format prefix is automatically used for delivered S3 files. You can specify an extra prefix to be added in front of the time format prefix. Note that if the prefix ends with a slash, it appears as a folder in the S3 bucket
- bucket
Arn String - The ARN of the S3 bucket
- role
Arn String - The ARN of the role that provides access to the source Kinesis stream.
- buffer
Interval Number - Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination. The default value is 300.
- buffer
Size Number - Buffer incoming data to the specified size, in MBs, before delivering it to the destination. The default value is 5. We recommend setting SizeInMBs to a value greater than the amount of data you typically ingest into the delivery stream in 10 seconds. For example, if you typically ingest data at 1 MB/sec set SizeInMBs to be 10 MB or higher.
- cloudwatch
Logging Property MapOptions The CloudWatch Logging Options for the delivery stream. More details are given below
The
extended_s3_configurationobject supports the same fields froms3_configurationas well as the following:- compression
Format String - The compression format. If no value is specified, the default is
UNCOMPRESSED. Other supported values areGZIP,ZIP,Snappy, &HADOOP_SNAPPY. - error
Output StringPrefix - Prefix added to failed records before writing them to S3. Not currently supported for
redshiftdestination. This prefix appears immediately following the bucket name. For information about how to specify this prefix, see Custom Prefixes for Amazon S3 Objects. - kms
Key StringArn - Specifies the KMS key ARN the stream will use to encrypt data. If not set, no encryption will be used.
- prefix String
- The "YYYY/MM/DD/HH" time format prefix is automatically used for delivered S3 files. You can specify an extra prefix to be added in front of the time format prefix. Note that if the prefix ends with a slash, it appears as a folder in the S3 bucket
FirehoseDeliveryStreamS3ConfigurationCloudwatchLoggingOptions, FirehoseDeliveryStreamS3ConfigurationCloudwatchLoggingOptionsArgs
- Enabled bool
- Enables or disables the logging. Defaults to
false. - Log
Group stringName - The CloudWatch group name for logging. This value is required if
enabledis true. - Log
Stream stringName - The CloudWatch log stream name for logging. This value is required if
enabledis true.
- Enabled bool
- Enables or disables the logging. Defaults to
false. - Log
Group stringName - The CloudWatch group name for logging. This value is required if
enabledis true. - Log
Stream stringName - The CloudWatch log stream name for logging. This value is required if
enabledis true.
- enabled Boolean
- Enables or disables the logging. Defaults to
false. - log
Group StringName - The CloudWatch group name for logging. This value is required if
enabledis true. - log
Stream StringName - The CloudWatch log stream name for logging. This value is required if
enabledis true.
- enabled boolean
- Enables or disables the logging. Defaults to
false. - log
Group stringName - The CloudWatch group name for logging. This value is required if
enabledis true. - log
Stream stringName - The CloudWatch log stream name for logging. This value is required if
enabledis true.
- enabled bool
- Enables or disables the logging. Defaults to
false. - log_
group_ strname - The CloudWatch group name for logging. This value is required if
enabledis true. - log_
stream_ strname - The CloudWatch log stream name for logging. This value is required if
enabledis true.
- enabled Boolean
- Enables or disables the logging. Defaults to
false. - log
Group StringName - The CloudWatch group name for logging. This value is required if
enabledis true. - log
Stream StringName - The CloudWatch log stream name for logging. This value is required if
enabledis true.
FirehoseDeliveryStreamServerSideEncryption, FirehoseDeliveryStreamServerSideEncryptionArgs
- Enabled bool
- Whether to enable encryption at rest. Default is
false. - Key
Arn string Amazon Resource Name (ARN) of the encryption key. Required when
key_typeisCUSTOMER_MANAGED_CMK.The
s3_configurationobject supports the following:NOTE: This configuration block is deprecated for the
s3destination.- Key
Type string - Type of encryption key. Default is
AWS_OWNED_CMK. Valid values areAWS_OWNED_CMKandCUSTOMER_MANAGED_CMK
- Enabled bool
- Whether to enable encryption at rest. Default is
false. - Key
Arn string Amazon Resource Name (ARN) of the encryption key. Required when
key_typeisCUSTOMER_MANAGED_CMK.The
s3_configurationobject supports the following:NOTE: This configuration block is deprecated for the
s3destination.- Key
Type string - Type of encryption key. Default is
AWS_OWNED_CMK. Valid values areAWS_OWNED_CMKandCUSTOMER_MANAGED_CMK
- enabled Boolean
- Whether to enable encryption at rest. Default is
false. - key
Arn String Amazon Resource Name (ARN) of the encryption key. Required when
key_typeisCUSTOMER_MANAGED_CMK.The
s3_configurationobject supports the following:NOTE: This configuration block is deprecated for the
s3destination.- key
Type String - Type of encryption key. Default is
AWS_OWNED_CMK. Valid values areAWS_OWNED_CMKandCUSTOMER_MANAGED_CMK
- enabled boolean
- Whether to enable encryption at rest. Default is
false. - key
Arn string Amazon Resource Name (ARN) of the encryption key. Required when
key_typeisCUSTOMER_MANAGED_CMK.The
s3_configurationobject supports the following:NOTE: This configuration block is deprecated for the
s3destination.- key
Type string - Type of encryption key. Default is
AWS_OWNED_CMK. Valid values areAWS_OWNED_CMKandCUSTOMER_MANAGED_CMK
- enabled bool
- Whether to enable encryption at rest. Default is
false. - key_
arn str Amazon Resource Name (ARN) of the encryption key. Required when
key_typeisCUSTOMER_MANAGED_CMK.The
s3_configurationobject supports the following:NOTE: This configuration block is deprecated for the
s3destination.- key_
type str - Type of encryption key. Default is
AWS_OWNED_CMK. Valid values areAWS_OWNED_CMKandCUSTOMER_MANAGED_CMK
- enabled Boolean
- Whether to enable encryption at rest. Default is
false. - key
Arn String Amazon Resource Name (ARN) of the encryption key. Required when
key_typeisCUSTOMER_MANAGED_CMK.The
s3_configurationobject supports the following:NOTE: This configuration block is deprecated for the
s3destination.- key
Type String - Type of encryption key. Default is
AWS_OWNED_CMK. Valid values areAWS_OWNED_CMKandCUSTOMER_MANAGED_CMK
FirehoseDeliveryStreamSplunkConfiguration, FirehoseDeliveryStreamSplunkConfigurationArgs
- Hec
Endpoint string - The HTTP Event Collector (HEC) endpoint to which Kinesis Firehose sends your data.
- Hec
Token string - The GUID that you obtain from your Splunk cluster when you create a new HEC endpoint.
- Cloudwatch
Logging FirehoseOptions Delivery Stream Splunk Configuration Cloudwatch Logging Options - The CloudWatch Logging Options for the delivery stream. More details are given below.
- Hec
Acknowledgment intTimeout - The amount of time, in seconds between 180 and 600, that Kinesis Firehose waits to receive an acknowledgment from Splunk after it sends it data.
- Hec
Endpoint stringType - The HEC endpoint type. Valid values are
RaworEvent. The default value isRaw. - Processing
Configuration FirehoseDelivery Stream Splunk Configuration Processing Configuration - The data processing configuration. More details are given below.
- Retry
Duration int - After an initial failure to deliver to Splunk, the total amount of time, in seconds between 0 to 7200, during which Firehose re-attempts delivery (including the first attempt). After this time has elapsed, the failed documents are written to Amazon S3. The default value is 300s. There will be no retry if the value is 0.
- S3Backup
Mode string - Defines how documents should be delivered to Amazon S3. Valid values are
FailedEventsOnlyandAllEvents. Default value isFailedEventsOnly.
- Hec
Endpoint string - The HTTP Event Collector (HEC) endpoint to which Kinesis Firehose sends your data.
- Hec
Token string - The GUID that you obtain from your Splunk cluster when you create a new HEC endpoint.
- Cloudwatch
Logging FirehoseOptions Delivery Stream Splunk Configuration Cloudwatch Logging Options - The CloudWatch Logging Options for the delivery stream. More details are given below.
- Hec
Acknowledgment intTimeout - The amount of time, in seconds between 180 and 600, that Kinesis Firehose waits to receive an acknowledgment from Splunk after it sends it data.
- Hec
Endpoint stringType - The HEC endpoint type. Valid values are
RaworEvent. The default value isRaw. - Processing
Configuration FirehoseDelivery Stream Splunk Configuration Processing Configuration - The data processing configuration. More details are given below.
- Retry
Duration int - After an initial failure to deliver to Splunk, the total amount of time, in seconds between 0 to 7200, during which Firehose re-attempts delivery (including the first attempt). After this time has elapsed, the failed documents are written to Amazon S3. The default value is 300s. There will be no retry if the value is 0.
- S3Backup
Mode string - Defines how documents should be delivered to Amazon S3. Valid values are
FailedEventsOnlyandAllEvents. Default value isFailedEventsOnly.
- hec
Endpoint String - The HTTP Event Collector (HEC) endpoint to which Kinesis Firehose sends your data.
- hec
Token String - The GUID that you obtain from your Splunk cluster when you create a new HEC endpoint.
- cloudwatch
Logging FirehoseOptions Delivery Stream Splunk Configuration Cloudwatch Logging Options - The CloudWatch Logging Options for the delivery stream. More details are given below.
- hec
Acknowledgment IntegerTimeout - The amount of time, in seconds between 180 and 600, that Kinesis Firehose waits to receive an acknowledgment from Splunk after it sends it data.
- hec
Endpoint StringType - The HEC endpoint type. Valid values are
RaworEvent. The default value isRaw. - processing
Configuration FirehoseDelivery Stream Splunk Configuration Processing Configuration - The data processing configuration. More details are given below.
- retry
Duration Integer - After an initial failure to deliver to Splunk, the total amount of time, in seconds between 0 to 7200, during which Firehose re-attempts delivery (including the first attempt). After this time has elapsed, the failed documents are written to Amazon S3. The default value is 300s. There will be no retry if the value is 0.
- s3Backup
Mode String - Defines how documents should be delivered to Amazon S3. Valid values are
FailedEventsOnlyandAllEvents. Default value isFailedEventsOnly.
- hec
Endpoint string - The HTTP Event Collector (HEC) endpoint to which Kinesis Firehose sends your data.
- hec
Token string - The GUID that you obtain from your Splunk cluster when you create a new HEC endpoint.
- cloudwatch
Logging FirehoseOptions Delivery Stream Splunk Configuration Cloudwatch Logging Options - The CloudWatch Logging Options for the delivery stream. More details are given below.
- hec
Acknowledgment numberTimeout - The amount of time, in seconds between 180 and 600, that Kinesis Firehose waits to receive an acknowledgment from Splunk after it sends it data.
- hec
Endpoint stringType - The HEC endpoint type. Valid values are
RaworEvent. The default value isRaw. - processing
Configuration FirehoseDelivery Stream Splunk Configuration Processing Configuration - The data processing configuration. More details are given below.
- retry
Duration number - After an initial failure to deliver to Splunk, the total amount of time, in seconds between 0 to 7200, during which Firehose re-attempts delivery (including the first attempt). After this time has elapsed, the failed documents are written to Amazon S3. The default value is 300s. There will be no retry if the value is 0.
- s3Backup
Mode string - Defines how documents should be delivered to Amazon S3. Valid values are
FailedEventsOnlyandAllEvents. Default value isFailedEventsOnly.
- hec_
endpoint str - The HTTP Event Collector (HEC) endpoint to which Kinesis Firehose sends your data.
- hec_
token str - The GUID that you obtain from your Splunk cluster when you create a new HEC endpoint.
- cloudwatch_
logging_ Firehoseoptions Delivery Stream Splunk Configuration Cloudwatch Logging Options - The CloudWatch Logging Options for the delivery stream. More details are given below.
- hec_
acknowledgment_ inttimeout - The amount of time, in seconds between 180 and 600, that Kinesis Firehose waits to receive an acknowledgment from Splunk after it sends it data.
- hec_
endpoint_ strtype - The HEC endpoint type. Valid values are
RaworEvent. The default value isRaw. - processing_
configuration FirehoseDelivery Stream Splunk Configuration Processing Configuration - The data processing configuration. More details are given below.
- retry_
duration int - After an initial failure to deliver to Splunk, the total amount of time, in seconds between 0 to 7200, during which Firehose re-attempts delivery (including the first attempt). After this time has elapsed, the failed documents are written to Amazon S3. The default value is 300s. There will be no retry if the value is 0.
- s3_
backup_ strmode - Defines how documents should be delivered to Amazon S3. Valid values are
FailedEventsOnlyandAllEvents. Default value isFailedEventsOnly.
- hec
Endpoint String - The HTTP Event Collector (HEC) endpoint to which Kinesis Firehose sends your data.
- hec
Token String - The GUID that you obtain from your Splunk cluster when you create a new HEC endpoint.
- cloudwatch
Logging Property MapOptions - The CloudWatch Logging Options for the delivery stream. More details are given below.
- hec
Acknowledgment NumberTimeout - The amount of time, in seconds between 180 and 600, that Kinesis Firehose waits to receive an acknowledgment from Splunk after it sends it data.
- hec
Endpoint StringType - The HEC endpoint type. Valid values are
RaworEvent. The default value isRaw. - processing
Configuration Property Map - The data processing configuration. More details are given below.
- retry
Duration Number - After an initial failure to deliver to Splunk, the total amount of time, in seconds between 0 to 7200, during which Firehose re-attempts delivery (including the first attempt). After this time has elapsed, the failed documents are written to Amazon S3. The default value is 300s. There will be no retry if the value is 0.
- s3Backup
Mode String - Defines how documents should be delivered to Amazon S3. Valid values are
FailedEventsOnlyandAllEvents. Default value isFailedEventsOnly.
FirehoseDeliveryStreamSplunkConfigurationCloudwatchLoggingOptions, FirehoseDeliveryStreamSplunkConfigurationCloudwatchLoggingOptionsArgs
- Enabled bool
- Enables or disables the logging. Defaults to
false. - Log
Group stringName - The CloudWatch group name for logging. This value is required if
enabledis true. - Log
Stream stringName - The CloudWatch log stream name for logging. This value is required if
enabledis true.
- Enabled bool
- Enables or disables the logging. Defaults to
false. - Log
Group stringName - The CloudWatch group name for logging. This value is required if
enabledis true. - Log
Stream stringName - The CloudWatch log stream name for logging. This value is required if
enabledis true.
- enabled Boolean
- Enables or disables the logging. Defaults to
false. - log
Group StringName - The CloudWatch group name for logging. This value is required if
enabledis true. - log
Stream StringName - The CloudWatch log stream name for logging. This value is required if
enabledis true.
- enabled boolean
- Enables or disables the logging. Defaults to
false. - log
Group stringName - The CloudWatch group name for logging. This value is required if
enabledis true. - log
Stream stringName - The CloudWatch log stream name for logging. This value is required if
enabledis true.
- enabled bool
- Enables or disables the logging. Defaults to
false. - log_
group_ strname - The CloudWatch group name for logging. This value is required if
enabledis true. - log_
stream_ strname - The CloudWatch log stream name for logging. This value is required if
enabledis true.
- enabled Boolean
- Enables or disables the logging. Defaults to
false. - log
Group StringName - The CloudWatch group name for logging. This value is required if
enabledis true. - log
Stream StringName - The CloudWatch log stream name for logging. This value is required if
enabledis true.
FirehoseDeliveryStreamSplunkConfigurationProcessingConfiguration, FirehoseDeliveryStreamSplunkConfigurationProcessingConfigurationArgs
- Enabled bool
- Enables or disables data processing.
- Processors
List<Firehose
Delivery Stream Splunk Configuration Processing Configuration Processor> - Array of data processors. More details are given below
- Enabled bool
- Enables or disables data processing.
- Processors
[]Firehose
Delivery Stream Splunk Configuration Processing Configuration Processor - Array of data processors. More details are given below
- enabled Boolean
- Enables or disables data processing.
- processors
List<Firehose
Delivery Stream Splunk Configuration Processing Configuration Processor> - Array of data processors. More details are given below
- enabled boolean
- Enables or disables data processing.
- processors
Firehose
Delivery Stream Splunk Configuration Processing Configuration Processor[] - Array of data processors. More details are given below
- enabled bool
- Enables or disables data processing.
- processors
Sequence[Firehose
Delivery Stream Splunk Configuration Processing Configuration Processor] - Array of data processors. More details are given below
- enabled Boolean
- Enables or disables data processing.
- processors List<Property Map>
- Array of data processors. More details are given below
FirehoseDeliveryStreamSplunkConfigurationProcessingConfigurationProcessor, FirehoseDeliveryStreamSplunkConfigurationProcessingConfigurationProcessorArgs
- Type string
- The type of processor. Valid Values:
RecordDeAggregation,Lambda,MetadataExtraction,AppendDelimiterToRecord. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - Parameters
List<Firehose
Delivery Stream Splunk Configuration Processing Configuration Processor Parameter> - Array of processor parameters. More details are given below
- Type string
- The type of processor. Valid Values:
RecordDeAggregation,Lambda,MetadataExtraction,AppendDelimiterToRecord. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - Parameters
[]Firehose
Delivery Stream Splunk Configuration Processing Configuration Processor Parameter - Array of processor parameters. More details are given below
- type String
- The type of processor. Valid Values:
RecordDeAggregation,Lambda,MetadataExtraction,AppendDelimiterToRecord. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - parameters
List<Firehose
Delivery Stream Splunk Configuration Processing Configuration Processor Parameter> - Array of processor parameters. More details are given below
- type string
- The type of processor. Valid Values:
RecordDeAggregation,Lambda,MetadataExtraction,AppendDelimiterToRecord. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - parameters
Firehose
Delivery Stream Splunk Configuration Processing Configuration Processor Parameter[] - Array of processor parameters. More details are given below
- type str
- The type of processor. Valid Values:
RecordDeAggregation,Lambda,MetadataExtraction,AppendDelimiterToRecord. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - parameters
Sequence[Firehose
Delivery Stream Splunk Configuration Processing Configuration Processor Parameter] - Array of processor parameters. More details are given below
- type String
- The type of processor. Valid Values:
RecordDeAggregation,Lambda,MetadataExtraction,AppendDelimiterToRecord. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - parameters List<Property Map>
- Array of processor parameters. More details are given below
FirehoseDeliveryStreamSplunkConfigurationProcessingConfigurationProcessorParameter, FirehoseDeliveryStreamSplunkConfigurationProcessingConfigurationProcessorParameterArgs
- Parameter
Name string - Parameter name. Valid Values:
LambdaArn,NumberOfRetries,MetadataExtractionQuery,JsonParsingEngine,RoleArn,BufferSizeInMBs,BufferIntervalInSeconds,SubRecordType,Delimiter. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - Parameter
Value string Parameter value. Must be between 1 and 512 length (inclusive). When providing a Lambda ARN, you should specify the resource version as well.
NOTE: Parameters with default values, including
NumberOfRetries(default: 3),RoleArn(default: firehose role ARN),BufferSizeInMBs(default: 3), andBufferIntervalInSeconds(default: 60), are not stored in state. To prevent perpetual differences, it is therefore recommended to only include parameters with non-default values.
- Parameter
Name string - Parameter name. Valid Values:
LambdaArn,NumberOfRetries,MetadataExtractionQuery,JsonParsingEngine,RoleArn,BufferSizeInMBs,BufferIntervalInSeconds,SubRecordType,Delimiter. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - Parameter
Value string Parameter value. Must be between 1 and 512 length (inclusive). When providing a Lambda ARN, you should specify the resource version as well.
NOTE: Parameters with default values, including
NumberOfRetries(default: 3),RoleArn(default: firehose role ARN),BufferSizeInMBs(default: 3), andBufferIntervalInSeconds(default: 60), are not stored in state. To prevent perpetual differences, it is therefore recommended to only include parameters with non-default values.
- parameter
Name String - Parameter name. Valid Values:
LambdaArn,NumberOfRetries,MetadataExtractionQuery,JsonParsingEngine,RoleArn,BufferSizeInMBs,BufferIntervalInSeconds,SubRecordType,Delimiter. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - parameter
Value String Parameter value. Must be between 1 and 512 length (inclusive). When providing a Lambda ARN, you should specify the resource version as well.
NOTE: Parameters with default values, including
NumberOfRetries(default: 3),RoleArn(default: firehose role ARN),BufferSizeInMBs(default: 3), andBufferIntervalInSeconds(default: 60), are not stored in state. To prevent perpetual differences, it is therefore recommended to only include parameters with non-default values.
- parameter
Name string - Parameter name. Valid Values:
LambdaArn,NumberOfRetries,MetadataExtractionQuery,JsonParsingEngine,RoleArn,BufferSizeInMBs,BufferIntervalInSeconds,SubRecordType,Delimiter. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - parameter
Value string Parameter value. Must be between 1 and 512 length (inclusive). When providing a Lambda ARN, you should specify the resource version as well.
NOTE: Parameters with default values, including
NumberOfRetries(default: 3),RoleArn(default: firehose role ARN),BufferSizeInMBs(default: 3), andBufferIntervalInSeconds(default: 60), are not stored in state. To prevent perpetual differences, it is therefore recommended to only include parameters with non-default values.
- parameter_
name str - Parameter name. Valid Values:
LambdaArn,NumberOfRetries,MetadataExtractionQuery,JsonParsingEngine,RoleArn,BufferSizeInMBs,BufferIntervalInSeconds,SubRecordType,Delimiter. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - parameter_
value str Parameter value. Must be between 1 and 512 length (inclusive). When providing a Lambda ARN, you should specify the resource version as well.
NOTE: Parameters with default values, including
NumberOfRetries(default: 3),RoleArn(default: firehose role ARN),BufferSizeInMBs(default: 3), andBufferIntervalInSeconds(default: 60), are not stored in state. To prevent perpetual differences, it is therefore recommended to only include parameters with non-default values.
- parameter
Name String - Parameter name. Valid Values:
LambdaArn,NumberOfRetries,MetadataExtractionQuery,JsonParsingEngine,RoleArn,BufferSizeInMBs,BufferIntervalInSeconds,SubRecordType,Delimiter. Validation is done against AWS SDK constants; so that values not explicitly listed may also work. - parameter
Value String Parameter value. Must be between 1 and 512 length (inclusive). When providing a Lambda ARN, you should specify the resource version as well.
NOTE: Parameters with default values, including
NumberOfRetries(default: 3),RoleArn(default: firehose role ARN),BufferSizeInMBs(default: 3), andBufferIntervalInSeconds(default: 60), are not stored in state. To prevent perpetual differences, it is therefore recommended to only include parameters with non-default values.
Import
Kinesis Firehose Delivery streams can be imported using the stream ARN, e.g.,
$ pulumi import aws:kinesis/firehoseDeliveryStream:FirehoseDeliveryStream foo arn:aws:firehose:us-east-1:XXX:deliverystream/example
NoteImport does not work for stream destination s3. Consider using extended_s3 since s3 destination is deprecated.
To learn more about importing existing cloud resources, see Importing resources.
Package Details
- Repository
- AWS Classic pulumi/pulumi-aws
- License
- Apache-2.0
- Notes
- This Pulumi package is based on the
awsTerraform Provider.
Viewing docs for AWS v5.43.0 (Older version)
published on Tuesday, Mar 10, 2026 by Pulumi
published on Tuesday, Mar 10, 2026 by Pulumi