Configure AWS Glue Connections

The aws:glue/connection:Connection resource, part of the Pulumi AWS provider, stores connection metadata for Glue jobs and crawlers: JDBC URLs, credentials, VPC placement, and connector configuration. This guide focuses on three capabilities: JDBC database connections with inline and Secrets Manager credentials, VPC networking for private database access, and custom connectors and cross-cloud integrations.

Connections reference Secrets Manager secrets, VPC infrastructure, S3-hosted JDBC drivers, and Lambda functions depending on the connection type. The examples are intentionally small. Combine them with your own database infrastructure, secrets, and network configuration.

Connect to a JDBC database with inline credentials

ETL jobs often need to read from or write to relational databases. Glue connections store JDBC URLs and credentials so jobs can access these databases without hardcoding connection details.

import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";

const example = new aws.glue.Connection("example", {
    name: "example",
    connectionProperties: {
        JDBC_CONNECTION_URL: "jdbc:mysql://example.com/exampledatabase",
        PASSWORD: "examplepassword",
        USERNAME: "exampleusername",
    },
});

import pulumi
import pulumi_aws as aws

example = aws.glue.Connection("example",
    name="example",
    connection_properties={
        "JDBC_CONNECTION_URL": "jdbc:mysql://example.com/exampledatabase",
        "PASSWORD": "examplepassword",
        "USERNAME": "exampleusername",
    })

package main

import (
	"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/glue"
	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)

func main() {
	pulumi.Run(func(ctx *pulumi.Context) error {
		_, err := glue.NewConnection(ctx, "example", &glue.ConnectionArgs{
			Name: pulumi.String("example"),
			ConnectionProperties: pulumi.StringMap{
				"JDBC_CONNECTION_URL": pulumi.String("jdbc:mysql://example.com/exampledatabase"),
				"PASSWORD":            pulumi.String("examplepassword"),
				"USERNAME":            pulumi.String("exampleusername"),
			},
		})
		if err != nil {
			return err
		}
		return nil
	})
}

using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Aws = Pulumi.Aws;

return await Deployment.RunAsync(() => 
{
    var example = new Aws.Glue.Connection("example", new()
    {
        Name = "example",
        ConnectionProperties = 
        {
            { "JDBC_CONNECTION_URL", "jdbc:mysql://example.com/exampledatabase" },
            { "PASSWORD", "examplepassword" },
            { "USERNAME", "exampleusername" },
        },
    });

});

package generated_program;

import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.glue.Connection;
import com.pulumi.aws.glue.ConnectionArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;

public class App {
    public static void main(String[] args) {
        Pulumi.run(App::stack);
    }

    public static void stack(Context ctx) {
        var example = new Connection("example", ConnectionArgs.builder()
            .name("example")
            .connectionProperties(Map.ofEntries(
                Map.entry("JDBC_CONNECTION_URL", "jdbc:mysql://example.com/exampledatabase"),
                Map.entry("PASSWORD", "examplepassword"),
                Map.entry("USERNAME", "exampleusername")
            ))
            .build());

    }
}

resources:
  example:
    type: aws:glue:Connection
    properties:
      name: example
      connectionProperties:
        JDBC_CONNECTION_URL: jdbc:mysql://example.com/exampledatabase
        PASSWORD: examplepassword
        USERNAME: exampleusername

The connectionProperties map holds the JDBC URL and credentials. Glue jobs reference this connection by name to access the database. The JDBC_CONNECTION_URL specifies the database endpoint and name; USERNAME and PASSWORD provide authentication.

Reference credentials from Secrets Manager

Production deployments avoid storing credentials directly in connection properties. Secrets Manager integration lets you rotate credentials without updating Glue connections.

import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";

const example = aws.secretsmanager.getSecret({
    name: "example-secret",
});
const exampleConnection = new aws.glue.Connection("example", {
    name: "example",
    connectionProperties: {
        JDBC_CONNECTION_URL: "jdbc:mysql://example.com/exampledatabase",
        SECRET_ID: example.then(example => example.name),
    },
});

import pulumi
import pulumi_aws as aws

example = aws.secretsmanager.get_secret(name="example-secret")
example_connection = aws.glue.Connection("example",
    name="example",
    connection_properties={
        "JDBC_CONNECTION_URL": "jdbc:mysql://example.com/exampledatabase",
        "SECRET_ID": example.name,
    })

package main

import (
	"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/glue"
	"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/secretsmanager"
	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)

func main() {
	pulumi.Run(func(ctx *pulumi.Context) error {
		example, err := secretsmanager.LookupSecret(ctx, &secretsmanager.LookupSecretArgs{
			Name: pulumi.StringRef("example-secret"),
		}, nil)
		if err != nil {
			return err
		}
		_, err = glue.NewConnection(ctx, "example", &glue.ConnectionArgs{
			Name: pulumi.String("example"),
			ConnectionProperties: pulumi.StringMap{
				"JDBC_CONNECTION_URL": pulumi.String("jdbc:mysql://example.com/exampledatabase"),
				"SECRET_ID":           pulumi.String(example.Name),
			},
		})
		if err != nil {
			return err
		}
		return nil
	})
}

using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Aws = Pulumi.Aws;

return await Deployment.RunAsync(() => 
{
    var example = Aws.SecretsManager.GetSecret.Invoke(new()
    {
        Name = "example-secret",
    });

    var exampleConnection = new Aws.Glue.Connection("example", new()
    {
        Name = "example",
        ConnectionProperties = 
        {
            { "JDBC_CONNECTION_URL", "jdbc:mysql://example.com/exampledatabase" },
            { "SECRET_ID", example.Apply(getSecretResult => getSecretResult.Name) },
        },
    });

});

package generated_program;

import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.secretsmanager.SecretsmanagerFunctions;
import com.pulumi.aws.secretsmanager.inputs.GetSecretArgs;
import com.pulumi.aws.glue.Connection;
import com.pulumi.aws.glue.ConnectionArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;

public class App {
    public static void main(String[] args) {
        Pulumi.run(App::stack);
    }

    public static void stack(Context ctx) {
        final var example = SecretsmanagerFunctions.getSecret(GetSecretArgs.builder()
            .name("example-secret")
            .build());

        var exampleConnection = new Connection("exampleConnection", ConnectionArgs.builder()
            .name("example")
            .connectionProperties(Map.ofEntries(
                Map.entry("JDBC_CONNECTION_URL", "jdbc:mysql://example.com/exampledatabase"),
                Map.entry("SECRET_ID", example.name())
            ))
            .build());

    }
}

resources:
  exampleConnection:
    type: aws:glue:Connection
    name: example
    properties:
      name: example
      connectionProperties:
        JDBC_CONNECTION_URL: jdbc:mysql://example.com/exampledatabase
        SECRET_ID: ${example.name}
variables:
  example:
    fn::invoke:
      function: aws:secretsmanager:getSecret
      arguments:
        name: example-secret

Instead of USERNAME and PASSWORD, the SECRET_ID property points to a Secrets Manager secret. Glue retrieves credentials at runtime, so rotating the secret automatically updates all jobs using this connection.

Access private RDS instances in a VPC

Databases in private subnets require VPC configuration so Glue can establish network connectivity. Glue creates elastic network interfaces in your subnet to reach the database.

import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";

const example = new aws.glue.Connection("example", {
    name: "example",
    connectionProperties: {
        JDBC_CONNECTION_URL: `jdbc:mysql://${exampleAwsRdsCluster.endpoint}/exampledatabase`,
        PASSWORD: "examplepassword",
        USERNAME: "exampleusername",
    },
    physicalConnectionRequirements: {
        availabilityZone: exampleAwsSubnet.availabilityZone,
        securityGroupIdLists: [exampleAwsSecurityGroup.id],
        subnetId: exampleAwsSubnet.id,
    },
});

import pulumi
import pulumi_aws as aws

example = aws.glue.Connection("example",
    name="example",
    connection_properties={
        "JDBC_CONNECTION_URL": f"jdbc:mysql://{example_aws_rds_cluster['endpoint']}/exampledatabase",
        "PASSWORD": "examplepassword",
        "USERNAME": "exampleusername",
    },
    physical_connection_requirements={
        "availability_zone": example_aws_subnet["availabilityZone"],
        "security_group_id_lists": [example_aws_security_group["id"]],
        "subnet_id": example_aws_subnet["id"],
    })

package main

import (
	"fmt"

	"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/glue"
	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)

func main() {
	pulumi.Run(func(ctx *pulumi.Context) error {
		_, err := glue.NewConnection(ctx, "example", &glue.ConnectionArgs{
			Name: pulumi.String("example"),
			ConnectionProperties: pulumi.StringMap{
				"JDBC_CONNECTION_URL": pulumi.Sprintf("jdbc:mysql://%v/exampledatabase", exampleAwsRdsCluster.Endpoint),
				"PASSWORD":            pulumi.String("examplepassword"),
				"USERNAME":            pulumi.String("exampleusername"),
			},
			PhysicalConnectionRequirements: &glue.ConnectionPhysicalConnectionRequirementsArgs{
				AvailabilityZone: pulumi.Any(exampleAwsSubnet.AvailabilityZone),
				SecurityGroupIdLists: pulumi.StringArray{
					exampleAwsSecurityGroup.Id,
				},
				SubnetId: pulumi.Any(exampleAwsSubnet.Id),
			},
		})
		if err != nil {
			return err
		}
		return nil
	})
}

using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Aws = Pulumi.Aws;

return await Deployment.RunAsync(() => 
{
    var example = new Aws.Glue.Connection("example", new()
    {
        Name = "example",
        ConnectionProperties = 
        {
            { "JDBC_CONNECTION_URL", $"jdbc:mysql://{exampleAwsRdsCluster.Endpoint}/exampledatabase" },
            { "PASSWORD", "examplepassword" },
            { "USERNAME", "exampleusername" },
        },
        PhysicalConnectionRequirements = new Aws.Glue.Inputs.ConnectionPhysicalConnectionRequirementsArgs
        {
            AvailabilityZone = exampleAwsSubnet.AvailabilityZone,
            SecurityGroupIdLists = new[]
            {
                exampleAwsSecurityGroup.Id,
            },
            SubnetId = exampleAwsSubnet.Id,
        },
    });

});

package generated_program;

import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.glue.Connection;
import com.pulumi.aws.glue.ConnectionArgs;
import com.pulumi.aws.glue.inputs.ConnectionPhysicalConnectionRequirementsArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;

public class App {
    public static void main(String[] args) {
        Pulumi.run(App::stack);
    }

    public static void stack(Context ctx) {
        var example = new Connection("example", ConnectionArgs.builder()
            .name("example")
            .connectionProperties(Map.ofEntries(
                Map.entry("JDBC_CONNECTION_URL", String.format("jdbc:mysql://%s/exampledatabase", exampleAwsRdsCluster.endpoint())),
                Map.entry("PASSWORD", "examplepassword"),
                Map.entry("USERNAME", "exampleusername")
            ))
            .physicalConnectionRequirements(ConnectionPhysicalConnectionRequirementsArgs.builder()
                .availabilityZone(exampleAwsSubnet.availabilityZone())
                .securityGroupIdLists(exampleAwsSecurityGroup.id())
                .subnetId(exampleAwsSubnet.id())
                .build())
            .build());

    }
}

resources:
  example:
    type: aws:glue:Connection
    properties:
      name: example
      connectionProperties:
        JDBC_CONNECTION_URL: jdbc:mysql://${exampleAwsRdsCluster.endpoint}/exampledatabase
        PASSWORD: examplepassword
        USERNAME: exampleusername
      physicalConnectionRequirements:
        availabilityZone: ${exampleAwsSubnet.availabilityZone}
        securityGroupIdLists:
          - ${exampleAwsSecurityGroup.id}
        subnetId: ${exampleAwsSubnet.id}

The physicalConnectionRequirements block places the connection in a specific subnet with security groups. Glue provisions network interfaces in that subnet to access the RDS cluster. The availabilityZone, subnetId, and securityGroupIdLists must match your VPC topology.

Define and use custom JDBC connectors

Glue supports databases beyond its built-in connectors by loading custom JDBC drivers from S3. You define a template connection with the driver details, then reference it from actual connections.

import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";

// Define the custom connector using the connection_type of `CUSTOM` with the match_criteria of `template_connection`
// Example here being a snowflake jdbc connector with a secret having user and password as keys
const example = aws.secretsmanager.getSecret({
    name: "example-secret",
});
const example1 = new aws.glue.Connection("example1", {
    name: "example1",
    connectionType: "CUSTOM",
    connectionProperties: {
        CONNECTOR_CLASS_NAME: "net.snowflake.client.jdbc.SnowflakeDriver",
        CONNECTION_TYPE: "Jdbc",
        CONNECTOR_URL: "s3://example/snowflake-jdbc.jar",
        JDBC_CONNECTION_URL: "[[\"default=jdbc:snowflake://example.com/?user=${user}&password=${password}\"],\",\"]",
    },
    matchCriterias: ["template-connection"],
});
// Reference the connector using match_criteria with the connector created above.
const example2 = new aws.glue.Connection("example2", {
    name: "example2",
    connectionType: "CUSTOM",
    connectionProperties: {
        CONNECTOR_CLASS_NAME: "net.snowflake.client.jdbc.SnowflakeDriver",
        CONNECTION_TYPE: "Jdbc",
        CONNECTOR_URL: "s3://example/snowflake-jdbc.jar",
        JDBC_CONNECTION_URL: "jdbc:snowflake://example.com/?user=${user}&password=${password}",
        SECRET_ID: example.then(example => example.name),
    },
    matchCriterias: [
        "Connection",
        example1.name,
    ],
});

import pulumi
import pulumi_aws as aws

# Define the custom connector using the connection_type of `CUSTOM` with the match_criteria of `template_connection`
# Example here being a snowflake jdbc connector with a secret having user and password as keys
example = aws.secretsmanager.get_secret(name="example-secret")
example1 = aws.glue.Connection("example1",
    name="example1",
    connection_type="CUSTOM",
    connection_properties={
        "CONNECTOR_CLASS_NAME": "net.snowflake.client.jdbc.SnowflakeDriver",
        "CONNECTION_TYPE": "Jdbc",
        "CONNECTOR_URL": "s3://example/snowflake-jdbc.jar",
        "JDBC_CONNECTION_URL": "[[\"default=jdbc:snowflake://example.com/?user=${user}&password=${password}\"],\",\"]",
    },
    match_criterias=["template-connection"])
# Reference the connector using match_criteria with the connector created above.
example2 = aws.glue.Connection("example2",
    name="example2",
    connection_type="CUSTOM",
    connection_properties={
        "CONNECTOR_CLASS_NAME": "net.snowflake.client.jdbc.SnowflakeDriver",
        "CONNECTION_TYPE": "Jdbc",
        "CONNECTOR_URL": "s3://example/snowflake-jdbc.jar",
        "JDBC_CONNECTION_URL": "jdbc:snowflake://example.com/?user=${user}&password=${password}",
        "SECRET_ID": example.name,
    },
    match_criterias=[
        "Connection",
        example1.name,
    ])

package main

import (
	"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/glue"
	"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/secretsmanager"
	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)

func main() {
	pulumi.Run(func(ctx *pulumi.Context) error {
		// Define the custom connector using the connection_type of `CUSTOM` with the match_criteria of `template_connection`
		// Example here being a snowflake jdbc connector with a secret having user and password as keys
		example, err := secretsmanager.LookupSecret(ctx, &secretsmanager.LookupSecretArgs{
			Name: pulumi.StringRef("example-secret"),
		}, nil)
		if err != nil {
			return err
		}
		example1, err := glue.NewConnection(ctx, "example1", &glue.ConnectionArgs{
			Name:           pulumi.String("example1"),
			ConnectionType: pulumi.String("CUSTOM"),
			ConnectionProperties: pulumi.StringMap{
				"CONNECTOR_CLASS_NAME": pulumi.String("net.snowflake.client.jdbc.SnowflakeDriver"),
				"CONNECTION_TYPE":      pulumi.String("Jdbc"),
				"CONNECTOR_URL":        pulumi.String("s3://example/snowflake-jdbc.jar"),
				"JDBC_CONNECTION_URL":  pulumi.String("[[\"default=jdbc:snowflake://example.com/?user=${user}&password=${password}\"],\",\"]"),
			},
			MatchCriterias: pulumi.StringArray{
				pulumi.String("template-connection"),
			},
		})
		if err != nil {
			return err
		}
		// Reference the connector using match_criteria with the connector created above.
		_, err = glue.NewConnection(ctx, "example2", &glue.ConnectionArgs{
			Name:           pulumi.String("example2"),
			ConnectionType: pulumi.String("CUSTOM"),
			ConnectionProperties: pulumi.StringMap{
				"CONNECTOR_CLASS_NAME": pulumi.String("net.snowflake.client.jdbc.SnowflakeDriver"),
				"CONNECTION_TYPE":      pulumi.String("Jdbc"),
				"CONNECTOR_URL":        pulumi.String("s3://example/snowflake-jdbc.jar"),
				"JDBC_CONNECTION_URL":  pulumi.String("jdbc:snowflake://example.com/?user=${user}&password=${password}"),
				"SECRET_ID":            pulumi.String(example.Name),
			},
			MatchCriterias: pulumi.StringArray{
				pulumi.String("Connection"),
				example1.Name,
			},
		})
		if err != nil {
			return err
		}
		return nil
	})
}

using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Aws = Pulumi.Aws;

return await Deployment.RunAsync(() => 
{
    // Define the custom connector using the connection_type of `CUSTOM` with the match_criteria of `template_connection`
    // Example here being a snowflake jdbc connector with a secret having user and password as keys
    var example = Aws.SecretsManager.GetSecret.Invoke(new()
    {
        Name = "example-secret",
    });

    var example1 = new Aws.Glue.Connection("example1", new()
    {
        Name = "example1",
        ConnectionType = "CUSTOM",
        ConnectionProperties = 
        {
            { "CONNECTOR_CLASS_NAME", "net.snowflake.client.jdbc.SnowflakeDriver" },
            { "CONNECTION_TYPE", "Jdbc" },
            { "CONNECTOR_URL", "s3://example/snowflake-jdbc.jar" },
            { "JDBC_CONNECTION_URL", "[[\"default=jdbc:snowflake://example.com/?user=${user}&password=${password}\"],\",\"]" },
        },
        MatchCriterias = new[]
        {
            "template-connection",
        },
    });

    // Reference the connector using match_criteria with the connector created above.
    var example2 = new Aws.Glue.Connection("example2", new()
    {
        Name = "example2",
        ConnectionType = "CUSTOM",
        ConnectionProperties = 
        {
            { "CONNECTOR_CLASS_NAME", "net.snowflake.client.jdbc.SnowflakeDriver" },
            { "CONNECTION_TYPE", "Jdbc" },
            { "CONNECTOR_URL", "s3://example/snowflake-jdbc.jar" },
            { "JDBC_CONNECTION_URL", "jdbc:snowflake://example.com/?user=${user}&password=${password}" },
            { "SECRET_ID", example.Apply(getSecretResult => getSecretResult.Name) },
        },
        MatchCriterias = new[]
        {
            "Connection",
            example1.Name,
        },
    });

});

package generated_program;

import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.secretsmanager.SecretsmanagerFunctions;
import com.pulumi.aws.secretsmanager.inputs.GetSecretArgs;
import com.pulumi.aws.glue.Connection;
import com.pulumi.aws.glue.ConnectionArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;

public class App {
    public static void main(String[] args) {
        Pulumi.run(App::stack);
    }

    public static void stack(Context ctx) {
        // Define the custom connector using the connection_type of `CUSTOM` with the match_criteria of `template_connection`
        // Example here being a snowflake jdbc connector with a secret having user and password as keys
        final var example = SecretsmanagerFunctions.getSecret(GetSecretArgs.builder()
            .name("example-secret")
            .build());

        var example1 = new Connection("example1", ConnectionArgs.builder()
            .name("example1")
            .connectionType("CUSTOM")
            .connectionProperties(Map.ofEntries(
                Map.entry("CONNECTOR_CLASS_NAME", "net.snowflake.client.jdbc.SnowflakeDriver"),
                Map.entry("CONNECTION_TYPE", "Jdbc"),
                Map.entry("CONNECTOR_URL", "s3://example/snowflake-jdbc.jar"),
                Map.entry("JDBC_CONNECTION_URL", "[[\"default=jdbc:snowflake://example.com/?user=${user}&password=${password}\"],\",\"]")
            ))
            .matchCriterias("template-connection")
            .build());

        // Reference the connector using match_criteria with the connector created above.
        var example2 = new Connection("example2", ConnectionArgs.builder()
            .name("example2")
            .connectionType("CUSTOM")
            .connectionProperties(Map.ofEntries(
                Map.entry("CONNECTOR_CLASS_NAME", "net.snowflake.client.jdbc.SnowflakeDriver"),
                Map.entry("CONNECTION_TYPE", "Jdbc"),
                Map.entry("CONNECTOR_URL", "s3://example/snowflake-jdbc.jar"),
                Map.entry("JDBC_CONNECTION_URL", "jdbc:snowflake://example.com/?user=${user}&password=${password}"),
                Map.entry("SECRET_ID", example.name())
            ))
            .matchCriterias(            
                "Connection",
                example1.name())
            .build());

    }
}

resources:
  example1:
    type: aws:glue:Connection
    properties:
      name: example1
      connectionType: CUSTOM
      connectionProperties:
        CONNECTOR_CLASS_NAME: net.snowflake.client.jdbc.SnowflakeDriver
        CONNECTION_TYPE: Jdbc
        CONNECTOR_URL: s3://example/snowflake-jdbc.jar
        JDBC_CONNECTION_URL: '[["default=jdbc:snowflake://example.com/?user=$${user}&password=$${password}"],","]'
      matchCriterias:
        - template-connection
  # Reference the connector using match_criteria with the connector created above.
  example2:
    type: aws:glue:Connection
    properties:
      name: example2
      connectionType: CUSTOM
      connectionProperties:
        CONNECTOR_CLASS_NAME: net.snowflake.client.jdbc.SnowflakeDriver
        CONNECTION_TYPE: Jdbc
        CONNECTOR_URL: s3://example/snowflake-jdbc.jar
        JDBC_CONNECTION_URL: jdbc:snowflake://example.com/?user=$${user}&password=$${password}
        SECRET_ID: ${example.name}
      matchCriterias:
        - Connection
        - ${example1.name}
variables:
  # Define the custom connector using the connection_type of `CUSTOM` with the match_criteria of `template_connection`
  # Example here being a snowflake jdbc connector with a secret having user and password as keys
  example:
    fn::invoke:
      function: aws:secretsmanager:getSecret
      arguments:
        name: example-secret

The first connection (connectionType CUSTOM with matchCriterias containing “template-connection”) defines the connector: CONNECTOR_CLASS_NAME specifies the driver class, CONNECTOR_URL points to the JAR in S3. The second connection references the template via matchCriterias and adds database-specific properties like JDBC_CONNECTION_URL and SECRET_ID.

Connect to Google BigQuery with service account credentials

Cross-cloud ETL pipelines often move data between AWS and Google Cloud. BigQuery connections use service account JSON credentials stored in Secrets Manager.

import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
import * as std from "@pulumi/std";

const example = new aws.secretsmanager.Secret("example", {name: "example-secret"});
const exampleSecretVersion = new aws.secretsmanager.SecretVersion("example", {
    secretId: example.id,
    secretString: JSON.stringify({
        credentials: std.base64encode({
            input: `{
  \\"type\\": \\"service_account\\",
  \\"project_id\\": \\"example-project\\",
  \\"private_key_id\\": \\"example-key\\",
  \\"private_key\\": \\"-----BEGIN RSA PRIVATE KEY-----\\
REDACTED\\
-----END RSA PRIVATE KEY-----\\",
  \\"client_email\\": \\"example-project@appspot.gserviceaccount.com\\",
  \\"client_id\\": example-client\\",
  \\"auth_uri\\": \\"https://accounts.google.com/o/oauth2/auth\\",
  \\"token_uri\\": \\"https://oauth2.googleapis.com/token\\",
  \\"auth_provider_x509_cert_url\\": \\"https://www.googleapis.com/oauth2/v1/certs\\",
  \\"client_x509_cert_url\\": \\"https://www.googleapis.com/robot/v1/metadata/x509/example-project%%40appspot.gserviceaccount.com\\",
  \\"universe_domain\\": \\"googleapis.com\\"
}
`,
        }).then(invoke => invoke.result),
    }),
});
const exampleConnection = new aws.glue.Connection("example", {
    name: "example",
    connectionType: "BIGQUERY",
    connectionProperties: {
        SparkProperties: pulumi.jsonStringify({
            secretId: example.name,
        }),
    },
});

import pulumi
import json
import pulumi_aws as aws
import pulumi_std as std

example = aws.secretsmanager.Secret("example", name="example-secret")
example_secret_version = aws.secretsmanager.SecretVersion("example",
    secret_id=example.id,
    secret_string=json.dumps({
        "credentials": std.base64encode(input="""{
  \"type\": \"service_account\",
  \"project_id\": \"example-project\",
  \"private_key_id\": \"example-key\",
  \"private_key\": \"-----BEGIN RSA PRIVATE KEY-----\
REDACTED\
-----END RSA PRIVATE KEY-----\",
  \"client_email\": \"example-project@appspot.gserviceaccount.com\",
  \"client_id\": example-client\",
  \"auth_uri\": \"https://accounts.google.com/o/oauth2/auth\",
  \"token_uri\": \"https://oauth2.googleapis.com/token\",
  \"auth_provider_x509_cert_url\": \"https://www.googleapis.com/oauth2/v1/certs\",
  \"client_x509_cert_url\": \"https://www.googleapis.com/robot/v1/metadata/x509/example-project%%40appspot.gserviceaccount.com\",
  \"universe_domain\": \"googleapis.com\"
}
""").result,
    }))
example_connection = aws.glue.Connection("example",
    name="example",
    connection_type="BIGQUERY",
    connection_properties={
        "SparkProperties": pulumi.Output.json_dumps({
            "secretId": example.name,
        }),
    })

package main

import (
	"encoding/json"

	"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/glue"
	"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/secretsmanager"
	"github.com/pulumi/pulumi-std/sdk/go/std"
	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)

func main() {
	pulumi.Run(func(ctx *pulumi.Context) error {
		example, err := secretsmanager.NewSecret(ctx, "example", &secretsmanager.SecretArgs{
			Name: pulumi.String("example-secret"),
		})
		if err != nil {
			return err
		}
		tmpJSON0, err := json.Marshal(map[string]interface{}{
			"credentials": std.Base64encode(ctx, &std.Base64encodeArgs{
				Input: `{
  \"type\": \"service_account\",
  \"project_id\": \"example-project\",
  \"private_key_id\": \"example-key\",
  \"private_key\": \"-----BEGIN RSA PRIVATE KEY-----\
REDACTED\
-----END RSA PRIVATE KEY-----\",
  \"client_email\": \"example-project@appspot.gserviceaccount.com\",
  \"client_id\": example-client\",
  \"auth_uri\": \"https://accounts.google.com/o/oauth2/auth\",
  \"token_uri\": \"https://oauth2.googleapis.com/token\",
  \"auth_provider_x509_cert_url\": \"https://www.googleapis.com/oauth2/v1/certs\",
  \"client_x509_cert_url\": \"https://www.googleapis.com/robot/v1/metadata/x509/example-project%%40appspot.gserviceaccount.com\",
  \"universe_domain\": \"googleapis.com\"
}
`,
			}, nil).Result,
		})
		if err != nil {
			return err
		}
		json0 := string(tmpJSON0)
		_, err = secretsmanager.NewSecretVersion(ctx, "example", &secretsmanager.SecretVersionArgs{
			SecretId:     example.ID(),
			SecretString: pulumi.String(json0),
		})
		if err != nil {
			return err
		}
		_, err = glue.NewConnection(ctx, "example", &glue.ConnectionArgs{
			Name:           pulumi.String("example"),
			ConnectionType: pulumi.String("BIGQUERY"),
			ConnectionProperties: pulumi.StringMap{
				"SparkProperties": example.Name.ApplyT(func(name string) (pulumi.String, error) {
					var _zero pulumi.String
					tmpJSON1, err := json.Marshal(map[string]interface{}{
						"secretId": name,
					})
					if err != nil {
						return _zero, err
					}
					json1 := string(tmpJSON1)
					return pulumi.String(json1), nil
				}).(pulumi.StringOutput),
			},
		})
		if err != nil {
			return err
		}
		return nil
	})
}

using System.Collections.Generic;
using System.Linq;
using System.Text.Json;
using Pulumi;
using Aws = Pulumi.Aws;
using Std = Pulumi.Std;

return await Deployment.RunAsync(() => 
{
    var example = new Aws.SecretsManager.Secret("example", new()
    {
        Name = "example-secret",
    });

    var exampleSecretVersion = new Aws.SecretsManager.SecretVersion("example", new()
    {
        SecretId = example.Id,
        SecretString = JsonSerializer.Serialize(new Dictionary<string, object?>
        {
            ["credentials"] = Std.Base64encode.Invoke(new()
            {
                Input = @"{
  \""type\"": \""service_account\"",
  \""project_id\"": \""example-project\"",
  \""private_key_id\"": \""example-key\"",
  \""private_key\"": \""-----BEGIN RSA PRIVATE KEY-----\
REDACTED\
-----END RSA PRIVATE KEY-----\"",
  \""client_email\"": \""example-project@appspot.gserviceaccount.com\"",
  \""client_id\"": example-client\"",
  \""auth_uri\"": \""https://accounts.google.com/o/oauth2/auth\"",
  \""token_uri\"": \""https://oauth2.googleapis.com/token\"",
  \""auth_provider_x509_cert_url\"": \""https://www.googleapis.com/oauth2/v1/certs\"",
  \""client_x509_cert_url\"": \""https://www.googleapis.com/robot/v1/metadata/x509/example-project%%40appspot.gserviceaccount.com\"",
  \""universe_domain\"": \""googleapis.com\""
}
",
            }).Apply(invoke => invoke.Result),
        }),
    });

    var exampleConnection = new Aws.Glue.Connection("example", new()
    {
        Name = "example",
        ConnectionType = "BIGQUERY",
        ConnectionProperties = 
        {
            { "SparkProperties", Output.JsonSerialize(Output.Create(new Dictionary<string, object?>
            {
                ["secretId"] = example.Name,
            })) },
        },
    });

});

package generated_program;

import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.secretsmanager.Secret;
import com.pulumi.aws.secretsmanager.SecretArgs;
import com.pulumi.aws.secretsmanager.SecretVersion;
import com.pulumi.aws.secretsmanager.SecretVersionArgs;
import com.pulumi.std.StdFunctions;
import com.pulumi.std.inputs.Base64encodeArgs;
import com.pulumi.aws.glue.Connection;
import com.pulumi.aws.glue.ConnectionArgs;
import static com.pulumi.codegen.internal.Serialization.*;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;

public class App {
    public static void main(String[] args) {
        Pulumi.run(App::stack);
    }

    public static void stack(Context ctx) {
        var example = new Secret("example", SecretArgs.builder()
            .name("example-secret")
            .build());

        var exampleSecretVersion = new SecretVersion("exampleSecretVersion", SecretVersionArgs.builder()
            .secretId(example.id())
            .secretString(serializeJson(
                jsonObject(
                    jsonProperty("credentials", StdFunctions.base64encode(Base64encodeArgs.builder()
                        .input("""
{
  \"type\": \"service_account\",
  \"project_id\": \"example-project\",
  \"private_key_id\": \"example-key\",
  \"private_key\": \"-----BEGIN RSA PRIVATE KEY-----\
REDACTED\
-----END RSA PRIVATE KEY-----\",
  \"client_email\": \"example-project@appspot.gserviceaccount.com\",
  \"client_id\": example-client\",
  \"auth_uri\": \"https://accounts.google.com/o/oauth2/auth\",
  \"token_uri\": \"https://oauth2.googleapis.com/token\",
  \"auth_provider_x509_cert_url\": \"https://www.googleapis.com/oauth2/v1/certs\",
  \"client_x509_cert_url\": \"https://www.googleapis.com/robot/v1/metadata/x509/example-project%%40appspot.gserviceaccount.com\",
  \"universe_domain\": \"googleapis.com\"
}
                        """)
                        .build()).result())
                )))
            .build());

        var exampleConnection = new Connection("exampleConnection", ConnectionArgs.builder()
            .name("example")
            .connectionType("BIGQUERY")
            .connectionProperties(Map.of("SparkProperties", example.name().applyValue(_name -> serializeJson(
                jsonObject(
                    jsonProperty("secretId", _name)
                )))))
            .build());

    }
}

resources:
  example:
    type: aws:secretsmanager:Secret
    properties:
      name: example-secret
  exampleSecretVersion:
    type: aws:secretsmanager:SecretVersion
    name: example
    properties:
      secretId: ${example.id}
      secretString:
        fn::toJSON:
          credentials:
            fn::invoke:
              function: std:base64encode
              arguments:
                input: |
                  {
                    \"type\": \"service_account\",
                    \"project_id\": \"example-project\",
                    \"private_key_id\": \"example-key\",
                    \"private_key\": \"-----BEGIN RSA PRIVATE KEY-----\
                  REDACTED\
                  -----END RSA PRIVATE KEY-----\",
                    \"client_email\": \"example-project@appspot.gserviceaccount.com\",
                    \"client_id\": example-client\",
                    \"auth_uri\": \"https://accounts.google.com/o/oauth2/auth\",
                    \"token_uri\": \"https://oauth2.googleapis.com/token\",
                    \"auth_provider_x509_cert_url\": \"https://www.googleapis.com/oauth2/v1/certs\",
                    \"client_x509_cert_url\": \"https://www.googleapis.com/robot/v1/metadata/x509/example-project%%40appspot.gserviceaccount.com\",
                    \"universe_domain\": \"googleapis.com\"
                  }                  
              return: result
  exampleConnection:
    type: aws:glue:Connection
    name: example
    properties:
      name: example
      connectionType: BIGQUERY
      connectionProperties:
        SparkProperties:
          fn::toJSON:
            secretId: ${example.name}

The connectionType BIGQUERY requires SparkProperties with a secretId pointing to base64-encoded service account credentials. Glue uses these credentials to authenticate with Google Cloud and access BigQuery datasets.

Query DynamoDB tables with Athena Federated Query

Athena Federated Query lets you join DynamoDB data with S3 data lakes. The connection points to a Lambda function that translates SQL queries into DynamoDB operations.

import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";

const test = new aws.glue.Connection("test", {
    name: "example",
    connectionType: "DYNAMODB",
    athenaProperties: {
        lambda_function_arn: "arn:aws:lambda:us-east-1:123456789012:function:athenafederatedcatalog_athena_abcdefgh",
        disable_spill_encryption: "false",
        spill_bucket: "example-bucket",
    },
});

import pulumi
import pulumi_aws as aws

test = aws.glue.Connection("test",
    name="example",
    connection_type="DYNAMODB",
    athena_properties={
        "lambda_function_arn": "arn:aws:lambda:us-east-1:123456789012:function:athenafederatedcatalog_athena_abcdefgh",
        "disable_spill_encryption": "false",
        "spill_bucket": "example-bucket",
    })

package main

import (
	"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/glue"
	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)

func main() {
	pulumi.Run(func(ctx *pulumi.Context) error {
		_, err := glue.NewConnection(ctx, "test", &glue.ConnectionArgs{
			Name:           pulumi.String("example"),
			ConnectionType: pulumi.String("DYNAMODB"),
			AthenaProperties: pulumi.StringMap{
				"lambda_function_arn":      pulumi.String("arn:aws:lambda:us-east-1:123456789012:function:athenafederatedcatalog_athena_abcdefgh"),
				"disable_spill_encryption": pulumi.String("false"),
				"spill_bucket":             pulumi.String("example-bucket"),
			},
		})
		if err != nil {
			return err
		}
		return nil
	})
}

using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Aws = Pulumi.Aws;

return await Deployment.RunAsync(() => 
{
    var test = new Aws.Glue.Connection("test", new()
    {
        Name = "example",
        ConnectionType = "DYNAMODB",
        AthenaProperties = 
        {
            { "lambda_function_arn", "arn:aws:lambda:us-east-1:123456789012:function:athenafederatedcatalog_athena_abcdefgh" },
            { "disable_spill_encryption", "false" },
            { "spill_bucket", "example-bucket" },
        },
    });

});

package generated_program;

import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.glue.Connection;
import com.pulumi.aws.glue.ConnectionArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;

public class App {
    public static void main(String[] args) {
        Pulumi.run(App::stack);
    }

    public static void stack(Context ctx) {
        var test = new Connection("test", ConnectionArgs.builder()
            .name("example")
            .connectionType("DYNAMODB")
            .athenaProperties(Map.ofEntries(
                Map.entry("lambda_function_arn", "arn:aws:lambda:us-east-1:123456789012:function:athenafederatedcatalog_athena_abcdefgh"),
                Map.entry("disable_spill_encryption", "false"),
                Map.entry("spill_bucket", "example-bucket")
            ))
            .build());

    }
}

resources:
  test:
    type: aws:glue:Connection
    properties:
      name: example
      connectionType: DYNAMODB
      athenaProperties:
        lambda_function_arn: arn:aws:lambda:us-east-1:123456789012:function:athenafederatedcatalog_athena_abcdefgh
        disable_spill_encryption: 'false'
        spill_bucket: example-bucket

The connectionType DYNAMODB uses athenaProperties instead of connectionProperties. The lambda_function_arn points to the Athena DynamoDB connector Lambda function; spill_bucket specifies where Athena writes intermediate query results.

Beyond these examples

These snippets focus on specific connection-level features: JDBC and custom connector configuration, Secrets Manager integration, and cross-cloud database connections. They’re intentionally minimal rather than full ETL pipelines.

The examples may reference pre-existing infrastructure such as VPC subnets and security groups, Secrets Manager secrets with database credentials, S3 buckets with JDBC driver JARs, and Lambda functions for Athena Federated Query. They focus on configuring the connection rather than provisioning everything around it.

To keep things focused, common connection patterns are omitted, including:

  • Kafka and MongoDB connection types
  • Network connection type for VPC peering
  • Connection validation and testing
  • IAM permissions for Glue to access secrets and VPC resources

These omissions are intentional: the goal is to illustrate how each connection feature is wired, not provide drop-in ETL modules. See the Glue Connection resource reference for all available configuration options.

Let's configure AWS Glue Connections

Get started with Pulumi Cloud, then follow our quick setup guide to deploy this infrastructure.

Try Pulumi Cloud for FREE

Frequently Asked Questions

Connection Types & Configuration
Which connection types require SparkProperties?
AZURECOSMOS, AZURESQL, BIGQUERY, OPENSEARCH, and SNOWFLAKE connections require SparkProperties within connectionProperties containing a JSON document with the actual connection properties.
What's the difference between athenaProperties and connectionProperties?
athenaProperties is used for Athena-specific connections like DYNAMODB, while connectionProperties is used for most other connection types like JDBC, MONGODB, and KAFKA.
Security & Credentials
How do I securely store database credentials instead of hardcoding them?
Use SECRET_ID in connectionProperties to reference an AWS Secrets Manager secret instead of hardcoding PASSWORD and USERNAME.
How do I format credentials for BigQuery connections?
BigQuery connections require base64-encoded service account JSON credentials stored in Secrets Manager, referenced via secretId in SparkProperties.
VPC & Networking
How do I connect to a database in a VPC?
Configure physicalConnectionRequirements with availabilityZone, securityGroupIdLists, and subnetId to access VPC-based resources.
Custom Connectors
How do I use a custom connector like Snowflake JDBC?
Create a template connection with connectionType: CUSTOM and matchCriterias: ['template-connection'], then reference it from other connections using matchCriterias: ['Connection', templateConnectionName].
Immutability & Limitations
What properties can't I change after creating a connection?
The name and catalogId properties are immutable. Changing them requires recreating the connection.
What is catalogId and do I need to set it?
catalogId is the Data Catalog ID where the connection is created. If not specified, it defaults to your AWS account ID.

Using a different cloud?

Explore analytics guides for other cloud providers:

Azure Guides GCP Guides