Configure GCP BigQuery Connections

The gcp:bigquery/connection:Connection resource, part of the Pulumi GCP provider, defines BigQuery connections to external data sources: Cloud SQL, Cloud Spanner, AWS, Azure, and Spark execution environments. This guide focuses on four capabilities: Cloud SQL federated queries, cross-cloud federation (AWS and Azure), Cloud Spanner integration with Data Boost, and Spark stored procedure execution.

Connections reference external infrastructure that must exist separately: Cloud SQL instances, AWS IAM roles, Azure applications, Spanner databases, and Dataproc clusters. The examples are intentionally small. Combine them with your own database instances, IAM configuration, and query workloads.

Delegate access to GCP resources

BigQuery often needs to access other GCP services like Cloud Storage or Pub/Sub without explicit credentials. The cloudResource configuration enables this through service account delegation.

import * as pulumi from "@pulumi/pulumi";
import * as gcp from "@pulumi/gcp";

const connection = new gcp.bigquery.Connection("connection", {
    connectionId: "my-connection",
    location: "US",
    friendlyName: "πŸ‘‹",
    description: "a riveting description",
    cloudResource: {},
});

import pulumi
import pulumi_gcp as gcp

connection = gcp.bigquery.Connection("connection",
    connection_id="my-connection",
    location="US",
    friendly_name="πŸ‘‹",
    description="a riveting description",
    cloud_resource={})

package main

import (
	"github.com/pulumi/pulumi-gcp/sdk/v9/go/gcp/bigquery"
	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)

func main() {
	pulumi.Run(func(ctx *pulumi.Context) error {
		_, err := bigquery.NewConnection(ctx, "connection", &bigquery.ConnectionArgs{
			ConnectionId:  pulumi.String("my-connection"),
			Location:      pulumi.String("US"),
			FriendlyName:  pulumi.String("πŸ‘‹"),
			Description:   pulumi.String("a riveting description"),
			CloudResource: &bigquery.ConnectionCloudResourceArgs{},
		})
		if err != nil {
			return err
		}
		return nil
	})
}

using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Gcp = Pulumi.Gcp;

return await Deployment.RunAsync(() => 
{
    var connection = new Gcp.BigQuery.Connection("connection", new()
    {
        ConnectionId = "my-connection",
        Location = "US",
        FriendlyName = "πŸ‘‹",
        Description = "a riveting description",
        CloudResource = null,
    });

});

package generated_program;

import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.gcp.bigquery.Connection;
import com.pulumi.gcp.bigquery.ConnectionArgs;
import com.pulumi.gcp.bigquery.inputs.ConnectionCloudResourceArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;

public class App {
    public static void main(String[] args) {
        Pulumi.run(App::stack);
    }

    public static void stack(Context ctx) {
        var connection = new Connection("connection", ConnectionArgs.builder()
            .connectionId("my-connection")
            .location("US")
            .friendlyName("πŸ‘‹")
            .description("a riveting description")
            .cloudResource(ConnectionCloudResourceArgs.builder()
                .build())
            .build());

    }
}

resources:
  connection:
    type: gcp:bigquery:Connection
    properties:
      connectionId: my-connection
      location: US
      friendlyName: "\U0001F44B"
      description: a riveting description
      cloudResource: {}

The cloudResource property creates a connection that uses BigQuery’s service account to access other GCP resources. The connectionId provides a stable identifier, and location determines where the connection metadata is stored. This connection type doesn’t require credentials because it relies on IAM permissions granted to BigQuery’s service account.

Connect to Cloud SQL databases

Federated queries let BigQuery join data from Cloud SQL databases with BigQuery tables, combining analytical and transactional data.

import * as pulumi from "@pulumi/pulumi";
import * as gcp from "@pulumi/gcp";
import * as random from "@pulumi/random";

const instance = new gcp.sql.DatabaseInstance("instance", {
    name: "my-database-instance",
    databaseVersion: "POSTGRES_11",
    region: "us-central1",
    settings: {
        tier: "db-f1-micro",
    },
    deletionProtection: true,
});
const db = new gcp.sql.Database("db", {
    instance: instance.name,
    name: "db",
});
const pwd = new random.index.Password("pwd", {
    length: 16,
    special: false,
});
const user = new gcp.sql.User("user", {
    name: "user",
    instance: instance.name,
    password: pwd.result,
});
const connection = new gcp.bigquery.Connection("connection", {
    friendlyName: "πŸ‘‹",
    description: "a riveting description",
    location: "US",
    cloudSql: {
        instanceId: instance.connectionName,
        database: db.name,
        type: "POSTGRES",
        credential: {
            username: user.name,
            password: user.password,
        },
    },
});

import pulumi
import pulumi_gcp as gcp
import pulumi_random as random

instance = gcp.sql.DatabaseInstance("instance",
    name="my-database-instance",
    database_version="POSTGRES_11",
    region="us-central1",
    settings={
        "tier": "db-f1-micro",
    },
    deletion_protection=True)
db = gcp.sql.Database("db",
    instance=instance.name,
    name="db")
pwd = random.index.Password("pwd",
    length=16,
    special=False)
user = gcp.sql.User("user",
    name="user",
    instance=instance.name,
    password=pwd["result"])
connection = gcp.bigquery.Connection("connection",
    friendly_name="πŸ‘‹",
    description="a riveting description",
    location="US",
    cloud_sql={
        "instance_id": instance.connection_name,
        "database": db.name,
        "type": "POSTGRES",
        "credential": {
            "username": user.name,
            "password": user.password,
        },
    })

package main

import (
	"github.com/pulumi/pulumi-gcp/sdk/v9/go/gcp/bigquery"
	"github.com/pulumi/pulumi-gcp/sdk/v9/go/gcp/sql"
	"github.com/pulumi/pulumi-random/sdk/v4/go/random"
	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)

func main() {
	pulumi.Run(func(ctx *pulumi.Context) error {
		instance, err := sql.NewDatabaseInstance(ctx, "instance", &sql.DatabaseInstanceArgs{
			Name:            pulumi.String("my-database-instance"),
			DatabaseVersion: pulumi.String("POSTGRES_11"),
			Region:          pulumi.String("us-central1"),
			Settings: &sql.DatabaseInstanceSettingsArgs{
				Tier: pulumi.String("db-f1-micro"),
			},
			DeletionProtection: pulumi.Bool(true),
		})
		if err != nil {
			return err
		}
		db, err := sql.NewDatabase(ctx, "db", &sql.DatabaseArgs{
			Instance: instance.Name,
			Name:     pulumi.String("db"),
		})
		if err != nil {
			return err
		}
		pwd, err := random.NewPassword(ctx, "pwd", &random.PasswordArgs{
			Length:  16,
			Special: false,
		})
		if err != nil {
			return err
		}
		user, err := sql.NewUser(ctx, "user", &sql.UserArgs{
			Name:     pulumi.String("user"),
			Instance: instance.Name,
			Password: pwd.Result,
		})
		if err != nil {
			return err
		}
		_, err = bigquery.NewConnection(ctx, "connection", &bigquery.ConnectionArgs{
			FriendlyName: pulumi.String("πŸ‘‹"),
			Description:  pulumi.String("a riveting description"),
			Location:     pulumi.String("US"),
			CloudSql: &bigquery.ConnectionCloudSqlArgs{
				InstanceId: instance.ConnectionName,
				Database:   db.Name,
				Type:       pulumi.String("POSTGRES"),
				Credential: &bigquery.ConnectionCloudSqlCredentialArgs{
					Username: user.Name,
					Password: user.Password,
				},
			},
		})
		if err != nil {
			return err
		}
		return nil
	})
}

using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Gcp = Pulumi.Gcp;
using Random = Pulumi.Random;

return await Deployment.RunAsync(() => 
{
    var instance = new Gcp.Sql.DatabaseInstance("instance", new()
    {
        Name = "my-database-instance",
        DatabaseVersion = "POSTGRES_11",
        Region = "us-central1",
        Settings = new Gcp.Sql.Inputs.DatabaseInstanceSettingsArgs
        {
            Tier = "db-f1-micro",
        },
        DeletionProtection = true,
    });

    var db = new Gcp.Sql.Database("db", new()
    {
        Instance = instance.Name,
        Name = "db",
    });

    var pwd = new Random.Index.Password("pwd", new()
    {
        Length = 16,
        Special = false,
    });

    var user = new Gcp.Sql.User("user", new()
    {
        Name = "user",
        Instance = instance.Name,
        Password = pwd.Result,
    });

    var connection = new Gcp.BigQuery.Connection("connection", new()
    {
        FriendlyName = "πŸ‘‹",
        Description = "a riveting description",
        Location = "US",
        CloudSql = new Gcp.BigQuery.Inputs.ConnectionCloudSqlArgs
        {
            InstanceId = instance.ConnectionName,
            Database = db.Name,
            Type = "POSTGRES",
            Credential = new Gcp.BigQuery.Inputs.ConnectionCloudSqlCredentialArgs
            {
                Username = user.Name,
                Password = user.Password,
            },
        },
    });

});

package generated_program;

import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.gcp.sql.DatabaseInstance;
import com.pulumi.gcp.sql.DatabaseInstanceArgs;
import com.pulumi.gcp.sql.inputs.DatabaseInstanceSettingsArgs;
import com.pulumi.gcp.sql.Database;
import com.pulumi.gcp.sql.DatabaseArgs;
import com.pulumi.random.Password;
import com.pulumi.random.PasswordArgs;
import com.pulumi.gcp.sql.User;
import com.pulumi.gcp.sql.UserArgs;
import com.pulumi.gcp.bigquery.Connection;
import com.pulumi.gcp.bigquery.ConnectionArgs;
import com.pulumi.gcp.bigquery.inputs.ConnectionCloudSqlArgs;
import com.pulumi.gcp.bigquery.inputs.ConnectionCloudSqlCredentialArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;

public class App {
    public static void main(String[] args) {
        Pulumi.run(App::stack);
    }

    public static void stack(Context ctx) {
        var instance = new DatabaseInstance("instance", DatabaseInstanceArgs.builder()
            .name("my-database-instance")
            .databaseVersion("POSTGRES_11")
            .region("us-central1")
            .settings(DatabaseInstanceSettingsArgs.builder()
                .tier("db-f1-micro")
                .build())
            .deletionProtection(true)
            .build());

        var db = new Database("db", DatabaseArgs.builder()
            .instance(instance.name())
            .name("db")
            .build());

        var pwd = new Password("pwd", PasswordArgs.builder()
            .length(16)
            .special(false)
            .build());

        var user = new User("user", UserArgs.builder()
            .name("user")
            .instance(instance.name())
            .password(pwd.result())
            .build());

        var connection = new Connection("connection", ConnectionArgs.builder()
            .friendlyName("πŸ‘‹")
            .description("a riveting description")
            .location("US")
            .cloudSql(ConnectionCloudSqlArgs.builder()
                .instanceId(instance.connectionName())
                .database(db.name())
                .type("POSTGRES")
                .credential(ConnectionCloudSqlCredentialArgs.builder()
                    .username(user.name())
                    .password(user.password())
                    .build())
                .build())
            .build());

    }
}

resources:
  instance:
    type: gcp:sql:DatabaseInstance
    properties:
      name: my-database-instance
      databaseVersion: POSTGRES_11
      region: us-central1
      settings:
        tier: db-f1-micro
      deletionProtection: true
  db:
    type: gcp:sql:Database
    properties:
      instance: ${instance.name}
      name: db
  pwd:
    type: random:Password
    properties:
      length: 16
      special: false
  user:
    type: gcp:sql:User
    properties:
      name: user
      instance: ${instance.name}
      password: ${pwd.result}
  connection:
    type: gcp:bigquery:Connection
    properties:
      friendlyName: "\U0001F44B"
      description: a riveting description
      location: US
      cloudSql:
        instanceId: ${instance.connectionName}
        database: ${db.name}
        type: POSTGRES
        credential:
          username: ${user.name}
          password: ${user.password}

The cloudSql property stores the Cloud SQL instance connection details. The instanceId references the Cloud SQL instance’s connection name, database specifies which database to query, and type indicates the database engine (POSTGRES, MYSQL). The credential block stores the username and password for authentication. BigQuery uses these credentials to establish connections when executing federated queries.

Query data in AWS using cross-cloud federation

BigQuery Omni extends BigQuery to AWS, letting you query S3 data or Redshift tables from BigQuery.

import * as pulumi from "@pulumi/pulumi";
import * as gcp from "@pulumi/gcp";

const connection = new gcp.bigquery.Connection("connection", {
    connectionId: "my-connection",
    location: "aws-us-east-1",
    friendlyName: "πŸ‘‹",
    description: "a riveting description",
    aws: {
        accessRole: {
            iamRoleId: "arn:aws:iam::999999999999:role/omnirole",
        },
    },
});

import pulumi
import pulumi_gcp as gcp

connection = gcp.bigquery.Connection("connection",
    connection_id="my-connection",
    location="aws-us-east-1",
    friendly_name="πŸ‘‹",
    description="a riveting description",
    aws={
        "access_role": {
            "iam_role_id": "arn:aws:iam::999999999999:role/omnirole",
        },
    })

package main

import (
	"github.com/pulumi/pulumi-gcp/sdk/v9/go/gcp/bigquery"
	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)

func main() {
	pulumi.Run(func(ctx *pulumi.Context) error {
		_, err := bigquery.NewConnection(ctx, "connection", &bigquery.ConnectionArgs{
			ConnectionId: pulumi.String("my-connection"),
			Location:     pulumi.String("aws-us-east-1"),
			FriendlyName: pulumi.String("πŸ‘‹"),
			Description:  pulumi.String("a riveting description"),
			Aws: &bigquery.ConnectionAwsArgs{
				AccessRole: &bigquery.ConnectionAwsAccessRoleArgs{
					IamRoleId: pulumi.String("arn:aws:iam::999999999999:role/omnirole"),
				},
			},
		})
		if err != nil {
			return err
		}
		return nil
	})
}

using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Gcp = Pulumi.Gcp;

return await Deployment.RunAsync(() => 
{
    var connection = new Gcp.BigQuery.Connection("connection", new()
    {
        ConnectionId = "my-connection",
        Location = "aws-us-east-1",
        FriendlyName = "πŸ‘‹",
        Description = "a riveting description",
        Aws = new Gcp.BigQuery.Inputs.ConnectionAwsArgs
        {
            AccessRole = new Gcp.BigQuery.Inputs.ConnectionAwsAccessRoleArgs
            {
                IamRoleId = "arn:aws:iam::999999999999:role/omnirole",
            },
        },
    });

});

package generated_program;

import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.gcp.bigquery.Connection;
import com.pulumi.gcp.bigquery.ConnectionArgs;
import com.pulumi.gcp.bigquery.inputs.ConnectionAwsArgs;
import com.pulumi.gcp.bigquery.inputs.ConnectionAwsAccessRoleArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;

public class App {
    public static void main(String[] args) {
        Pulumi.run(App::stack);
    }

    public static void stack(Context ctx) {
        var connection = new Connection("connection", ConnectionArgs.builder()
            .connectionId("my-connection")
            .location("aws-us-east-1")
            .friendlyName("πŸ‘‹")
            .description("a riveting description")
            .aws(ConnectionAwsArgs.builder()
                .accessRole(ConnectionAwsAccessRoleArgs.builder()
                    .iamRoleId("arn:aws:iam::999999999999:role/omnirole")
                    .build())
                .build())
            .build());

    }
}

resources:
  connection:
    type: gcp:bigquery:Connection
    properties:
      connectionId: my-connection
      location: aws-us-east-1
      friendlyName: "\U0001F44B"
      description: a riveting description
      aws:
        accessRole:
          iamRoleId: arn:aws:iam::999999999999:role/omnirole

The aws property configures cross-cloud access through an IAM role. The iamRoleId references an AWS IAM role ARN that BigQuery assumes to access AWS resources. The location must use an AWS region identifier (aws-us-east-1). This connection type requires a trust policy in AWS that allows BigQuery’s service account to assume the role.

Query data in Azure using cross-cloud federation

BigQuery Omni also supports Azure, enabling queries against Azure Blob Storage or Azure Synapse.

import * as pulumi from "@pulumi/pulumi";
import * as gcp from "@pulumi/gcp";

const connection = new gcp.bigquery.Connection("connection", {
    connectionId: "my-connection",
    location: "azure-eastus2",
    friendlyName: "πŸ‘‹",
    description: "a riveting description",
    azure: {
        customerTenantId: "customer-tenant-id",
        federatedApplicationClientId: "b43eeeee-eeee-eeee-eeee-a480155501ce",
    },
});

import pulumi
import pulumi_gcp as gcp

connection = gcp.bigquery.Connection("connection",
    connection_id="my-connection",
    location="azure-eastus2",
    friendly_name="πŸ‘‹",
    description="a riveting description",
    azure={
        "customer_tenant_id": "customer-tenant-id",
        "federated_application_client_id": "b43eeeee-eeee-eeee-eeee-a480155501ce",
    })

package main

import (
	"github.com/pulumi/pulumi-gcp/sdk/v9/go/gcp/bigquery"
	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)

func main() {
	pulumi.Run(func(ctx *pulumi.Context) error {
		_, err := bigquery.NewConnection(ctx, "connection", &bigquery.ConnectionArgs{
			ConnectionId: pulumi.String("my-connection"),
			Location:     pulumi.String("azure-eastus2"),
			FriendlyName: pulumi.String("πŸ‘‹"),
			Description:  pulumi.String("a riveting description"),
			Azure: &bigquery.ConnectionAzureArgs{
				CustomerTenantId:             pulumi.String("customer-tenant-id"),
				FederatedApplicationClientId: pulumi.String("b43eeeee-eeee-eeee-eeee-a480155501ce"),
			},
		})
		if err != nil {
			return err
		}
		return nil
	})
}

using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Gcp = Pulumi.Gcp;

return await Deployment.RunAsync(() => 
{
    var connection = new Gcp.BigQuery.Connection("connection", new()
    {
        ConnectionId = "my-connection",
        Location = "azure-eastus2",
        FriendlyName = "πŸ‘‹",
        Description = "a riveting description",
        Azure = new Gcp.BigQuery.Inputs.ConnectionAzureArgs
        {
            CustomerTenantId = "customer-tenant-id",
            FederatedApplicationClientId = "b43eeeee-eeee-eeee-eeee-a480155501ce",
        },
    });

});

package generated_program;

import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.gcp.bigquery.Connection;
import com.pulumi.gcp.bigquery.ConnectionArgs;
import com.pulumi.gcp.bigquery.inputs.ConnectionAzureArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;

public class App {
    public static void main(String[] args) {
        Pulumi.run(App::stack);
    }

    public static void stack(Context ctx) {
        var connection = new Connection("connection", ConnectionArgs.builder()
            .connectionId("my-connection")
            .location("azure-eastus2")
            .friendlyName("πŸ‘‹")
            .description("a riveting description")
            .azure(ConnectionAzureArgs.builder()
                .customerTenantId("customer-tenant-id")
                .federatedApplicationClientId("b43eeeee-eeee-eeee-eeee-a480155501ce")
                .build())
            .build());

    }
}

resources:
  connection:
    type: gcp:bigquery:Connection
    properties:
      connectionId: my-connection
      location: azure-eastus2
      friendlyName: "\U0001F44B"
      description: a riveting description
      azure:
        customerTenantId: customer-tenant-id
        federatedApplicationClientId: b43eeeee-eeee-eeee-eeee-a480155501ce

The azure property configures Azure AD federation. The customerTenantId identifies your Azure tenant, and federatedApplicationClientId references the application configured in Azure AD. The location must use an Azure region identifier (azure-eastus2). This connection type uses workload identity federation to authenticate without storing credentials.

Query Cloud Spanner databases

Federated queries can join BigQuery tables with Cloud Spanner data, useful for combining analytical workloads with transactional data.

import * as pulumi from "@pulumi/pulumi";
import * as gcp from "@pulumi/gcp";

const connection = new gcp.bigquery.Connection("connection", {
    connectionId: "my-connection",
    location: "US",
    friendlyName: "πŸ‘‹",
    description: "a riveting description",
    cloudSpanner: {
        database: "projects/project/instances/instance/databases/database",
        databaseRole: "database_role",
    },
});

import pulumi
import pulumi_gcp as gcp

connection = gcp.bigquery.Connection("connection",
    connection_id="my-connection",
    location="US",
    friendly_name="πŸ‘‹",
    description="a riveting description",
    cloud_spanner={
        "database": "projects/project/instances/instance/databases/database",
        "database_role": "database_role",
    })

package main

import (
	"github.com/pulumi/pulumi-gcp/sdk/v9/go/gcp/bigquery"
	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)

func main() {
	pulumi.Run(func(ctx *pulumi.Context) error {
		_, err := bigquery.NewConnection(ctx, "connection", &bigquery.ConnectionArgs{
			ConnectionId: pulumi.String("my-connection"),
			Location:     pulumi.String("US"),
			FriendlyName: pulumi.String("πŸ‘‹"),
			Description:  pulumi.String("a riveting description"),
			CloudSpanner: &bigquery.ConnectionCloudSpannerArgs{
				Database:     pulumi.String("projects/project/instances/instance/databases/database"),
				DatabaseRole: pulumi.String("database_role"),
			},
		})
		if err != nil {
			return err
		}
		return nil
	})
}

using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Gcp = Pulumi.Gcp;

return await Deployment.RunAsync(() => 
{
    var connection = new Gcp.BigQuery.Connection("connection", new()
    {
        ConnectionId = "my-connection",
        Location = "US",
        FriendlyName = "πŸ‘‹",
        Description = "a riveting description",
        CloudSpanner = new Gcp.BigQuery.Inputs.ConnectionCloudSpannerArgs
        {
            Database = "projects/project/instances/instance/databases/database",
            DatabaseRole = "database_role",
        },
    });

});

package generated_program;

import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.gcp.bigquery.Connection;
import com.pulumi.gcp.bigquery.ConnectionArgs;
import com.pulumi.gcp.bigquery.inputs.ConnectionCloudSpannerArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;

public class App {
    public static void main(String[] args) {
        Pulumi.run(App::stack);
    }

    public static void stack(Context ctx) {
        var connection = new Connection("connection", ConnectionArgs.builder()
            .connectionId("my-connection")
            .location("US")
            .friendlyName("πŸ‘‹")
            .description("a riveting description")
            .cloudSpanner(ConnectionCloudSpannerArgs.builder()
                .database("projects/project/instances/instance/databases/database")
                .databaseRole("database_role")
                .build())
            .build());

    }
}

resources:
  connection:
    type: gcp:bigquery:Connection
    properties:
      connectionId: my-connection
      location: US
      friendlyName: "\U0001F44B"
      description: a riveting description
      cloudSpanner:
        database: projects/project/instances/instance/databases/database
        databaseRole: database_role

The cloudSpanner property connects to a Spanner database. The database property uses the full resource path format (projects/project/instances/instance/databases/database), and databaseRole specifies which Spanner database role to use for access control. BigQuery queries execute against Spanner using this role’s permissions.

Enable Spanner Data Boost for query performance

Data Boost provides independent compute resources for analytical queries against Spanner, preventing impact on transactional workloads.

import * as pulumi from "@pulumi/pulumi";
import * as gcp from "@pulumi/gcp";

const connection = new gcp.bigquery.Connection("connection", {
    connectionId: "my-connection",
    location: "US",
    friendlyName: "πŸ‘‹",
    description: "a riveting description",
    cloudSpanner: {
        database: "projects/project/instances/instance/databases/database",
        useParallelism: true,
        useDataBoost: true,
        maxParallelism: 100,
    },
});

import pulumi
import pulumi_gcp as gcp

connection = gcp.bigquery.Connection("connection",
    connection_id="my-connection",
    location="US",
    friendly_name="πŸ‘‹",
    description="a riveting description",
    cloud_spanner={
        "database": "projects/project/instances/instance/databases/database",
        "use_parallelism": True,
        "use_data_boost": True,
        "max_parallelism": 100,
    })

package main

import (
	"github.com/pulumi/pulumi-gcp/sdk/v9/go/gcp/bigquery"
	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)

func main() {
	pulumi.Run(func(ctx *pulumi.Context) error {
		_, err := bigquery.NewConnection(ctx, "connection", &bigquery.ConnectionArgs{
			ConnectionId: pulumi.String("my-connection"),
			Location:     pulumi.String("US"),
			FriendlyName: pulumi.String("πŸ‘‹"),
			Description:  pulumi.String("a riveting description"),
			CloudSpanner: &bigquery.ConnectionCloudSpannerArgs{
				Database:       pulumi.String("projects/project/instances/instance/databases/database"),
				UseParallelism: pulumi.Bool(true),
				UseDataBoost:   pulumi.Bool(true),
				MaxParallelism: pulumi.Int(100),
			},
		})
		if err != nil {
			return err
		}
		return nil
	})
}

using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Gcp = Pulumi.Gcp;

return await Deployment.RunAsync(() => 
{
    var connection = new Gcp.BigQuery.Connection("connection", new()
    {
        ConnectionId = "my-connection",
        Location = "US",
        FriendlyName = "πŸ‘‹",
        Description = "a riveting description",
        CloudSpanner = new Gcp.BigQuery.Inputs.ConnectionCloudSpannerArgs
        {
            Database = "projects/project/instances/instance/databases/database",
            UseParallelism = true,
            UseDataBoost = true,
            MaxParallelism = 100,
        },
    });

});

package generated_program;

import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.gcp.bigquery.Connection;
import com.pulumi.gcp.bigquery.ConnectionArgs;
import com.pulumi.gcp.bigquery.inputs.ConnectionCloudSpannerArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;

public class App {
    public static void main(String[] args) {
        Pulumi.run(App::stack);
    }

    public static void stack(Context ctx) {
        var connection = new Connection("connection", ConnectionArgs.builder()
            .connectionId("my-connection")
            .location("US")
            .friendlyName("πŸ‘‹")
            .description("a riveting description")
            .cloudSpanner(ConnectionCloudSpannerArgs.builder()
                .database("projects/project/instances/instance/databases/database")
                .useParallelism(true)
                .useDataBoost(true)
                .maxParallelism(100)
                .build())
            .build());

    }
}

resources:
  connection:
    type: gcp:bigquery:Connection
    properties:
      connectionId: my-connection
      location: US
      friendlyName: "\U0001F44B"
      description: a riveting description
      cloudSpanner:
        database: projects/project/instances/instance/databases/database
        useParallelism: true
        useDataBoost: true
        maxParallelism: 100

This configuration extends the basic Spanner connection with performance tuning. Setting useDataBoost to true enables Data Boost, which runs queries on separate compute resources. The useParallelism property enables parallel query execution, and maxParallelism controls the degree of parallelism. These settings isolate analytical load from production traffic while improving query performance.

Execute Spark stored procedures

BigQuery can execute Apache Spark code through stored procedures, with results tracked in a Dataproc cluster’s Spark History Server.

import * as pulumi from "@pulumi/pulumi";
import * as gcp from "@pulumi/gcp";

const basic = new gcp.dataproc.Cluster("basic", {
    name: "my-connection",
    region: "us-central1",
    clusterConfig: {
        softwareConfig: {
            overrideProperties: {
                "dataproc:dataproc.allow.zero.workers": "true",
            },
        },
        masterConfig: {
            numInstances: 1,
            machineType: "e2-standard-2",
            diskConfig: {
                bootDiskSizeGb: 35,
            },
        },
    },
});
const connection = new gcp.bigquery.Connection("connection", {
    connectionId: "my-connection",
    location: "US",
    friendlyName: "πŸ‘‹",
    description: "a riveting description",
    spark: {
        sparkHistoryServerConfig: {
            dataprocCluster: basic.id,
        },
    },
});

import pulumi
import pulumi_gcp as gcp

basic = gcp.dataproc.Cluster("basic",
    name="my-connection",
    region="us-central1",
    cluster_config={
        "software_config": {
            "override_properties": {
                "dataproc:dataproc.allow.zero.workers": "true",
            },
        },
        "master_config": {
            "num_instances": 1,
            "machine_type": "e2-standard-2",
            "disk_config": {
                "boot_disk_size_gb": 35,
            },
        },
    })
connection = gcp.bigquery.Connection("connection",
    connection_id="my-connection",
    location="US",
    friendly_name="πŸ‘‹",
    description="a riveting description",
    spark={
        "spark_history_server_config": {
            "dataproc_cluster": basic.id,
        },
    })

package main

import (
	"github.com/pulumi/pulumi-gcp/sdk/v9/go/gcp/bigquery"
	"github.com/pulumi/pulumi-gcp/sdk/v9/go/gcp/dataproc"
	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)

func main() {
	pulumi.Run(func(ctx *pulumi.Context) error {
		basic, err := dataproc.NewCluster(ctx, "basic", &dataproc.ClusterArgs{
			Name:   pulumi.String("my-connection"),
			Region: pulumi.String("us-central1"),
			ClusterConfig: &dataproc.ClusterClusterConfigArgs{
				SoftwareConfig: &dataproc.ClusterClusterConfigSoftwareConfigArgs{
					OverrideProperties: pulumi.StringMap{
						"dataproc:dataproc.allow.zero.workers": pulumi.String("true"),
					},
				},
				MasterConfig: &dataproc.ClusterClusterConfigMasterConfigArgs{
					NumInstances: pulumi.Int(1),
					MachineType:  pulumi.String("e2-standard-2"),
					DiskConfig: &dataproc.ClusterClusterConfigMasterConfigDiskConfigArgs{
						BootDiskSizeGb: pulumi.Int(35),
					},
				},
			},
		})
		if err != nil {
			return err
		}
		_, err = bigquery.NewConnection(ctx, "connection", &bigquery.ConnectionArgs{
			ConnectionId: pulumi.String("my-connection"),
			Location:     pulumi.String("US"),
			FriendlyName: pulumi.String("πŸ‘‹"),
			Description:  pulumi.String("a riveting description"),
			Spark: &bigquery.ConnectionSparkArgs{
				SparkHistoryServerConfig: &bigquery.ConnectionSparkSparkHistoryServerConfigArgs{
					DataprocCluster: basic.ID(),
				},
			},
		})
		if err != nil {
			return err
		}
		return nil
	})
}

using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Gcp = Pulumi.Gcp;

return await Deployment.RunAsync(() => 
{
    var basic = new Gcp.Dataproc.Cluster("basic", new()
    {
        Name = "my-connection",
        Region = "us-central1",
        ClusterConfig = new Gcp.Dataproc.Inputs.ClusterClusterConfigArgs
        {
            SoftwareConfig = new Gcp.Dataproc.Inputs.ClusterClusterConfigSoftwareConfigArgs
            {
                OverrideProperties = 
                {
                    { "dataproc:dataproc.allow.zero.workers", "true" },
                },
            },
            MasterConfig = new Gcp.Dataproc.Inputs.ClusterClusterConfigMasterConfigArgs
            {
                NumInstances = 1,
                MachineType = "e2-standard-2",
                DiskConfig = new Gcp.Dataproc.Inputs.ClusterClusterConfigMasterConfigDiskConfigArgs
                {
                    BootDiskSizeGb = 35,
                },
            },
        },
    });

    var connection = new Gcp.BigQuery.Connection("connection", new()
    {
        ConnectionId = "my-connection",
        Location = "US",
        FriendlyName = "πŸ‘‹",
        Description = "a riveting description",
        Spark = new Gcp.BigQuery.Inputs.ConnectionSparkArgs
        {
            SparkHistoryServerConfig = new Gcp.BigQuery.Inputs.ConnectionSparkSparkHistoryServerConfigArgs
            {
                DataprocCluster = basic.Id,
            },
        },
    });

});

package generated_program;

import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.gcp.dataproc.Cluster;
import com.pulumi.gcp.dataproc.ClusterArgs;
import com.pulumi.gcp.dataproc.inputs.ClusterClusterConfigArgs;
import com.pulumi.gcp.dataproc.inputs.ClusterClusterConfigSoftwareConfigArgs;
import com.pulumi.gcp.dataproc.inputs.ClusterClusterConfigMasterConfigArgs;
import com.pulumi.gcp.dataproc.inputs.ClusterClusterConfigMasterConfigDiskConfigArgs;
import com.pulumi.gcp.bigquery.Connection;
import com.pulumi.gcp.bigquery.ConnectionArgs;
import com.pulumi.gcp.bigquery.inputs.ConnectionSparkArgs;
import com.pulumi.gcp.bigquery.inputs.ConnectionSparkSparkHistoryServerConfigArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;

public class App {
    public static void main(String[] args) {
        Pulumi.run(App::stack);
    }

    public static void stack(Context ctx) {
        var basic = new Cluster("basic", ClusterArgs.builder()
            .name("my-connection")
            .region("us-central1")
            .clusterConfig(ClusterClusterConfigArgs.builder()
                .softwareConfig(ClusterClusterConfigSoftwareConfigArgs.builder()
                    .overrideProperties(Map.of("dataproc:dataproc.allow.zero.workers", "true"))
                    .build())
                .masterConfig(ClusterClusterConfigMasterConfigArgs.builder()
                    .numInstances(1)
                    .machineType("e2-standard-2")
                    .diskConfig(ClusterClusterConfigMasterConfigDiskConfigArgs.builder()
                        .bootDiskSizeGb(35)
                        .build())
                    .build())
                .build())
            .build());

        var connection = new Connection("connection", ConnectionArgs.builder()
            .connectionId("my-connection")
            .location("US")
            .friendlyName("πŸ‘‹")
            .description("a riveting description")
            .spark(ConnectionSparkArgs.builder()
                .sparkHistoryServerConfig(ConnectionSparkSparkHistoryServerConfigArgs.builder()
                    .dataprocCluster(basic.id())
                    .build())
                .build())
            .build());

    }
}

resources:
  connection:
    type: gcp:bigquery:Connection
    properties:
      connectionId: my-connection
      location: US
      friendlyName: "\U0001F44B"
      description: a riveting description
      spark:
        sparkHistoryServerConfig:
          dataprocCluster: ${basic.id}
  basic:
    type: gcp:dataproc:Cluster
    properties:
      name: my-connection
      region: us-central1
      clusterConfig:
        softwareConfig:
          overrideProperties:
            dataproc:dataproc.allow.zero.workers: 'true'
        masterConfig:
          numInstances: 1
          machineType: e2-standard-2
          diskConfig:
            bootDiskSizeGb: 35

The spark property configures Spark execution. The sparkHistoryServerConfig block references a Dataproc cluster through dataprocCluster, which stores execution history and logs. BigQuery submits Spark jobs to this cluster when you call Spark stored procedures, and the History Server provides visibility into job execution and performance.

Beyond these examples

These snippets focus on specific connection-level features: GCP resource delegation and Cloud SQL federation, cross-cloud connections (AWS and Azure), and Cloud Spanner and Spark integration. They’re intentionally minimal rather than full data pipeline configurations.

The examples reference pre-existing infrastructure such as Cloud SQL instances, databases, and users, AWS IAM roles with trust policies, Azure tenant IDs and federated applications, Cloud Spanner databases and roles, and Dataproc clusters for Spark execution. They focus on configuring the connection rather than provisioning the underlying data sources.

To keep things focused, common connection patterns are omitted, including:

  • Customer-managed encryption keys (kmsKeyName)
  • Connection metadata (friendlyName, description)
  • Credential management and rotation
  • Location-specific constraints and region mapping

These omissions are intentional: the goal is to illustrate how each connection type is wired, not provide drop-in data integration modules. See the BigQuery Connection resource reference for all available configuration options.

Let's configure GCP BigQuery Connections

Get started with Pulumi Cloud, then follow our quick setup guide to deploy this infrastructure.

Try Pulumi Cloud for FREE

Frequently Asked Questions

Connection Types & Configuration
What types of external data sources can I connect BigQuery to?
BigQuery connections support six types: Cloud Resource (for GCP resource delegation), Cloud SQL, AWS, Azure, Cloud Spanner, and Spark (via Dataproc clusters).
How do I connect BigQuery to a Cloud SQL database?
Configure the cloudSql property with instanceId, database, type (e.g., POSTGRES), and credential containing username and password.
How do I connect BigQuery to AWS resources?
Use the aws property with accessRole.iamRoleId set to your AWS IAM role ARN (e.g., arn:aws:iam::999999999999:role/omnirole), and set location to aws-us-east-1.
How do I connect BigQuery to Azure resources?
Configure the azure property with customerTenantId and federatedApplicationClientId, and set location to azure-eastus2.
Location & Regional Requirements
Why does my Cloud SQL connection use 'US' instead of 'us-central1'?
Cloud SQL regions map to BigQuery regions: us-central1 becomes US, and europe-west1 becomes EU. Use these BigQuery region names in the location property.
What are the location restrictions for AWS and Azure connections?
AWS connections are limited to aws-us-east-1, and Azure connections are limited to azure-eastus2.
Immutability & Lifecycle
What properties can't I change after creating a connection?
The connectionId, project, and location properties are immutable and require recreating the connection to modify.
Security & Encryption
How do I enable encryption for my BigQuery connection?
Set the kmsKeyName property to your Cloud KMS key path in the format: projects/[project]/locations/[region]/keyRings/[ring]/cryptoKeys/[key].
How can I verify if my connection has credentials configured?
Check the hasCredential output property, which returns true if credentials are assigned to the connection.
Performance & Optimization
What's Cloud Spanner Data Boost and when should I use it?
Data Boost improves Cloud Spanner query performance. Enable it by setting useDataBoost: true along with useParallelism: true and maxParallelism (e.g., 100) in the cloudSpanner configuration.

Using a different cloud?

Explore analytics guides for other cloud providers:

AWS Guides Azure Guides