This resource to configure root bucket new workspaces within AWS.
This resource can only be used with an account-level provider!
It is important to understand that this will require you to configure your provider separately for the multiple workspaces resources. This will point to https://accounts.cloud.databricks.com for the HOST and it will use basic auth as that is the only authentication method available for multiple workspaces api.
Please follow this complete runnable example with new VPC and new workspace setup. Please pay special attention to the fact that there you have two different instances of a databricks provider - one for deploying workspaces (with host="https://accounts.cloud.databricks.com/") and another for the workspace you’ve created with databricks.MwsWorkspaces resource. If you want both creation of workspaces & clusters within workspace within the same terraform module (essentially same directory), you should use the provider aliasing feature of Pulumi. We strongly recommend having one terraform module for creation of workspace + PAT token and the rest in different modules.
Example Usage
import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
import * as databricks from "@pulumi/databricks";
const config = new pulumi.Config();
// Account Id that could be found in the top right corner of https://accounts.cloud.databricks.com/
const databricksAccountId = config.requireObject<any>("databricksAccountId");
const rootStorageBucket = new aws.index.S3Bucket("root_storage_bucket", {
bucket: `${prefix}-rootbucket`,
acl: "private",
});
const rootVersioning = new aws.index.S3BucketVersioning("root_versioning", {
bucket: rootStorageBucket.id,
versioningConfiguration: [{
status: "Disabled",
}],
});
const _this = new databricks.MwsStorageConfigurations("this", {
accountId: databricksAccountId,
storageConfigurationName: `${prefix}-storage`,
bucketName: rootStorageBucket.bucket,
});
import pulumi
import pulumi_aws as aws
import pulumi_databricks as databricks
config = pulumi.Config()
# Account Id that could be found in the top right corner of https://accounts.cloud.databricks.com/
databricks_account_id = config.require_object("databricksAccountId")
root_storage_bucket = aws.index.S3Bucket("root_storage_bucket",
bucket=f{prefix}-rootbucket,
acl=private)
root_versioning = aws.index.S3BucketVersioning("root_versioning",
bucket=root_storage_bucket.id,
versioning_configuration=[{
status: Disabled,
}])
this = databricks.MwsStorageConfigurations("this",
account_id=databricks_account_id,
storage_configuration_name=f"{prefix}-storage",
bucket_name=root_storage_bucket["bucket"])
package main
import (
"fmt"
"github.com/pulumi/pulumi-aws/sdk/v7/go/aws"
"github.com/pulumi/pulumi-databricks/sdk/go/databricks"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi/config"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
cfg := config.New(ctx, "")
// Account Id that could be found in the top right corner of https://accounts.cloud.databricks.com/
databricksAccountId := cfg.RequireObject("databricksAccountId")
rootStorageBucket, err := aws.NewS3Bucket(ctx, "root_storage_bucket", &aws.S3BucketArgs{
Bucket: fmt.Sprintf("%v-rootbucket", prefix),
Acl: "private",
})
if err != nil {
return err
}
_, err = aws.NewS3BucketVersioning(ctx, "root_versioning", &aws.S3BucketVersioningArgs{
Bucket: rootStorageBucket.Id,
VersioningConfiguration: []map[string]interface{}{
map[string]interface{}{
"status": "Disabled",
},
},
})
if err != nil {
return err
}
_, err = databricks.NewMwsStorageConfigurations(ctx, "this", &databricks.MwsStorageConfigurationsArgs{
AccountId: pulumi.Any(databricksAccountId),
StorageConfigurationName: pulumi.Sprintf("%v-storage", prefix),
BucketName: rootStorageBucket.Bucket,
})
if err != nil {
return err
}
return nil
})
}
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Aws = Pulumi.Aws;
using Databricks = Pulumi.Databricks;
return await Deployment.RunAsync(() =>
{
var config = new Config();
// Account Id that could be found in the top right corner of https://accounts.cloud.databricks.com/
var databricksAccountId = config.RequireObject<dynamic>("databricksAccountId");
var rootStorageBucket = new Aws.Index.S3Bucket("root_storage_bucket", new()
{
Bucket = $"{prefix}-rootbucket",
Acl = "private",
});
var rootVersioning = new Aws.Index.S3BucketVersioning("root_versioning", new()
{
Bucket = rootStorageBucket.Id,
VersioningConfiguration = new[]
{
{
{ "status", "Disabled" },
},
},
});
var @this = new Databricks.MwsStorageConfigurations("this", new()
{
AccountId = databricksAccountId,
StorageConfigurationName = $"{prefix}-storage",
BucketName = rootStorageBucket.Bucket,
});
});
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.S3Bucket;
import com.pulumi.aws.S3BucketArgs;
import com.pulumi.aws.S3BucketVersioning;
import com.pulumi.aws.S3BucketVersioningArgs;
import com.pulumi.databricks.MwsStorageConfigurations;
import com.pulumi.databricks.MwsStorageConfigurationsArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
final var config = ctx.config();
final var databricksAccountId = config.get("databricksAccountId");
var rootStorageBucket = new S3Bucket("rootStorageBucket", S3BucketArgs.builder()
.bucket(String.format("%s-rootbucket", prefix))
.acl("private")
.build());
var rootVersioning = new S3BucketVersioning("rootVersioning", S3BucketVersioningArgs.builder()
.bucket(rootStorageBucket.id())
.versioningConfiguration(List.of(Map.of("status", "Disabled")))
.build());
var this_ = new MwsStorageConfigurations("this", MwsStorageConfigurationsArgs.builder()
.accountId(databricksAccountId)
.storageConfigurationName(String.format("%s-storage", prefix))
.bucketName(rootStorageBucket.bucket())
.build());
}
}
configuration:
databricksAccountId:
type: dynamic
resources:
rootStorageBucket:
type: aws:S3Bucket
name: root_storage_bucket
properties:
bucket: ${prefix}-rootbucket
acl: private
rootVersioning:
type: aws:S3BucketVersioning
name: root_versioning
properties:
bucket: ${rootStorageBucket.id}
versioningConfiguration:
- status: Disabled
this:
type: databricks:MwsStorageConfigurations
properties:
accountId: ${databricksAccountId}
storageConfigurationName: ${prefix}-storage
bucketName: ${rootStorageBucket.bucket}
Example Usage with Role ARN
When sharing an S3 bucket between root storage and a Unity Catalog metastore, you can specify a role ARN:
import * as pulumi from "@pulumi/pulumi";
import * as databricks from "@pulumi/databricks";
const _this = new databricks.MwsStorageConfigurations("this", {
accountId: databricksAccountId,
storageConfigurationName: `${prefix}-storage`,
bucketName: rootStorageBucket.bucket,
roleArn: unityCatalogRole.arn,
});
import pulumi
import pulumi_databricks as databricks
this = databricks.MwsStorageConfigurations("this",
account_id=databricks_account_id,
storage_configuration_name=f"{prefix}-storage",
bucket_name=root_storage_bucket["bucket"],
role_arn=unity_catalog_role["arn"])
package main
import (
"github.com/pulumi/pulumi-databricks/sdk/go/databricks"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
_, err := databricks.NewMwsStorageConfigurations(ctx, "this", &databricks.MwsStorageConfigurationsArgs{
AccountId: pulumi.Any(databricksAccountId),
StorageConfigurationName: pulumi.Sprintf("%v-storage", prefix),
BucketName: pulumi.Any(rootStorageBucket.Bucket),
RoleArn: pulumi.Any(unityCatalogRole.Arn),
})
if err != nil {
return err
}
return nil
})
}
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Databricks = Pulumi.Databricks;
return await Deployment.RunAsync(() =>
{
var @this = new Databricks.MwsStorageConfigurations("this", new()
{
AccountId = databricksAccountId,
StorageConfigurationName = $"{prefix}-storage",
BucketName = rootStorageBucket.Bucket,
RoleArn = unityCatalogRole.Arn,
});
});
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.databricks.MwsStorageConfigurations;
import com.pulumi.databricks.MwsStorageConfigurationsArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var this_ = new MwsStorageConfigurations("this", MwsStorageConfigurationsArgs.builder()
.accountId(databricksAccountId)
.storageConfigurationName(String.format("%s-storage", prefix))
.bucketName(rootStorageBucket.bucket())
.roleArn(unityCatalogRole.arn())
.build());
}
}
resources:
this:
type: databricks:MwsStorageConfigurations
properties:
accountId: ${databricksAccountId}
storageConfigurationName: ${prefix}-storage
bucketName: ${rootStorageBucket.bucket}
roleArn: ${unityCatalogRole.arn}
Related Resources
The following resources are used in the same context:
- Provisioning Databricks on AWS guide.
- Provisioning Databricks on AWS with Private Link guide. * databricks.MwsCredentials to configure the cross-account role for creation of new workspaces within AWS. * databricks.MwsCustomerManagedKeys to configure KMS keys for new workspaces within AWS. * databricks.MwsLogDelivery to configure delivery of billable usage logs and audit logs. * databricks.MwsNetworks to configure VPC & subnets for new workspaces within AWS. * databricks.MwsWorkspaces to set up AWS and GCP workspaces.
Create MwsStorageConfigurations Resource
Resources are created with functions called constructors. To learn more about declaring and configuring resources, see Resources.
Constructor syntax
new MwsStorageConfigurations(name: string, args: MwsStorageConfigurationsArgs, opts?: CustomResourceOptions);@overload
def MwsStorageConfigurations(resource_name: str,
args: MwsStorageConfigurationsArgs,
opts: Optional[ResourceOptions] = None)
@overload
def MwsStorageConfigurations(resource_name: str,
opts: Optional[ResourceOptions] = None,
account_id: Optional[str] = None,
bucket_name: Optional[str] = None,
storage_configuration_name: Optional[str] = None,
role_arn: Optional[str] = None)func NewMwsStorageConfigurations(ctx *Context, name string, args MwsStorageConfigurationsArgs, opts ...ResourceOption) (*MwsStorageConfigurations, error)public MwsStorageConfigurations(string name, MwsStorageConfigurationsArgs args, CustomResourceOptions? opts = null)
public MwsStorageConfigurations(String name, MwsStorageConfigurationsArgs args)
public MwsStorageConfigurations(String name, MwsStorageConfigurationsArgs args, CustomResourceOptions options)
type: databricks:MwsStorageConfigurations
properties: # The arguments to resource properties.
options: # Bag of options to control resource's behavior.
Parameters
- name string
- The unique name of the resource.
- args MwsStorageConfigurationsArgs
- The arguments to resource properties.
- opts CustomResourceOptions
- Bag of options to control resource's behavior.
- resource_name str
- The unique name of the resource.
- args MwsStorageConfigurationsArgs
- The arguments to resource properties.
- opts ResourceOptions
- Bag of options to control resource's behavior.
- ctx Context
- Context object for the current deployment.
- name string
- The unique name of the resource.
- args MwsStorageConfigurationsArgs
- The arguments to resource properties.
- opts ResourceOption
- Bag of options to control resource's behavior.
- name string
- The unique name of the resource.
- args MwsStorageConfigurationsArgs
- The arguments to resource properties.
- opts CustomResourceOptions
- Bag of options to control resource's behavior.
- name String
- The unique name of the resource.
- args MwsStorageConfigurationsArgs
- The arguments to resource properties.
- options CustomResourceOptions
- Bag of options to control resource's behavior.
Constructor example
The following reference example uses placeholder values for all input properties.
var mwsStorageConfigurationsResource = new Databricks.MwsStorageConfigurations("mwsStorageConfigurationsResource", new()
{
AccountId = "string",
BucketName = "string",
StorageConfigurationName = "string",
RoleArn = "string",
});
example, err := databricks.NewMwsStorageConfigurations(ctx, "mwsStorageConfigurationsResource", &databricks.MwsStorageConfigurationsArgs{
AccountId: pulumi.String("string"),
BucketName: pulumi.String("string"),
StorageConfigurationName: pulumi.String("string"),
RoleArn: pulumi.String("string"),
})
var mwsStorageConfigurationsResource = new MwsStorageConfigurations("mwsStorageConfigurationsResource", MwsStorageConfigurationsArgs.builder()
.accountId("string")
.bucketName("string")
.storageConfigurationName("string")
.roleArn("string")
.build());
mws_storage_configurations_resource = databricks.MwsStorageConfigurations("mwsStorageConfigurationsResource",
account_id="string",
bucket_name="string",
storage_configuration_name="string",
role_arn="string")
const mwsStorageConfigurationsResource = new databricks.MwsStorageConfigurations("mwsStorageConfigurationsResource", {
accountId: "string",
bucketName: "string",
storageConfigurationName: "string",
roleArn: "string",
});
type: databricks:MwsStorageConfigurations
properties:
accountId: string
bucketName: string
roleArn: string
storageConfigurationName: string
MwsStorageConfigurations Resource Properties
To learn more about resource properties and how to use them, see Inputs and Outputs in the Architecture and Concepts docs.
Inputs
In Python, inputs that are objects can be passed either as argument classes or as dictionary literals.
The MwsStorageConfigurations resource accepts the following input properties:
- Account
Id string - Account Id that could be found in the top right corner of Accounts Console
- Bucket
Name string - name of AWS S3 bucket
- Storage
Configuration stringName name under which this storage configuration is stored
The following arguments are optional:
- Role
Arn string - The ARN of the IAM role that Databricks will assume to access the S3 bucket. This allows sharing an S3 bucket between root storage and the default catalog for a workspace. See the Databricks API documentation for more details.
- Account
Id string - Account Id that could be found in the top right corner of Accounts Console
- Bucket
Name string - name of AWS S3 bucket
- Storage
Configuration stringName name under which this storage configuration is stored
The following arguments are optional:
- Role
Arn string - The ARN of the IAM role that Databricks will assume to access the S3 bucket. This allows sharing an S3 bucket between root storage and the default catalog for a workspace. See the Databricks API documentation for more details.
- account
Id String - Account Id that could be found in the top right corner of Accounts Console
- bucket
Name String - name of AWS S3 bucket
- storage
Configuration StringName name under which this storage configuration is stored
The following arguments are optional:
- role
Arn String - The ARN of the IAM role that Databricks will assume to access the S3 bucket. This allows sharing an S3 bucket between root storage and the default catalog for a workspace. See the Databricks API documentation for more details.
- account
Id string - Account Id that could be found in the top right corner of Accounts Console
- bucket
Name string - name of AWS S3 bucket
- storage
Configuration stringName name under which this storage configuration is stored
The following arguments are optional:
- role
Arn string - The ARN of the IAM role that Databricks will assume to access the S3 bucket. This allows sharing an S3 bucket between root storage and the default catalog for a workspace. See the Databricks API documentation for more details.
- account_
id str - Account Id that could be found in the top right corner of Accounts Console
- bucket_
name str - name of AWS S3 bucket
- storage_
configuration_ strname name under which this storage configuration is stored
The following arguments are optional:
- role_
arn str - The ARN of the IAM role that Databricks will assume to access the S3 bucket. This allows sharing an S3 bucket between root storage and the default catalog for a workspace. See the Databricks API documentation for more details.
- account
Id String - Account Id that could be found in the top right corner of Accounts Console
- bucket
Name String - name of AWS S3 bucket
- storage
Configuration StringName name under which this storage configuration is stored
The following arguments are optional:
- role
Arn String - The ARN of the IAM role that Databricks will assume to access the S3 bucket. This allows sharing an S3 bucket between root storage and the default catalog for a workspace. See the Databricks API documentation for more details.
Outputs
All input properties are implicitly available as output properties. Additionally, the MwsStorageConfigurations resource produces the following output properties:
- Creation
Time int - Id string
- The provider-assigned unique ID for this managed resource.
- Storage
Configuration stringId - (String) id of storage config to be used for
databricks_mws_workspaceresource.
- Creation
Time int - Id string
- The provider-assigned unique ID for this managed resource.
- Storage
Configuration stringId - (String) id of storage config to be used for
databricks_mws_workspaceresource.
- creation
Time Integer - id String
- The provider-assigned unique ID for this managed resource.
- storage
Configuration StringId - (String) id of storage config to be used for
databricks_mws_workspaceresource.
- creation
Time number - id string
- The provider-assigned unique ID for this managed resource.
- storage
Configuration stringId - (String) id of storage config to be used for
databricks_mws_workspaceresource.
- creation_
time int - id str
- The provider-assigned unique ID for this managed resource.
- storage_
configuration_ strid - (String) id of storage config to be used for
databricks_mws_workspaceresource.
- creation
Time Number - id String
- The provider-assigned unique ID for this managed resource.
- storage
Configuration StringId - (String) id of storage config to be used for
databricks_mws_workspaceresource.
Look up Existing MwsStorageConfigurations Resource
Get an existing MwsStorageConfigurations resource’s state with the given name, ID, and optional extra properties used to qualify the lookup.
public static get(name: string, id: Input<ID>, state?: MwsStorageConfigurationsState, opts?: CustomResourceOptions): MwsStorageConfigurations@staticmethod
def get(resource_name: str,
id: str,
opts: Optional[ResourceOptions] = None,
account_id: Optional[str] = None,
bucket_name: Optional[str] = None,
creation_time: Optional[int] = None,
role_arn: Optional[str] = None,
storage_configuration_id: Optional[str] = None,
storage_configuration_name: Optional[str] = None) -> MwsStorageConfigurationsfunc GetMwsStorageConfigurations(ctx *Context, name string, id IDInput, state *MwsStorageConfigurationsState, opts ...ResourceOption) (*MwsStorageConfigurations, error)public static MwsStorageConfigurations Get(string name, Input<string> id, MwsStorageConfigurationsState? state, CustomResourceOptions? opts = null)public static MwsStorageConfigurations get(String name, Output<String> id, MwsStorageConfigurationsState state, CustomResourceOptions options)resources: _: type: databricks:MwsStorageConfigurations get: id: ${id}- name
- The unique name of the resulting resource.
- id
- The unique provider ID of the resource to lookup.
- state
- Any extra arguments used during the lookup.
- opts
- A bag of options that control this resource's behavior.
- resource_name
- The unique name of the resulting resource.
- id
- The unique provider ID of the resource to lookup.
- name
- The unique name of the resulting resource.
- id
- The unique provider ID of the resource to lookup.
- state
- Any extra arguments used during the lookup.
- opts
- A bag of options that control this resource's behavior.
- name
- The unique name of the resulting resource.
- id
- The unique provider ID of the resource to lookup.
- state
- Any extra arguments used during the lookup.
- opts
- A bag of options that control this resource's behavior.
- name
- The unique name of the resulting resource.
- id
- The unique provider ID of the resource to lookup.
- state
- Any extra arguments used during the lookup.
- opts
- A bag of options that control this resource's behavior.
- Account
Id string - Account Id that could be found in the top right corner of Accounts Console
- Bucket
Name string - name of AWS S3 bucket
- Creation
Time int - Role
Arn string - The ARN of the IAM role that Databricks will assume to access the S3 bucket. This allows sharing an S3 bucket between root storage and the default catalog for a workspace. See the Databricks API documentation for more details.
- Storage
Configuration stringId - (String) id of storage config to be used for
databricks_mws_workspaceresource. - Storage
Configuration stringName name under which this storage configuration is stored
The following arguments are optional:
- Account
Id string - Account Id that could be found in the top right corner of Accounts Console
- Bucket
Name string - name of AWS S3 bucket
- Creation
Time int - Role
Arn string - The ARN of the IAM role that Databricks will assume to access the S3 bucket. This allows sharing an S3 bucket between root storage and the default catalog for a workspace. See the Databricks API documentation for more details.
- Storage
Configuration stringId - (String) id of storage config to be used for
databricks_mws_workspaceresource. - Storage
Configuration stringName name under which this storage configuration is stored
The following arguments are optional:
- account
Id String - Account Id that could be found in the top right corner of Accounts Console
- bucket
Name String - name of AWS S3 bucket
- creation
Time Integer - role
Arn String - The ARN of the IAM role that Databricks will assume to access the S3 bucket. This allows sharing an S3 bucket between root storage and the default catalog for a workspace. See the Databricks API documentation for more details.
- storage
Configuration StringId - (String) id of storage config to be used for
databricks_mws_workspaceresource. - storage
Configuration StringName name under which this storage configuration is stored
The following arguments are optional:
- account
Id string - Account Id that could be found in the top right corner of Accounts Console
- bucket
Name string - name of AWS S3 bucket
- creation
Time number - role
Arn string - The ARN of the IAM role that Databricks will assume to access the S3 bucket. This allows sharing an S3 bucket between root storage and the default catalog for a workspace. See the Databricks API documentation for more details.
- storage
Configuration stringId - (String) id of storage config to be used for
databricks_mws_workspaceresource. - storage
Configuration stringName name under which this storage configuration is stored
The following arguments are optional:
- account_
id str - Account Id that could be found in the top right corner of Accounts Console
- bucket_
name str - name of AWS S3 bucket
- creation_
time int - role_
arn str - The ARN of the IAM role that Databricks will assume to access the S3 bucket. This allows sharing an S3 bucket between root storage and the default catalog for a workspace. See the Databricks API documentation for more details.
- storage_
configuration_ strid - (String) id of storage config to be used for
databricks_mws_workspaceresource. - storage_
configuration_ strname name under which this storage configuration is stored
The following arguments are optional:
- account
Id String - Account Id that could be found in the top right corner of Accounts Console
- bucket
Name String - name of AWS S3 bucket
- creation
Time Number - role
Arn String - The ARN of the IAM role that Databricks will assume to access the S3 bucket. This allows sharing an S3 bucket between root storage and the default catalog for a workspace. See the Databricks API documentation for more details.
- storage
Configuration StringId - (String) id of storage config to be used for
databricks_mws_workspaceresource. - storage
Configuration StringName name under which this storage configuration is stored
The following arguments are optional:
Package Details
- Repository
- databricks pulumi/pulumi-databricks
- License
- Apache-2.0
- Notes
- This Pulumi package is based on the
databricksTerraform Provider.
