Databricks v1.36.0, Apr 19 24

Databricks v1.36.0 published on Friday, Apr 19, 2024 by Pulumi

databricks.getSparkVersion

Explore with Pulumi AI

Databricks v1.36.0 published on Friday, Apr 19, 2024 by Pulumi

Gets Databricks Runtime (DBR) version that could be used for spark_version parameter in databricks.Cluster and other resources that fits search criteria, like specific Spark or Scala version, ML or Genomics runtime, etc., similar to executing databricks clusters spark-versions, and filters it to return the latest version that matches criteria. Often used along databricks.getNodeType data source.

Note This is experimental functionality, which aims to simplify things. In case of wrong parameters given (e.g. together ml = true and genomics = true, or something like), data source will throw an error. Similarly, if search returns multiple results, and latest = false, data source will throw an error.

Example Usage

import * as pulumi from "@pulumi/pulumi";
import * as databricks from "@pulumi/databricks";

const withGpu = databricks.getNodeType({
    localDisk: true,
    minCores: 16,
    gbPerCore: 1,
    minGpus: 1,
});
const gpuMl = databricks.getSparkVersion({
    gpu: true,
    ml: true,
});
const research = new databricks.Cluster("research", {
    clusterName: "Research Cluster",
    sparkVersion: gpuMl.then(gpuMl => gpuMl.id),
    nodeTypeId: withGpu.then(withGpu => withGpu.id),
    autoterminationMinutes: 20,
    autoscale: {
        minWorkers: 1,
        maxWorkers: 50,
    },
});

import pulumi
import pulumi_databricks as databricks

with_gpu = databricks.get_node_type(local_disk=True,
    min_cores=16,
    gb_per_core=1,
    min_gpus=1)
gpu_ml = databricks.get_spark_version(gpu=True,
    ml=True)
research = databricks.Cluster("research",
    cluster_name="Research Cluster",
    spark_version=gpu_ml.id,
    node_type_id=with_gpu.id,
    autotermination_minutes=20,
    autoscale=databricks.ClusterAutoscaleArgs(
        min_workers=1,
        max_workers=50,
    ))

package main

import (
	"github.com/pulumi/pulumi-databricks/sdk/go/databricks"
	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)

func main() {
	pulumi.Run(func(ctx *pulumi.Context) error {
		withGpu, err := databricks.GetNodeType(ctx, &databricks.GetNodeTypeArgs{
			LocalDisk: pulumi.BoolRef(true),
			MinCores:  pulumi.IntRef(16),
			GbPerCore: pulumi.IntRef(1),
			MinGpus:   pulumi.IntRef(1),
		}, nil)
		if err != nil {
			return err
		}
		gpuMl, err := databricks.GetSparkVersion(ctx, &databricks.GetSparkVersionArgs{
			Gpu: pulumi.BoolRef(true),
			Ml:  pulumi.BoolRef(true),
		}, nil)
		if err != nil {
			return err
		}
		_, err = databricks.NewCluster(ctx, "research", &databricks.ClusterArgs{
			ClusterName:            pulumi.String("Research Cluster"),
			SparkVersion:           pulumi.String(gpuMl.Id),
			NodeTypeId:             pulumi.String(withGpu.Id),
			AutoterminationMinutes: pulumi.Int(20),
			Autoscale: &databricks.ClusterAutoscaleArgs{
				MinWorkers: pulumi.Int(1),
				MaxWorkers: pulumi.Int(50),
			},
		})
		if err != nil {
			return err
		}
		return nil
	})
}

using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Databricks = Pulumi.Databricks;

return await Deployment.RunAsync(() => 
{
    var withGpu = Databricks.GetNodeType.Invoke(new()
    {
        LocalDisk = true,
        MinCores = 16,
        GbPerCore = 1,
        MinGpus = 1,
    });

    var gpuMl = Databricks.GetSparkVersion.Invoke(new()
    {
        Gpu = true,
        Ml = true,
    });

    var research = new Databricks.Cluster("research", new()
    {
        ClusterName = "Research Cluster",
        SparkVersion = gpuMl.Apply(getSparkVersionResult => getSparkVersionResult.Id),
        NodeTypeId = withGpu.Apply(getNodeTypeResult => getNodeTypeResult.Id),
        AutoterminationMinutes = 20,
        Autoscale = new Databricks.Inputs.ClusterAutoscaleArgs
        {
            MinWorkers = 1,
            MaxWorkers = 50,
        },
    });

});

package generated_program;

import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.databricks.DatabricksFunctions;
import com.pulumi.databricks.inputs.GetNodeTypeArgs;
import com.pulumi.databricks.inputs.GetSparkVersionArgs;
import com.pulumi.databricks.Cluster;
import com.pulumi.databricks.ClusterArgs;
import com.pulumi.databricks.inputs.ClusterAutoscaleArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;

public class App {
    public static void main(String[] args) {
        Pulumi.run(App::stack);
    }

    public static void stack(Context ctx) {
        final var withGpu = DatabricksFunctions.getNodeType(GetNodeTypeArgs.builder()
            .localDisk(true)
            .minCores(16)
            .gbPerCore(1)
            .minGpus(1)
            .build());

        final var gpuMl = DatabricksFunctions.getSparkVersion(GetSparkVersionArgs.builder()
            .gpu(true)
            .ml(true)
            .build());

        var research = new Cluster("research", ClusterArgs.builder()        
            .clusterName("Research Cluster")
            .sparkVersion(gpuMl.applyValue(getSparkVersionResult -> getSparkVersionResult.id()))
            .nodeTypeId(withGpu.applyValue(getNodeTypeResult -> getNodeTypeResult.id()))
            .autoterminationMinutes(20)
            .autoscale(ClusterAutoscaleArgs.builder()
                .minWorkers(1)
                .maxWorkers(50)
                .build())
            .build());

    }
}

resources:
  research:
    type: databricks:Cluster
    properties:
      clusterName: Research Cluster
      sparkVersion: ${gpuMl.id}
      nodeTypeId: ${withGpu.id}
      autoterminationMinutes: 20
      autoscale:
        minWorkers: 1
        maxWorkers: 50
variables:
  withGpu:
    fn::invoke:
      Function: databricks:getNodeType
      Arguments:
        localDisk: true
        minCores: 16
        gbPerCore: 1
        minGpus: 1
  gpuMl:
    fn::invoke:
      Function: databricks:getSparkVersion
      Arguments:
        gpu: true
        ml: true

The following resources are used in the same context:

End to end workspace management guide.
databricks.Cluster to create Databricks Clusters.
databricks.ClusterPolicy to create a databricks.Cluster policy, which limits the ability to create clusters based on a set of rules.
databricks.InstancePool to manage instance pools to reduce cluster start and auto-scaling times by maintaining a set of idle, ready-to-use instances.
databricks.Job to manage Databricks Jobs to run non-interactive code in a databricks_cluster.

Using getSparkVersion

Two invocation forms are available. The direct form accepts plain arguments and either blocks until the result value is available, or returns a Promise-wrapped result. The output form accepts Input-wrapped arguments and returns an Output-wrapped result.

function getSparkVersion(args: GetSparkVersionArgs, opts?: InvokeOptions): Promise<GetSparkVersionResult>
function getSparkVersionOutput(args: GetSparkVersionOutputArgs, opts?: InvokeOptions): Output<GetSparkVersionResult>

def get_spark_version(beta: Optional[bool] = None,
                      genomics: Optional[bool] = None,
                      gpu: Optional[bool] = None,
                      graviton: Optional[bool] = None,
                      latest: Optional[bool] = None,
                      long_term_support: Optional[bool] = None,
                      ml: Optional[bool] = None,
                      photon: Optional[bool] = None,
                      scala: Optional[str] = None,
                      spark_version: Optional[str] = None,
                      opts: Optional[InvokeOptions] = None) -> GetSparkVersionResult
def get_spark_version_output(beta: Optional[pulumi.Input[bool]] = None,
                      genomics: Optional[pulumi.Input[bool]] = None,
                      gpu: Optional[pulumi.Input[bool]] = None,
                      graviton: Optional[pulumi.Input[bool]] = None,
                      latest: Optional[pulumi.Input[bool]] = None,
                      long_term_support: Optional[pulumi.Input[bool]] = None,
                      ml: Optional[pulumi.Input[bool]] = None,
                      photon: Optional[pulumi.Input[bool]] = None,
                      scala: Optional[pulumi.Input[str]] = None,
                      spark_version: Optional[pulumi.Input[str]] = None,
                      opts: Optional[InvokeOptions] = None) -> Output[GetSparkVersionResult]

func GetSparkVersion(ctx *Context, args *GetSparkVersionArgs, opts ...InvokeOption) (*GetSparkVersionResult, error)
func GetSparkVersionOutput(ctx *Context, args *GetSparkVersionOutputArgs, opts ...InvokeOption) GetSparkVersionResultOutput

> Note: This function is named GetSparkVersion in the Go SDK.

public static class GetSparkVersion 
{
    public static Task<GetSparkVersionResult> InvokeAsync(GetSparkVersionArgs args, InvokeOptions? opts = null)
    public static Output<GetSparkVersionResult> Invoke(GetSparkVersionInvokeArgs args, InvokeOptions? opts = null)
}

public static CompletableFuture<GetSparkVersionResult> getSparkVersion(GetSparkVersionArgs args, InvokeOptions options)
// Output-based functions aren't available in Java yet

fn::invoke:
  function: databricks:index/getSparkVersion:getSparkVersion
  arguments:
    # arguments dictionary

The following arguments are supported:

Beta bool: if we should limit the search only to runtimes that are in Beta stage. Default to false.
Genomics bool: if we should limit the search only to Genomics (HLS) runtimes. Default to false.
Gpu bool: if we should limit the search only to runtimes that support GPUs. Default to false.
Graviton bool: if we should limit the search only to runtimes supporting AWS Graviton CPUs. Default to false. Deprecated with DBR 14.0 release. DBR version compiled for Graviton will be automatically installed when nodes with Graviton CPUs are specified in the cluster configuration.
Deprecated: Not required anymore - it's automatically enabled on the Graviton-based node types
Latest bool: if we should return only the latest version if there is more than one result. Default to true. If set to false and multiple versions are matching, throws an error.
LongTermSupport bool: if we should limit the search only to LTS (long term support) & ESR (extended support) versions. Default to false.
Ml bool: if we should limit the search only to ML runtimes. Default to false.
Photon bool: if we should limit the search only to Photon runtimes. Default to false. Deprecated with DBR 14.0 release. Specify runtime_engine=\"PHOTON\" in the cluster configuration instead!
Deprecated: Specify runtime_engine="PHOTON" in the cluster configuration
Scala string: if we should limit the search only to runtimes that are based on specific Scala version. Default to 2.12.
SparkVersion string: if we should limit the search only to runtimes that are based on specific Spark version. Default to empty string. It could be specified as 3, or 3.0, or full version, like, 3.0.1.

Beta bool: if we should limit the search only to runtimes that are in Beta stage. Default to false.
Genomics bool: if we should limit the search only to Genomics (HLS) runtimes. Default to false.
Gpu bool: if we should limit the search only to runtimes that support GPUs. Default to false.
Graviton bool: if we should limit the search only to runtimes supporting AWS Graviton CPUs. Default to false. Deprecated with DBR 14.0 release. DBR version compiled for Graviton will be automatically installed when nodes with Graviton CPUs are specified in the cluster configuration.
Deprecated: Not required anymore - it's automatically enabled on the Graviton-based node types
Latest bool: if we should return only the latest version if there is more than one result. Default to true. If set to false and multiple versions are matching, throws an error.
LongTermSupport bool: if we should limit the search only to LTS (long term support) & ESR (extended support) versions. Default to false.
Ml bool: if we should limit the search only to ML runtimes. Default to false.
Photon bool: if we should limit the search only to Photon runtimes. Default to false. Deprecated with DBR 14.0 release. Specify runtime_engine=\"PHOTON\" in the cluster configuration instead!
Deprecated: Specify runtime_engine="PHOTON" in the cluster configuration
Scala string: if we should limit the search only to runtimes that are based on specific Scala version. Default to 2.12.
SparkVersion string: if we should limit the search only to runtimes that are based on specific Spark version. Default to empty string. It could be specified as 3, or 3.0, or full version, like, 3.0.1.

beta Boolean: if we should limit the search only to runtimes that are in Beta stage. Default to false.
genomics Boolean: if we should limit the search only to Genomics (HLS) runtimes. Default to false.
gpu Boolean: if we should limit the search only to runtimes that support GPUs. Default to false.
graviton Boolean: if we should limit the search only to runtimes supporting AWS Graviton CPUs. Default to false. Deprecated with DBR 14.0 release. DBR version compiled for Graviton will be automatically installed when nodes with Graviton CPUs are specified in the cluster configuration.
Deprecated: Not required anymore - it's automatically enabled on the Graviton-based node types
latest Boolean: if we should return only the latest version if there is more than one result. Default to true. If set to false and multiple versions are matching, throws an error.
longTermSupport Boolean: if we should limit the search only to LTS (long term support) & ESR (extended support) versions. Default to false.
ml Boolean: if we should limit the search only to ML runtimes. Default to false.
photon Boolean: if we should limit the search only to Photon runtimes. Default to false. Deprecated with DBR 14.0 release. Specify runtime_engine=\"PHOTON\" in the cluster configuration instead!
Deprecated: Specify runtime_engine="PHOTON" in the cluster configuration
scala String: if we should limit the search only to runtimes that are based on specific Scala version. Default to 2.12.
sparkVersion String: if we should limit the search only to runtimes that are based on specific Spark version. Default to empty string. It could be specified as 3, or 3.0, or full version, like, 3.0.1.

beta boolean: if we should limit the search only to runtimes that are in Beta stage. Default to false.
genomics boolean: if we should limit the search only to Genomics (HLS) runtimes. Default to false.
gpu boolean: if we should limit the search only to runtimes that support GPUs. Default to false.
graviton boolean: if we should limit the search only to runtimes supporting AWS Graviton CPUs. Default to false. Deprecated with DBR 14.0 release. DBR version compiled for Graviton will be automatically installed when nodes with Graviton CPUs are specified in the cluster configuration.
Deprecated: Not required anymore - it's automatically enabled on the Graviton-based node types
latest boolean: if we should return only the latest version if there is more than one result. Default to true. If set to false and multiple versions are matching, throws an error.
longTermSupport boolean: if we should limit the search only to LTS (long term support) & ESR (extended support) versions. Default to false.
ml boolean: if we should limit the search only to ML runtimes. Default to false.
photon boolean: if we should limit the search only to Photon runtimes. Default to false. Deprecated with DBR 14.0 release. Specify runtime_engine=\"PHOTON\" in the cluster configuration instead!
Deprecated: Specify runtime_engine="PHOTON" in the cluster configuration
scala string: if we should limit the search only to runtimes that are based on specific Scala version. Default to 2.12.
sparkVersion string: if we should limit the search only to runtimes that are based on specific Spark version. Default to empty string. It could be specified as 3, or 3.0, or full version, like, 3.0.1.

beta bool: if we should limit the search only to runtimes that are in Beta stage. Default to false.
genomics bool: if we should limit the search only to Genomics (HLS) runtimes. Default to false.
gpu bool: if we should limit the search only to runtimes that support GPUs. Default to false.
graviton bool: if we should limit the search only to runtimes supporting AWS Graviton CPUs. Default to false. Deprecated with DBR 14.0 release. DBR version compiled for Graviton will be automatically installed when nodes with Graviton CPUs are specified in the cluster configuration.
Deprecated: Not required anymore - it's automatically enabled on the Graviton-based node types
latest bool: if we should return only the latest version if there is more than one result. Default to true. If set to false and multiple versions are matching, throws an error.
long_term_support bool: if we should limit the search only to LTS (long term support) & ESR (extended support) versions. Default to false.
ml bool: if we should limit the search only to ML runtimes. Default to false.
photon bool: if we should limit the search only to Photon runtimes. Default to false. Deprecated with DBR 14.0 release. Specify runtime_engine=\"PHOTON\" in the cluster configuration instead!
Deprecated: Specify runtime_engine="PHOTON" in the cluster configuration
scala str: if we should limit the search only to runtimes that are based on specific Scala version. Default to 2.12.
spark_version str: if we should limit the search only to runtimes that are based on specific Spark version. Default to empty string. It could be specified as 3, or 3.0, or full version, like, 3.0.1.

beta Boolean: if we should limit the search only to runtimes that are in Beta stage. Default to false.
genomics Boolean: if we should limit the search only to Genomics (HLS) runtimes. Default to false.
gpu Boolean: if we should limit the search only to runtimes that support GPUs. Default to false.
graviton Boolean: if we should limit the search only to runtimes supporting AWS Graviton CPUs. Default to false. Deprecated with DBR 14.0 release. DBR version compiled for Graviton will be automatically installed when nodes with Graviton CPUs are specified in the cluster configuration.
Deprecated: Not required anymore - it's automatically enabled on the Graviton-based node types
latest Boolean: if we should return only the latest version if there is more than one result. Default to true. If set to false and multiple versions are matching, throws an error.
longTermSupport Boolean: if we should limit the search only to LTS (long term support) & ESR (extended support) versions. Default to false.
ml Boolean: if we should limit the search only to ML runtimes. Default to false.
photon Boolean: if we should limit the search only to Photon runtimes. Default to false. Deprecated with DBR 14.0 release. Specify runtime_engine=\"PHOTON\" in the cluster configuration instead!
Deprecated: Specify runtime_engine="PHOTON" in the cluster configuration
scala String: if we should limit the search only to runtimes that are based on specific Scala version. Default to 2.12.
sparkVersion String: if we should limit the search only to runtimes that are based on specific Spark version. Default to empty string. It could be specified as 3, or 3.0, or full version, like, 3.0.1.

getSparkVersion Result

The following output properties are available:

Id string: The provider-assigned unique ID for this managed resource.
Beta bool
Genomics bool
Gpu bool
Graviton bool: Deprecated: Not required anymore - it's automatically enabled on the Graviton-based node types
Latest bool
LongTermSupport bool
Ml bool
Photon bool: Deprecated: Specify runtime_engine="PHOTON" in the cluster configuration
Scala string
SparkVersion string

Id string: The provider-assigned unique ID for this managed resource.
Beta bool
Genomics bool
Gpu bool
Graviton bool: Deprecated: Not required anymore - it's automatically enabled on the Graviton-based node types
Latest bool
LongTermSupport bool
Ml bool
Photon bool: Deprecated: Specify runtime_engine="PHOTON" in the cluster configuration
Scala string
SparkVersion string

id String: The provider-assigned unique ID for this managed resource.
beta Boolean
genomics Boolean
gpu Boolean
graviton Boolean: Deprecated: Not required anymore - it's automatically enabled on the Graviton-based node types
latest Boolean
longTermSupport Boolean
ml Boolean
photon Boolean: Deprecated: Specify runtime_engine="PHOTON" in the cluster configuration
scala String
sparkVersion String

id string: The provider-assigned unique ID for this managed resource.
beta boolean
genomics boolean
gpu boolean
graviton boolean: Deprecated: Not required anymore - it's automatically enabled on the Graviton-based node types
latest boolean
longTermSupport boolean
ml boolean
photon boolean: Deprecated: Specify runtime_engine="PHOTON" in the cluster configuration
scala string
sparkVersion string

id str: The provider-assigned unique ID for this managed resource.
beta bool
genomics bool
gpu bool
graviton bool: Deprecated: Not required anymore - it's automatically enabled on the Graviton-based node types
latest bool
long_term_support bool
ml bool
photon bool: Deprecated: Specify runtime_engine="PHOTON" in the cluster configuration
scala str
spark_version str

id String: The provider-assigned unique ID for this managed resource.
beta Boolean
genomics Boolean
gpu Boolean
graviton Boolean: Deprecated: Not required anymore - it's automatically enabled on the Graviton-based node types
latest Boolean
longTermSupport Boolean
ml Boolean
photon Boolean: Deprecated: Specify runtime_engine="PHOTON" in the cluster configuration
scala String
sparkVersion String

Package Details

Repository: databricks pulumi/pulumi-databricks
License: Apache-2.0
Notes: This Pulumi package is based on the databricks Terraform Provider.

Databricks v1.36.0 published on Friday, Apr 19, 2024 by Pulumi

pulumi/pulumi-databricks

databricks.getSparkVersion

On this page

On this page

Example Usage

Using getSparkVersion

getSparkVersion Result

Package Details

On this page

On this page

databricks.getSparkVersion

On this page

On this page

Example Usage

Related Resources

Using getSparkVersion

getSparkVersion Result

Package Details

On this page

On this page