Databricks v1.14.0, May 23 23

databricks.getSparkVersion

Explore with Pulumi AI

Note If you have a fully automated setup with workspaces created by databricks_mws_workspaces, please make sure to add depends_on attribute in order to prevent default auth: cannot configure default credentials errors.

Gets Databricks Runtime (DBR) version that could be used for spark_version parameter in databricks.Cluster and other resources that fits search criteria, like specific Spark or Scala version, ML or Genomics runtime, etc., similar to executing databricks clusters spark-versions, and filters it to return the latest version that matches criteria. Often used along databricks.getNodeType data source.

Note This is experimental functionality, which aims to simplify things. In case of wrong parameters given (e.g. together ml = true and genomics = true, or something like), data source will throw an error. Similarly, if search returns multiple results, and latest = false, data source will throw an error.

The following resources are used in the same context:

End to end workspace management guide.
databricks.Cluster to create Databricks Clusters.
databricks.ClusterPolicy to create a databricks.Cluster policy, which limits the ability to create clusters based on a set of rules.
databricks.InstancePool to manage instance pools to reduce cluster start and auto-scaling times by maintaining a set of idle, ready-to-use instances.
databricks.Job to manage Databricks Jobs to run non-interactive code in a databricks_cluster.

Example Usage

using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Databricks = Pulumi.Databricks;

return await Deployment.RunAsync(() => 
{
    var withGpu = Databricks.GetNodeType.Invoke(new()
    {
        LocalDisk = true,
        MinCores = 16,
        GbPerCore = 1,
        MinGpus = 1,
    });

    var gpuMl = Databricks.GetSparkVersion.Invoke(new()
    {
        Gpu = true,
        Ml = true,
    });

    var research = new Databricks.Cluster("research", new()
    {
        ClusterName = "Research Cluster",
        SparkVersion = gpuMl.Apply(getSparkVersionResult => getSparkVersionResult.Id),
        NodeTypeId = withGpu.Apply(getNodeTypeResult => getNodeTypeResult.Id),
        AutoterminationMinutes = 20,
        Autoscale = new Databricks.Inputs.ClusterAutoscaleArgs
        {
            MinWorkers = 1,
            MaxWorkers = 50,
        },
    });

});

package main

import (
	"github.com/pulumi/pulumi-databricks/sdk/go/databricks"
	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)

func main() {
	pulumi.Run(func(ctx *pulumi.Context) error {
		withGpu, err := databricks.GetNodeType(ctx, &databricks.GetNodeTypeArgs{
			LocalDisk: pulumi.BoolRef(true),
			MinCores:  pulumi.IntRef(16),
			GbPerCore: pulumi.IntRef(1),
			MinGpus:   pulumi.IntRef(1),
		}, nil)
		if err != nil {
			return err
		}
		gpuMl, err := databricks.GetSparkVersion(ctx, &databricks.GetSparkVersionArgs{
			Gpu: pulumi.BoolRef(true),
			Ml:  pulumi.BoolRef(true),
		}, nil)
		if err != nil {
			return err
		}
		_, err = databricks.NewCluster(ctx, "research", &databricks.ClusterArgs{
			ClusterName:            pulumi.String("Research Cluster"),
			SparkVersion:           *pulumi.String(gpuMl.Id),
			NodeTypeId:             *pulumi.String(withGpu.Id),
			AutoterminationMinutes: pulumi.Int(20),
			Autoscale: &databricks.ClusterAutoscaleArgs{
				MinWorkers: pulumi.Int(1),
				MaxWorkers: pulumi.Int(50),
			},
		})
		if err != nil {
			return err
		}
		return nil
	})
}

package generated_program;

import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.databricks.DatabricksFunctions;
import com.pulumi.databricks.inputs.GetNodeTypeArgs;
import com.pulumi.databricks.inputs.GetSparkVersionArgs;
import com.pulumi.databricks.Cluster;
import com.pulumi.databricks.ClusterArgs;
import com.pulumi.databricks.inputs.ClusterAutoscaleArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;

public class App {
    public static void main(String[] args) {
        Pulumi.run(App::stack);
    }

    public static void stack(Context ctx) {
        final var withGpu = DatabricksFunctions.getNodeType(GetNodeTypeArgs.builder()
            .localDisk(true)
            .minCores(16)
            .gbPerCore(1)
            .minGpus(1)
            .build());

        final var gpuMl = DatabricksFunctions.getSparkVersion(GetSparkVersionArgs.builder()
            .gpu(true)
            .ml(true)
            .build());

        var research = new Cluster("research", ClusterArgs.builder()        
            .clusterName("Research Cluster")
            .sparkVersion(gpuMl.applyValue(getSparkVersionResult -> getSparkVersionResult.id()))
            .nodeTypeId(withGpu.applyValue(getNodeTypeResult -> getNodeTypeResult.id()))
            .autoterminationMinutes(20)
            .autoscale(ClusterAutoscaleArgs.builder()
                .minWorkers(1)
                .maxWorkers(50)
                .build())
            .build());

    }
}

import pulumi
import pulumi_databricks as databricks

with_gpu = databricks.get_node_type(local_disk=True,
    min_cores=16,
    gb_per_core=1,
    min_gpus=1)
gpu_ml = databricks.get_spark_version(gpu=True,
    ml=True)
research = databricks.Cluster("research",
    cluster_name="Research Cluster",
    spark_version=gpu_ml.id,
    node_type_id=with_gpu.id,
    autotermination_minutes=20,
    autoscale=databricks.ClusterAutoscaleArgs(
        min_workers=1,
        max_workers=50,
    ))

import * as pulumi from "@pulumi/pulumi";
import * as databricks from "@pulumi/databricks";

const withGpu = databricks.getNodeType({
    localDisk: true,
    minCores: 16,
    gbPerCore: 1,
    minGpus: 1,
});
const gpuMl = databricks.getSparkVersion({
    gpu: true,
    ml: true,
});
const research = new databricks.Cluster("research", {
    clusterName: "Research Cluster",
    sparkVersion: gpuMl.then(gpuMl => gpuMl.id),
    nodeTypeId: withGpu.then(withGpu => withGpu.id),
    autoterminationMinutes: 20,
    autoscale: {
        minWorkers: 1,
        maxWorkers: 50,
    },
});

resources:
  research:
    type: databricks:Cluster
    properties:
      clusterName: Research Cluster
      sparkVersion: ${gpuMl.id}
      nodeTypeId: ${withGpu.id}
      autoterminationMinutes: 20
      autoscale:
        minWorkers: 1
        maxWorkers: 50
variables:
  withGpu:
    fn::invoke:
      Function: databricks:getNodeType
      Arguments:
        localDisk: true
        minCores: 16
        gbPerCore: 1
        minGpus: 1
  gpuMl:
    fn::invoke:
      Function: databricks:getSparkVersion
      Arguments:
        gpu: true
        ml: true

Using getSparkVersion

Two invocation forms are available. The direct form accepts plain arguments and either blocks until the result value is available, or returns a Promise-wrapped result. The output form accepts Input-wrapped arguments and returns an Output-wrapped result.

function getSparkVersion(args: GetSparkVersionArgs, opts?: InvokeOptions): Promise<GetSparkVersionResult>
function getSparkVersionOutput(args: GetSparkVersionOutputArgs, opts?: InvokeOptions): Output<GetSparkVersionResult>

def get_spark_version(beta: Optional[bool] = None,
                      genomics: Optional[bool] = None,
                      gpu: Optional[bool] = None,
                      graviton: Optional[bool] = None,
                      latest: Optional[bool] = None,
                      long_term_support: Optional[bool] = None,
                      ml: Optional[bool] = None,
                      photon: Optional[bool] = None,
                      scala: Optional[str] = None,
                      spark_version: Optional[str] = None,
                      opts: Optional[InvokeOptions] = None) -> GetSparkVersionResult
def get_spark_version_output(beta: Optional[pulumi.Input[bool]] = None,
                      genomics: Optional[pulumi.Input[bool]] = None,
                      gpu: Optional[pulumi.Input[bool]] = None,
                      graviton: Optional[pulumi.Input[bool]] = None,
                      latest: Optional[pulumi.Input[bool]] = None,
                      long_term_support: Optional[pulumi.Input[bool]] = None,
                      ml: Optional[pulumi.Input[bool]] = None,
                      photon: Optional[pulumi.Input[bool]] = None,
                      scala: Optional[pulumi.Input[str]] = None,
                      spark_version: Optional[pulumi.Input[str]] = None,
                      opts: Optional[InvokeOptions] = None) -> Output[GetSparkVersionResult]

func GetSparkVersion(ctx *Context, args *GetSparkVersionArgs, opts ...InvokeOption) (*GetSparkVersionResult, error)
func GetSparkVersionOutput(ctx *Context, args *GetSparkVersionOutputArgs, opts ...InvokeOption) GetSparkVersionResultOutput

> Note: This function is named GetSparkVersion in the Go SDK.

public static class GetSparkVersion 
{
    public static Task<GetSparkVersionResult> InvokeAsync(GetSparkVersionArgs args, InvokeOptions? opts = null)
    public static Output<GetSparkVersionResult> Invoke(GetSparkVersionInvokeArgs args, InvokeOptions? opts = null)
}

public static CompletableFuture<GetSparkVersionResult> getSparkVersion(GetSparkVersionArgs args, InvokeOptions options)
// Output-based functions aren't available in Java yet

fn::invoke:
  function: databricks:index/getSparkVersion:getSparkVersion
  arguments:
    # arguments dictionary

The following arguments are supported:

Beta bool: if we should limit the search only to runtimes that are in Beta stage. Default to false.
Genomics bool: if we should limit the search only to Genomics (HLS) runtimes. Default to false.
Gpu bool: if we should limit the search only to runtimes that support GPUs. Default to false.
Graviton bool: if we should limit the search only to runtimes supporting AWS Graviton CPUs. Default to false.
Latest bool: if we should return only the latest version if there is more than one result. Default to true. If set to false and multiple versions are matching, throws an error.
LongTermSupport bool: if we should limit the search only to LTS (long term support) & ESR (extended support) versions. Default to false.
Ml bool: if we should limit the search only to ML runtimes. Default to false.
Photon bool: if we should limit the search only to Photon runtimes. Default to false.
Scala string: if we should limit the search only to runtimes that are based on specific Scala version. Default to 2.12.
SparkVersion string: if we should limit the search only to runtimes that are based on specific Spark version. Default to empty string. It could be specified as 3, or 3.0, or full version, like, 3.0.1.

Beta bool: if we should limit the search only to runtimes that are in Beta stage. Default to false.
Genomics bool: if we should limit the search only to Genomics (HLS) runtimes. Default to false.
Gpu bool: if we should limit the search only to runtimes that support GPUs. Default to false.
Graviton bool: if we should limit the search only to runtimes supporting AWS Graviton CPUs. Default to false.
Latest bool: if we should return only the latest version if there is more than one result. Default to true. If set to false and multiple versions are matching, throws an error.
LongTermSupport bool: if we should limit the search only to LTS (long term support) & ESR (extended support) versions. Default to false.
Ml bool: if we should limit the search only to ML runtimes. Default to false.
Photon bool: if we should limit the search only to Photon runtimes. Default to false.
Scala string: if we should limit the search only to runtimes that are based on specific Scala version. Default to 2.12.
SparkVersion string: if we should limit the search only to runtimes that are based on specific Spark version. Default to empty string. It could be specified as 3, or 3.0, or full version, like, 3.0.1.

beta Boolean: if we should limit the search only to runtimes that are in Beta stage. Default to false.
genomics Boolean: if we should limit the search only to Genomics (HLS) runtimes. Default to false.
gpu Boolean: if we should limit the search only to runtimes that support GPUs. Default to false.
graviton Boolean: if we should limit the search only to runtimes supporting AWS Graviton CPUs. Default to false.
latest Boolean: if we should return only the latest version if there is more than one result. Default to true. If set to false and multiple versions are matching, throws an error.
longTermSupport Boolean: if we should limit the search only to LTS (long term support) & ESR (extended support) versions. Default to false.
ml Boolean: if we should limit the search only to ML runtimes. Default to false.
photon Boolean: if we should limit the search only to Photon runtimes. Default to false.
scala String: if we should limit the search only to runtimes that are based on specific Scala version. Default to 2.12.
sparkVersion String: if we should limit the search only to runtimes that are based on specific Spark version. Default to empty string. It could be specified as 3, or 3.0, or full version, like, 3.0.1.

beta boolean: if we should limit the search only to runtimes that are in Beta stage. Default to false.
genomics boolean: if we should limit the search only to Genomics (HLS) runtimes. Default to false.
gpu boolean: if we should limit the search only to runtimes that support GPUs. Default to false.
graviton boolean: if we should limit the search only to runtimes supporting AWS Graviton CPUs. Default to false.
latest boolean: if we should return only the latest version if there is more than one result. Default to true. If set to false and multiple versions are matching, throws an error.
longTermSupport boolean: if we should limit the search only to LTS (long term support) & ESR (extended support) versions. Default to false.
ml boolean: if we should limit the search only to ML runtimes. Default to false.
photon boolean: if we should limit the search only to Photon runtimes. Default to false.
scala string: if we should limit the search only to runtimes that are based on specific Scala version. Default to 2.12.
sparkVersion string: if we should limit the search only to runtimes that are based on specific Spark version. Default to empty string. It could be specified as 3, or 3.0, or full version, like, 3.0.1.

beta bool: if we should limit the search only to runtimes that are in Beta stage. Default to false.
genomics bool: if we should limit the search only to Genomics (HLS) runtimes. Default to false.
gpu bool: if we should limit the search only to runtimes that support GPUs. Default to false.
graviton bool: if we should limit the search only to runtimes supporting AWS Graviton CPUs. Default to false.
latest bool: if we should return only the latest version if there is more than one result. Default to true. If set to false and multiple versions are matching, throws an error.
long_term_support bool: if we should limit the search only to LTS (long term support) & ESR (extended support) versions. Default to false.
ml bool: if we should limit the search only to ML runtimes. Default to false.
photon bool: if we should limit the search only to Photon runtimes. Default to false.
scala str: if we should limit the search only to runtimes that are based on specific Scala version. Default to 2.12.
spark_version str: if we should limit the search only to runtimes that are based on specific Spark version. Default to empty string. It could be specified as 3, or 3.0, or full version, like, 3.0.1.

beta Boolean: if we should limit the search only to runtimes that are in Beta stage. Default to false.
genomics Boolean: if we should limit the search only to Genomics (HLS) runtimes. Default to false.
gpu Boolean: if we should limit the search only to runtimes that support GPUs. Default to false.
graviton Boolean: if we should limit the search only to runtimes supporting AWS Graviton CPUs. Default to false.
latest Boolean: if we should return only the latest version if there is more than one result. Default to true. If set to false and multiple versions are matching, throws an error.
longTermSupport Boolean: if we should limit the search only to LTS (long term support) & ESR (extended support) versions. Default to false.
ml Boolean: if we should limit the search only to ML runtimes. Default to false.
photon Boolean: if we should limit the search only to Photon runtimes. Default to false.
scala String: if we should limit the search only to runtimes that are based on specific Scala version. Default to 2.12.
sparkVersion String: if we should limit the search only to runtimes that are based on specific Spark version. Default to empty string. It could be specified as 3, or 3.0, or full version, like, 3.0.1.

getSparkVersion Result

The following output properties are available:

Id string: The provider-assigned unique ID for this managed resource.
Beta bool
Genomics bool
Gpu bool
Graviton bool
Latest bool
LongTermSupport bool
Ml bool
Photon bool
Scala string
SparkVersion string

Id string: The provider-assigned unique ID for this managed resource.
Beta bool
Genomics bool
Gpu bool
Graviton bool
Latest bool
LongTermSupport bool
Ml bool
Photon bool
Scala string
SparkVersion string

id String: The provider-assigned unique ID for this managed resource.
beta Boolean
genomics Boolean
gpu Boolean
graviton Boolean
latest Boolean
longTermSupport Boolean
ml Boolean
photon Boolean
scala String
sparkVersion String

id string: The provider-assigned unique ID for this managed resource.
beta boolean
genomics boolean
gpu boolean
graviton boolean
latest boolean
longTermSupport boolean
ml boolean
photon boolean
scala string
sparkVersion string

id str: The provider-assigned unique ID for this managed resource.
beta bool
genomics bool
gpu bool
graviton bool
latest bool
long_term_support bool
ml bool
photon bool
scala str
spark_version str

id String: The provider-assigned unique ID for this managed resource.
beta Boolean
genomics Boolean
gpu Boolean
graviton Boolean
latest Boolean
longTermSupport Boolean
ml Boolean
photon Boolean
scala String
sparkVersion String

Package Details

Repository: databricks pulumi/pulumi-databricks
License: Apache-2.0
Notes: This Pulumi package is based on the databricks Terraform Provider.

databricks.getSparkVersion

Related Resources

Example Usage

Using getSparkVersion

getSparkVersion Result

Package Details

On this page