aws.glue.Job
Explore with Pulumi AI
Provides a Glue Job resource.
Glue functionality, such as monitoring and logging of jobs, is typically managed with the
default_arguments
argument. See the Special Parameters Used by AWS Glue topic in the Glue developer guide for additional information.
Example Usage
Python Job
import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
const example = new aws.glue.Job("example", {
name: "example",
roleArn: exampleAwsIamRole.arn,
command: {
scriptLocation: `s3://${exampleAwsS3Bucket.bucket}/example.py`,
},
});
import pulumi
import pulumi_aws as aws
example = aws.glue.Job("example",
name="example",
role_arn=example_aws_iam_role["arn"],
command={
"script_location": f"s3://{example_aws_s3_bucket['bucket']}/example.py",
})
package main
import (
"fmt"
"github.com/pulumi/pulumi-aws/sdk/v6/go/aws/glue"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
_, err := glue.NewJob(ctx, "example", &glue.JobArgs{
Name: pulumi.String("example"),
RoleArn: pulumi.Any(exampleAwsIamRole.Arn),
Command: &glue.JobCommandArgs{
ScriptLocation: pulumi.Sprintf("s3://%v/example.py", exampleAwsS3Bucket.Bucket),
},
})
if err != nil {
return err
}
return nil
})
}
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Aws = Pulumi.Aws;
return await Deployment.RunAsync(() =>
{
var example = new Aws.Glue.Job("example", new()
{
Name = "example",
RoleArn = exampleAwsIamRole.Arn,
Command = new Aws.Glue.Inputs.JobCommandArgs
{
ScriptLocation = $"s3://{exampleAwsS3Bucket.Bucket}/example.py",
},
});
});
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.glue.Job;
import com.pulumi.aws.glue.JobArgs;
import com.pulumi.aws.glue.inputs.JobCommandArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var example = new Job("example", JobArgs.builder()
.name("example")
.roleArn(exampleAwsIamRole.arn())
.command(JobCommandArgs.builder()
.scriptLocation(String.format("s3://%s/example.py", exampleAwsS3Bucket.bucket()))
.build())
.build());
}
}
resources:
example:
type: aws:glue:Job
properties:
name: example
roleArn: ${exampleAwsIamRole.arn}
command:
scriptLocation: s3://${exampleAwsS3Bucket.bucket}/example.py
Ray Job
import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
const example = new aws.glue.Job("example", {
name: "example",
roleArn: exampleAwsIamRole.arn,
glueVersion: "4.0",
workerType: "Z.2X",
command: {
name: "glueray",
pythonVersion: "3.9",
runtime: "Ray2.4",
scriptLocation: `s3://${exampleAwsS3Bucket.bucket}/example.py`,
},
});
import pulumi
import pulumi_aws as aws
example = aws.glue.Job("example",
name="example",
role_arn=example_aws_iam_role["arn"],
glue_version="4.0",
worker_type="Z.2X",
command={
"name": "glueray",
"python_version": "3.9",
"runtime": "Ray2.4",
"script_location": f"s3://{example_aws_s3_bucket['bucket']}/example.py",
})
package main
import (
"fmt"
"github.com/pulumi/pulumi-aws/sdk/v6/go/aws/glue"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
_, err := glue.NewJob(ctx, "example", &glue.JobArgs{
Name: pulumi.String("example"),
RoleArn: pulumi.Any(exampleAwsIamRole.Arn),
GlueVersion: pulumi.String("4.0"),
WorkerType: pulumi.String("Z.2X"),
Command: &glue.JobCommandArgs{
Name: pulumi.String("glueray"),
PythonVersion: pulumi.String("3.9"),
Runtime: pulumi.String("Ray2.4"),
ScriptLocation: pulumi.Sprintf("s3://%v/example.py", exampleAwsS3Bucket.Bucket),
},
})
if err != nil {
return err
}
return nil
})
}
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Aws = Pulumi.Aws;
return await Deployment.RunAsync(() =>
{
var example = new Aws.Glue.Job("example", new()
{
Name = "example",
RoleArn = exampleAwsIamRole.Arn,
GlueVersion = "4.0",
WorkerType = "Z.2X",
Command = new Aws.Glue.Inputs.JobCommandArgs
{
Name = "glueray",
PythonVersion = "3.9",
Runtime = "Ray2.4",
ScriptLocation = $"s3://{exampleAwsS3Bucket.Bucket}/example.py",
},
});
});
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.glue.Job;
import com.pulumi.aws.glue.JobArgs;
import com.pulumi.aws.glue.inputs.JobCommandArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var example = new Job("example", JobArgs.builder()
.name("example")
.roleArn(exampleAwsIamRole.arn())
.glueVersion("4.0")
.workerType("Z.2X")
.command(JobCommandArgs.builder()
.name("glueray")
.pythonVersion("3.9")
.runtime("Ray2.4")
.scriptLocation(String.format("s3://%s/example.py", exampleAwsS3Bucket.bucket()))
.build())
.build());
}
}
resources:
example:
type: aws:glue:Job
properties:
name: example
roleArn: ${exampleAwsIamRole.arn}
glueVersion: '4.0'
workerType: Z.2X
command:
name: glueray
pythonVersion: '3.9'
runtime: Ray2.4
scriptLocation: s3://${exampleAwsS3Bucket.bucket}/example.py
Scala Job
import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
const example = new aws.glue.Job("example", {
name: "example",
roleArn: exampleAwsIamRole.arn,
command: {
scriptLocation: `s3://${exampleAwsS3Bucket.bucket}/example.scala`,
},
defaultArguments: {
"--job-language": "scala",
},
});
import pulumi
import pulumi_aws as aws
example = aws.glue.Job("example",
name="example",
role_arn=example_aws_iam_role["arn"],
command={
"script_location": f"s3://{example_aws_s3_bucket['bucket']}/example.scala",
},
default_arguments={
"--job-language": "scala",
})
package main
import (
"fmt"
"github.com/pulumi/pulumi-aws/sdk/v6/go/aws/glue"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
_, err := glue.NewJob(ctx, "example", &glue.JobArgs{
Name: pulumi.String("example"),
RoleArn: pulumi.Any(exampleAwsIamRole.Arn),
Command: &glue.JobCommandArgs{
ScriptLocation: pulumi.Sprintf("s3://%v/example.scala", exampleAwsS3Bucket.Bucket),
},
DefaultArguments: pulumi.StringMap{
"--job-language": pulumi.String("scala"),
},
})
if err != nil {
return err
}
return nil
})
}
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Aws = Pulumi.Aws;
return await Deployment.RunAsync(() =>
{
var example = new Aws.Glue.Job("example", new()
{
Name = "example",
RoleArn = exampleAwsIamRole.Arn,
Command = new Aws.Glue.Inputs.JobCommandArgs
{
ScriptLocation = $"s3://{exampleAwsS3Bucket.Bucket}/example.scala",
},
DefaultArguments =
{
{ "--job-language", "scala" },
},
});
});
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.glue.Job;
import com.pulumi.aws.glue.JobArgs;
import com.pulumi.aws.glue.inputs.JobCommandArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var example = new Job("example", JobArgs.builder()
.name("example")
.roleArn(exampleAwsIamRole.arn())
.command(JobCommandArgs.builder()
.scriptLocation(String.format("s3://%s/example.scala", exampleAwsS3Bucket.bucket()))
.build())
.defaultArguments(Map.of("--job-language", "scala"))
.build());
}
}
resources:
example:
type: aws:glue:Job
properties:
name: example
roleArn: ${exampleAwsIamRole.arn}
command:
scriptLocation: s3://${exampleAwsS3Bucket.bucket}/example.scala
defaultArguments:
--job-language: scala
Streaming Job
import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
const example = new aws.glue.Job("example", {
name: "example streaming job",
roleArn: exampleAwsIamRole.arn,
command: {
name: "gluestreaming",
scriptLocation: `s3://${exampleAwsS3Bucket.bucket}/example.script`,
},
});
import pulumi
import pulumi_aws as aws
example = aws.glue.Job("example",
name="example streaming job",
role_arn=example_aws_iam_role["arn"],
command={
"name": "gluestreaming",
"script_location": f"s3://{example_aws_s3_bucket['bucket']}/example.script",
})
package main
import (
"fmt"
"github.com/pulumi/pulumi-aws/sdk/v6/go/aws/glue"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
_, err := glue.NewJob(ctx, "example", &glue.JobArgs{
Name: pulumi.String("example streaming job"),
RoleArn: pulumi.Any(exampleAwsIamRole.Arn),
Command: &glue.JobCommandArgs{
Name: pulumi.String("gluestreaming"),
ScriptLocation: pulumi.Sprintf("s3://%v/example.script", exampleAwsS3Bucket.Bucket),
},
})
if err != nil {
return err
}
return nil
})
}
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Aws = Pulumi.Aws;
return await Deployment.RunAsync(() =>
{
var example = new Aws.Glue.Job("example", new()
{
Name = "example streaming job",
RoleArn = exampleAwsIamRole.Arn,
Command = new Aws.Glue.Inputs.JobCommandArgs
{
Name = "gluestreaming",
ScriptLocation = $"s3://{exampleAwsS3Bucket.Bucket}/example.script",
},
});
});
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.glue.Job;
import com.pulumi.aws.glue.JobArgs;
import com.pulumi.aws.glue.inputs.JobCommandArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var example = new Job("example", JobArgs.builder()
.name("example streaming job")
.roleArn(exampleAwsIamRole.arn())
.command(JobCommandArgs.builder()
.name("gluestreaming")
.scriptLocation(String.format("s3://%s/example.script", exampleAwsS3Bucket.bucket()))
.build())
.build());
}
}
resources:
example:
type: aws:glue:Job
properties:
name: example streaming job
roleArn: ${exampleAwsIamRole.arn}
command:
name: gluestreaming
scriptLocation: s3://${exampleAwsS3Bucket.bucket}/example.script
Enabling CloudWatch Logs and Metrics
import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
const example = new aws.cloudwatch.LogGroup("example", {
name: "example",
retentionInDays: 14,
});
const exampleJob = new aws.glue.Job("example", {defaultArguments: {
"--continuous-log-logGroup": example.name,
"--enable-continuous-cloudwatch-log": "true",
"--enable-continuous-log-filter": "true",
"--enable-metrics": "",
}});
import pulumi
import pulumi_aws as aws
example = aws.cloudwatch.LogGroup("example",
name="example",
retention_in_days=14)
example_job = aws.glue.Job("example", default_arguments={
"--continuous-log-logGroup": example.name,
"--enable-continuous-cloudwatch-log": "true",
"--enable-continuous-log-filter": "true",
"--enable-metrics": "",
})
package main
import (
"github.com/pulumi/pulumi-aws/sdk/v6/go/aws/cloudwatch"
"github.com/pulumi/pulumi-aws/sdk/v6/go/aws/glue"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
example, err := cloudwatch.NewLogGroup(ctx, "example", &cloudwatch.LogGroupArgs{
Name: pulumi.String("example"),
RetentionInDays: pulumi.Int(14),
})
if err != nil {
return err
}
_, err = glue.NewJob(ctx, "example", &glue.JobArgs{
DefaultArguments: pulumi.StringMap{
"--continuous-log-logGroup": example.Name,
"--enable-continuous-cloudwatch-log": pulumi.String("true"),
"--enable-continuous-log-filter": pulumi.String("true"),
"--enable-metrics": pulumi.String(""),
},
})
if err != nil {
return err
}
return nil
})
}
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Aws = Pulumi.Aws;
return await Deployment.RunAsync(() =>
{
var example = new Aws.CloudWatch.LogGroup("example", new()
{
Name = "example",
RetentionInDays = 14,
});
var exampleJob = new Aws.Glue.Job("example", new()
{
DefaultArguments =
{
{ "--continuous-log-logGroup", example.Name },
{ "--enable-continuous-cloudwatch-log", "true" },
{ "--enable-continuous-log-filter", "true" },
{ "--enable-metrics", "" },
},
});
});
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.cloudwatch.LogGroup;
import com.pulumi.aws.cloudwatch.LogGroupArgs;
import com.pulumi.aws.glue.Job;
import com.pulumi.aws.glue.JobArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var example = new LogGroup("example", LogGroupArgs.builder()
.name("example")
.retentionInDays(14)
.build());
var exampleJob = new Job("exampleJob", JobArgs.builder()
.defaultArguments(Map.ofEntries(
Map.entry("--continuous-log-logGroup", example.name()),
Map.entry("--enable-continuous-cloudwatch-log", "true"),
Map.entry("--enable-continuous-log-filter", "true"),
Map.entry("--enable-metrics", "")
))
.build());
}
}
resources:
example:
type: aws:cloudwatch:LogGroup
properties:
name: example
retentionInDays: 14
exampleJob:
type: aws:glue:Job
name: example
properties:
defaultArguments:
--continuous-log-logGroup: ${example.name}
--enable-continuous-cloudwatch-log: 'true'
--enable-continuous-log-filter: 'true'
--enable-metrics:
Create Job Resource
Resources are created with functions called constructors. To learn more about declaring and configuring resources, see Resources.
Constructor syntax
new Job(name: string, args: JobArgs, opts?: CustomResourceOptions);
@overload
def Job(resource_name: str,
args: JobArgs,
opts: Optional[ResourceOptions] = None)
@overload
def Job(resource_name: str,
opts: Optional[ResourceOptions] = None,
command: Optional[JobCommandArgs] = None,
role_arn: Optional[str] = None,
execution_property: Optional[JobExecutionPropertyArgs] = None,
name: Optional[str] = None,
execution_class: Optional[str] = None,
default_arguments: Optional[Mapping[str, str]] = None,
glue_version: Optional[str] = None,
job_run_queuing_enabled: Optional[bool] = None,
maintenance_window: Optional[str] = None,
max_capacity: Optional[float] = None,
max_retries: Optional[int] = None,
description: Optional[str] = None,
non_overridable_arguments: Optional[Mapping[str, str]] = None,
notification_property: Optional[JobNotificationPropertyArgs] = None,
number_of_workers: Optional[int] = None,
connections: Optional[Sequence[str]] = None,
security_configuration: Optional[str] = None,
tags: Optional[Mapping[str, str]] = None,
timeout: Optional[int] = None,
worker_type: Optional[str] = None)
func NewJob(ctx *Context, name string, args JobArgs, opts ...ResourceOption) (*Job, error)
public Job(string name, JobArgs args, CustomResourceOptions? opts = null)
type: aws:glue:Job
properties: # The arguments to resource properties.
options: # Bag of options to control resource's behavior.
Parameters
- name string
- The unique name of the resource.
- args JobArgs
- The arguments to resource properties.
- opts CustomResourceOptions
- Bag of options to control resource's behavior.
- resource_name str
- The unique name of the resource.
- args JobArgs
- The arguments to resource properties.
- opts ResourceOptions
- Bag of options to control resource's behavior.
- ctx Context
- Context object for the current deployment.
- name string
- The unique name of the resource.
- args JobArgs
- The arguments to resource properties.
- opts ResourceOption
- Bag of options to control resource's behavior.
- name string
- The unique name of the resource.
- args JobArgs
- The arguments to resource properties.
- opts CustomResourceOptions
- Bag of options to control resource's behavior.
- name String
- The unique name of the resource.
- args JobArgs
- The arguments to resource properties.
- options CustomResourceOptions
- Bag of options to control resource's behavior.
Constructor example
The following reference example uses placeholder values for all input properties.
var jobResource = new Aws.Glue.Job("jobResource", new()
{
Command = new Aws.Glue.Inputs.JobCommandArgs
{
ScriptLocation = "string",
Name = "string",
PythonVersion = "string",
Runtime = "string",
},
RoleArn = "string",
ExecutionProperty = new Aws.Glue.Inputs.JobExecutionPropertyArgs
{
MaxConcurrentRuns = 0,
},
Name = "string",
ExecutionClass = "string",
DefaultArguments =
{
{ "string", "string" },
},
GlueVersion = "string",
JobRunQueuingEnabled = false,
MaintenanceWindow = "string",
MaxCapacity = 0,
MaxRetries = 0,
Description = "string",
NonOverridableArguments =
{
{ "string", "string" },
},
NotificationProperty = new Aws.Glue.Inputs.JobNotificationPropertyArgs
{
NotifyDelayAfter = 0,
},
NumberOfWorkers = 0,
Connections = new[]
{
"string",
},
SecurityConfiguration = "string",
Tags =
{
{ "string", "string" },
},
Timeout = 0,
WorkerType = "string",
});
example, err := glue.NewJob(ctx, "jobResource", &glue.JobArgs{
Command: &glue.JobCommandArgs{
ScriptLocation: pulumi.String("string"),
Name: pulumi.String("string"),
PythonVersion: pulumi.String("string"),
Runtime: pulumi.String("string"),
},
RoleArn: pulumi.String("string"),
ExecutionProperty: &glue.JobExecutionPropertyArgs{
MaxConcurrentRuns: pulumi.Int(0),
},
Name: pulumi.String("string"),
ExecutionClass: pulumi.String("string"),
DefaultArguments: pulumi.StringMap{
"string": pulumi.String("string"),
},
GlueVersion: pulumi.String("string"),
JobRunQueuingEnabled: pulumi.Bool(false),
MaintenanceWindow: pulumi.String("string"),
MaxCapacity: pulumi.Float64(0),
MaxRetries: pulumi.Int(0),
Description: pulumi.String("string"),
NonOverridableArguments: pulumi.StringMap{
"string": pulumi.String("string"),
},
NotificationProperty: &glue.JobNotificationPropertyArgs{
NotifyDelayAfter: pulumi.Int(0),
},
NumberOfWorkers: pulumi.Int(0),
Connections: pulumi.StringArray{
pulumi.String("string"),
},
SecurityConfiguration: pulumi.String("string"),
Tags: pulumi.StringMap{
"string": pulumi.String("string"),
},
Timeout: pulumi.Int(0),
WorkerType: pulumi.String("string"),
})
var jobResource = new Job("jobResource", JobArgs.builder()
.command(JobCommandArgs.builder()
.scriptLocation("string")
.name("string")
.pythonVersion("string")
.runtime("string")
.build())
.roleArn("string")
.executionProperty(JobExecutionPropertyArgs.builder()
.maxConcurrentRuns(0)
.build())
.name("string")
.executionClass("string")
.defaultArguments(Map.of("string", "string"))
.glueVersion("string")
.jobRunQueuingEnabled(false)
.maintenanceWindow("string")
.maxCapacity(0)
.maxRetries(0)
.description("string")
.nonOverridableArguments(Map.of("string", "string"))
.notificationProperty(JobNotificationPropertyArgs.builder()
.notifyDelayAfter(0)
.build())
.numberOfWorkers(0)
.connections("string")
.securityConfiguration("string")
.tags(Map.of("string", "string"))
.timeout(0)
.workerType("string")
.build());
job_resource = aws.glue.Job("jobResource",
command={
"scriptLocation": "string",
"name": "string",
"pythonVersion": "string",
"runtime": "string",
},
role_arn="string",
execution_property={
"maxConcurrentRuns": 0,
},
name="string",
execution_class="string",
default_arguments={
"string": "string",
},
glue_version="string",
job_run_queuing_enabled=False,
maintenance_window="string",
max_capacity=0,
max_retries=0,
description="string",
non_overridable_arguments={
"string": "string",
},
notification_property={
"notifyDelayAfter": 0,
},
number_of_workers=0,
connections=["string"],
security_configuration="string",
tags={
"string": "string",
},
timeout=0,
worker_type="string")
const jobResource = new aws.glue.Job("jobResource", {
command: {
scriptLocation: "string",
name: "string",
pythonVersion: "string",
runtime: "string",
},
roleArn: "string",
executionProperty: {
maxConcurrentRuns: 0,
},
name: "string",
executionClass: "string",
defaultArguments: {
string: "string",
},
glueVersion: "string",
jobRunQueuingEnabled: false,
maintenanceWindow: "string",
maxCapacity: 0,
maxRetries: 0,
description: "string",
nonOverridableArguments: {
string: "string",
},
notificationProperty: {
notifyDelayAfter: 0,
},
numberOfWorkers: 0,
connections: ["string"],
securityConfiguration: "string",
tags: {
string: "string",
},
timeout: 0,
workerType: "string",
});
type: aws:glue:Job
properties:
command:
name: string
pythonVersion: string
runtime: string
scriptLocation: string
connections:
- string
defaultArguments:
string: string
description: string
executionClass: string
executionProperty:
maxConcurrentRuns: 0
glueVersion: string
jobRunQueuingEnabled: false
maintenanceWindow: string
maxCapacity: 0
maxRetries: 0
name: string
nonOverridableArguments:
string: string
notificationProperty:
notifyDelayAfter: 0
numberOfWorkers: 0
roleArn: string
securityConfiguration: string
tags:
string: string
timeout: 0
workerType: string
Job Resource Properties
To learn more about resource properties and how to use them, see Inputs and Outputs in the Architecture and Concepts docs.
Inputs
The Job resource accepts the following input properties:
- Command
Job
Command - The command of the job. Defined below.
- Role
Arn string - The ARN of the IAM role associated with this job.
- Connections List<string>
- The list of connections used for this job.
- Default
Arguments Dictionary<string, string> - The map of default arguments for this job. You can specify arguments here that your own job-execution script consumes, as well as arguments that AWS Glue itself consumes. For information about how to specify and consume your own Job arguments, see the Calling AWS Glue APIs in Python topic in the developer guide. For information about the key-value pairs that AWS Glue consumes to set up your job, see the Special Parameters Used by AWS Glue topic in the developer guide.
- Description string
- Description of the job.
- Execution
Class string - Indicates whether the job is run with a standard or flexible execution class. The standard execution class is ideal for time-sensitive workloads that require fast job startup and dedicated resources. Valid value:
FLEX
,STANDARD
. - Execution
Property JobExecution Property - Execution property of the job. Defined below.
- Glue
Version string - The version of glue to use, for example "1.0". Ray jobs should set this to 4.0 or greater. For information about available versions, see the AWS Glue Release Notes.
- Job
Run boolQueuing Enabled - Specifies whether job run queuing is enabled for the job runs for this job. A value of true means job run queuing is enabled for the job runs. If false or not populated, the job runs will not be considered for queueing.
- Maintenance
Window string - Specifies the day of the week and hour for the maintenance window for streaming jobs.
- Max
Capacity double - The maximum number of AWS Glue data processing units (DPUs) that can be allocated when this job runs.
Required
whenpythonshell
is set, accept either0.0625
or1.0
. Usenumber_of_workers
andworker_type
arguments instead withglue_version
2.0
and above. - Max
Retries int - The maximum number of times to retry this job if it fails.
- Name string
- The name you assign to this job. It must be unique in your account.
- Non
Overridable Dictionary<string, string>Arguments - Non-overridable arguments for this job, specified as name-value pairs.
- Notification
Property JobNotification Property - Notification property of the job. Defined below.
- Number
Of intWorkers - The number of workers of a defined workerType that are allocated when a job runs.
- Security
Configuration string - The name of the Security Configuration to be associated with the job.
- Dictionary<string, string>
- Key-value map of resource tags. If configured with a provider
default_tags
configuration block present, tags with matching keys will overwrite those defined at the provider-level. - Timeout int
- The job timeout in minutes. The default is 2880 minutes (48 hours) for
glueetl
andpythonshell
jobs, and null (unlimited) forgluestreaming
jobs. - Worker
Type string - The type of predefined worker that is allocated when a job runs. Accepts a value of Standard, G.1X, G.2X, or G.025X for Spark jobs. Accepts the value Z.2X for Ray jobs.
- For the Standard worker type, each worker provides 4 vCPU, 16 GB of memory and a 50GB disk, and 2 executors per worker.
- For the G.1X worker type, each worker maps to 1 DPU (4 vCPU, 16 GB of memory, 64 GB disk), and provides 1 executor per worker. Recommended for memory-intensive jobs.
- For the G.2X worker type, each worker maps to 2 DPU (8 vCPU, 32 GB of memory, 128 GB disk), and provides 1 executor per worker. Recommended for memory-intensive jobs.
- For the G.4X worker type, each worker maps to 4 DPU (16 vCPUs, 64 GB of memory) with 256GB disk (approximately 235GB free), and provides 1 executor per worker. Recommended for memory-intensive jobs. Only available for Glue version 3.0. Available AWS Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm).
- For the G.8X worker type, each worker maps to 8 DPU (32 vCPUs, 128 GB of memory) with 512GB disk (approximately 487GB free), and provides 1 executor per worker. Recommended for memory-intensive jobs. Only available for Glue version 3.0. Available AWS Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm).
- For the G.025X worker type, each worker maps to 0.25 DPU (2 vCPU, 4GB of memory, 64 GB disk), and provides 1 executor per worker. Recommended for low volume streaming jobs. Only available for Glue version 3.0.
- For the Z.2X worker type, each worker maps to 2 M-DPU (8vCPU, 64 GB of m emory, 128 GB disk), and provides up to 8 Ray workers based on the autoscaler.
- Command
Job
Command Args - The command of the job. Defined below.
- Role
Arn string - The ARN of the IAM role associated with this job.
- Connections []string
- The list of connections used for this job.
- Default
Arguments map[string]string - The map of default arguments for this job. You can specify arguments here that your own job-execution script consumes, as well as arguments that AWS Glue itself consumes. For information about how to specify and consume your own Job arguments, see the Calling AWS Glue APIs in Python topic in the developer guide. For information about the key-value pairs that AWS Glue consumes to set up your job, see the Special Parameters Used by AWS Glue topic in the developer guide.
- Description string
- Description of the job.
- Execution
Class string - Indicates whether the job is run with a standard or flexible execution class. The standard execution class is ideal for time-sensitive workloads that require fast job startup and dedicated resources. Valid value:
FLEX
,STANDARD
. - Execution
Property JobExecution Property Args - Execution property of the job. Defined below.
- Glue
Version string - The version of glue to use, for example "1.0". Ray jobs should set this to 4.0 or greater. For information about available versions, see the AWS Glue Release Notes.
- Job
Run boolQueuing Enabled - Specifies whether job run queuing is enabled for the job runs for this job. A value of true means job run queuing is enabled for the job runs. If false or not populated, the job runs will not be considered for queueing.
- Maintenance
Window string - Specifies the day of the week and hour for the maintenance window for streaming jobs.
- Max
Capacity float64 - The maximum number of AWS Glue data processing units (DPUs) that can be allocated when this job runs.
Required
whenpythonshell
is set, accept either0.0625
or1.0
. Usenumber_of_workers
andworker_type
arguments instead withglue_version
2.0
and above. - Max
Retries int - The maximum number of times to retry this job if it fails.
- Name string
- The name you assign to this job. It must be unique in your account.
- Non
Overridable map[string]stringArguments - Non-overridable arguments for this job, specified as name-value pairs.
- Notification
Property JobNotification Property Args - Notification property of the job. Defined below.
- Number
Of intWorkers - The number of workers of a defined workerType that are allocated when a job runs.
- Security
Configuration string - The name of the Security Configuration to be associated with the job.
- map[string]string
- Key-value map of resource tags. If configured with a provider
default_tags
configuration block present, tags with matching keys will overwrite those defined at the provider-level. - Timeout int
- The job timeout in minutes. The default is 2880 minutes (48 hours) for
glueetl
andpythonshell
jobs, and null (unlimited) forgluestreaming
jobs. - Worker
Type string - The type of predefined worker that is allocated when a job runs. Accepts a value of Standard, G.1X, G.2X, or G.025X for Spark jobs. Accepts the value Z.2X for Ray jobs.
- For the Standard worker type, each worker provides 4 vCPU, 16 GB of memory and a 50GB disk, and 2 executors per worker.
- For the G.1X worker type, each worker maps to 1 DPU (4 vCPU, 16 GB of memory, 64 GB disk), and provides 1 executor per worker. Recommended for memory-intensive jobs.
- For the G.2X worker type, each worker maps to 2 DPU (8 vCPU, 32 GB of memory, 128 GB disk), and provides 1 executor per worker. Recommended for memory-intensive jobs.
- For the G.4X worker type, each worker maps to 4 DPU (16 vCPUs, 64 GB of memory) with 256GB disk (approximately 235GB free), and provides 1 executor per worker. Recommended for memory-intensive jobs. Only available for Glue version 3.0. Available AWS Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm).
- For the G.8X worker type, each worker maps to 8 DPU (32 vCPUs, 128 GB of memory) with 512GB disk (approximately 487GB free), and provides 1 executor per worker. Recommended for memory-intensive jobs. Only available for Glue version 3.0. Available AWS Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm).
- For the G.025X worker type, each worker maps to 0.25 DPU (2 vCPU, 4GB of memory, 64 GB disk), and provides 1 executor per worker. Recommended for low volume streaming jobs. Only available for Glue version 3.0.
- For the Z.2X worker type, each worker maps to 2 M-DPU (8vCPU, 64 GB of m emory, 128 GB disk), and provides up to 8 Ray workers based on the autoscaler.
- command
Job
Command - The command of the job. Defined below.
- role
Arn String - The ARN of the IAM role associated with this job.
- connections List<String>
- The list of connections used for this job.
- default
Arguments Map<String,String> - The map of default arguments for this job. You can specify arguments here that your own job-execution script consumes, as well as arguments that AWS Glue itself consumes. For information about how to specify and consume your own Job arguments, see the Calling AWS Glue APIs in Python topic in the developer guide. For information about the key-value pairs that AWS Glue consumes to set up your job, see the Special Parameters Used by AWS Glue topic in the developer guide.
- description String
- Description of the job.
- execution
Class String - Indicates whether the job is run with a standard or flexible execution class. The standard execution class is ideal for time-sensitive workloads that require fast job startup and dedicated resources. Valid value:
FLEX
,STANDARD
. - execution
Property JobExecution Property - Execution property of the job. Defined below.
- glue
Version String - The version of glue to use, for example "1.0". Ray jobs should set this to 4.0 or greater. For information about available versions, see the AWS Glue Release Notes.
- job
Run BooleanQueuing Enabled - Specifies whether job run queuing is enabled for the job runs for this job. A value of true means job run queuing is enabled for the job runs. If false or not populated, the job runs will not be considered for queueing.
- maintenance
Window String - Specifies the day of the week and hour for the maintenance window for streaming jobs.
- max
Capacity Double - The maximum number of AWS Glue data processing units (DPUs) that can be allocated when this job runs.
Required
whenpythonshell
is set, accept either0.0625
or1.0
. Usenumber_of_workers
andworker_type
arguments instead withglue_version
2.0
and above. - max
Retries Integer - The maximum number of times to retry this job if it fails.
- name String
- The name you assign to this job. It must be unique in your account.
- non
Overridable Map<String,String>Arguments - Non-overridable arguments for this job, specified as name-value pairs.
- notification
Property JobNotification Property - Notification property of the job. Defined below.
- number
Of IntegerWorkers - The number of workers of a defined workerType that are allocated when a job runs.
- security
Configuration String - The name of the Security Configuration to be associated with the job.
- Map<String,String>
- Key-value map of resource tags. If configured with a provider
default_tags
configuration block present, tags with matching keys will overwrite those defined at the provider-level. - timeout Integer
- The job timeout in minutes. The default is 2880 minutes (48 hours) for
glueetl
andpythonshell
jobs, and null (unlimited) forgluestreaming
jobs. - worker
Type String - The type of predefined worker that is allocated when a job runs. Accepts a value of Standard, G.1X, G.2X, or G.025X for Spark jobs. Accepts the value Z.2X for Ray jobs.
- For the Standard worker type, each worker provides 4 vCPU, 16 GB of memory and a 50GB disk, and 2 executors per worker.
- For the G.1X worker type, each worker maps to 1 DPU (4 vCPU, 16 GB of memory, 64 GB disk), and provides 1 executor per worker. Recommended for memory-intensive jobs.
- For the G.2X worker type, each worker maps to 2 DPU (8 vCPU, 32 GB of memory, 128 GB disk), and provides 1 executor per worker. Recommended for memory-intensive jobs.
- For the G.4X worker type, each worker maps to 4 DPU (16 vCPUs, 64 GB of memory) with 256GB disk (approximately 235GB free), and provides 1 executor per worker. Recommended for memory-intensive jobs. Only available for Glue version 3.0. Available AWS Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm).
- For the G.8X worker type, each worker maps to 8 DPU (32 vCPUs, 128 GB of memory) with 512GB disk (approximately 487GB free), and provides 1 executor per worker. Recommended for memory-intensive jobs. Only available for Glue version 3.0. Available AWS Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm).
- For the G.025X worker type, each worker maps to 0.25 DPU (2 vCPU, 4GB of memory, 64 GB disk), and provides 1 executor per worker. Recommended for low volume streaming jobs. Only available for Glue version 3.0.
- For the Z.2X worker type, each worker maps to 2 M-DPU (8vCPU, 64 GB of m emory, 128 GB disk), and provides up to 8 Ray workers based on the autoscaler.
- command
Job
Command - The command of the job. Defined below.
- role
Arn string - The ARN of the IAM role associated with this job.
- connections string[]
- The list of connections used for this job.
- default
Arguments {[key: string]: string} - The map of default arguments for this job. You can specify arguments here that your own job-execution script consumes, as well as arguments that AWS Glue itself consumes. For information about how to specify and consume your own Job arguments, see the Calling AWS Glue APIs in Python topic in the developer guide. For information about the key-value pairs that AWS Glue consumes to set up your job, see the Special Parameters Used by AWS Glue topic in the developer guide.
- description string
- Description of the job.
- execution
Class string - Indicates whether the job is run with a standard or flexible execution class. The standard execution class is ideal for time-sensitive workloads that require fast job startup and dedicated resources. Valid value:
FLEX
,STANDARD
. - execution
Property JobExecution Property - Execution property of the job. Defined below.
- glue
Version string - The version of glue to use, for example "1.0". Ray jobs should set this to 4.0 or greater. For information about available versions, see the AWS Glue Release Notes.
- job
Run booleanQueuing Enabled - Specifies whether job run queuing is enabled for the job runs for this job. A value of true means job run queuing is enabled for the job runs. If false or not populated, the job runs will not be considered for queueing.
- maintenance
Window string - Specifies the day of the week and hour for the maintenance window for streaming jobs.
- max
Capacity number - The maximum number of AWS Glue data processing units (DPUs) that can be allocated when this job runs.
Required
whenpythonshell
is set, accept either0.0625
or1.0
. Usenumber_of_workers
andworker_type
arguments instead withglue_version
2.0
and above. - max
Retries number - The maximum number of times to retry this job if it fails.
- name string
- The name you assign to this job. It must be unique in your account.
- non
Overridable {[key: string]: string}Arguments - Non-overridable arguments for this job, specified as name-value pairs.
- notification
Property JobNotification Property - Notification property of the job. Defined below.
- number
Of numberWorkers - The number of workers of a defined workerType that are allocated when a job runs.
- security
Configuration string - The name of the Security Configuration to be associated with the job.
- {[key: string]: string}
- Key-value map of resource tags. If configured with a provider
default_tags
configuration block present, tags with matching keys will overwrite those defined at the provider-level. - timeout number
- The job timeout in minutes. The default is 2880 minutes (48 hours) for
glueetl
andpythonshell
jobs, and null (unlimited) forgluestreaming
jobs. - worker
Type string - The type of predefined worker that is allocated when a job runs. Accepts a value of Standard, G.1X, G.2X, or G.025X for Spark jobs. Accepts the value Z.2X for Ray jobs.
- For the Standard worker type, each worker provides 4 vCPU, 16 GB of memory and a 50GB disk, and 2 executors per worker.
- For the G.1X worker type, each worker maps to 1 DPU (4 vCPU, 16 GB of memory, 64 GB disk), and provides 1 executor per worker. Recommended for memory-intensive jobs.
- For the G.2X worker type, each worker maps to 2 DPU (8 vCPU, 32 GB of memory, 128 GB disk), and provides 1 executor per worker. Recommended for memory-intensive jobs.
- For the G.4X worker type, each worker maps to 4 DPU (16 vCPUs, 64 GB of memory) with 256GB disk (approximately 235GB free), and provides 1 executor per worker. Recommended for memory-intensive jobs. Only available for Glue version 3.0. Available AWS Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm).
- For the G.8X worker type, each worker maps to 8 DPU (32 vCPUs, 128 GB of memory) with 512GB disk (approximately 487GB free), and provides 1 executor per worker. Recommended for memory-intensive jobs. Only available for Glue version 3.0. Available AWS Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm).
- For the G.025X worker type, each worker maps to 0.25 DPU (2 vCPU, 4GB of memory, 64 GB disk), and provides 1 executor per worker. Recommended for low volume streaming jobs. Only available for Glue version 3.0.
- For the Z.2X worker type, each worker maps to 2 M-DPU (8vCPU, 64 GB of m emory, 128 GB disk), and provides up to 8 Ray workers based on the autoscaler.
- command
Job
Command Args - The command of the job. Defined below.
- role_
arn str - The ARN of the IAM role associated with this job.
- connections Sequence[str]
- The list of connections used for this job.
- default_
arguments Mapping[str, str] - The map of default arguments for this job. You can specify arguments here that your own job-execution script consumes, as well as arguments that AWS Glue itself consumes. For information about how to specify and consume your own Job arguments, see the Calling AWS Glue APIs in Python topic in the developer guide. For information about the key-value pairs that AWS Glue consumes to set up your job, see the Special Parameters Used by AWS Glue topic in the developer guide.
- description str
- Description of the job.
- execution_
class str - Indicates whether the job is run with a standard or flexible execution class. The standard execution class is ideal for time-sensitive workloads that require fast job startup and dedicated resources. Valid value:
FLEX
,STANDARD
. - execution_
property JobExecution Property Args - Execution property of the job. Defined below.
- glue_
version str - The version of glue to use, for example "1.0". Ray jobs should set this to 4.0 or greater. For information about available versions, see the AWS Glue Release Notes.
- job_
run_ boolqueuing_ enabled - Specifies whether job run queuing is enabled for the job runs for this job. A value of true means job run queuing is enabled for the job runs. If false or not populated, the job runs will not be considered for queueing.
- maintenance_
window str - Specifies the day of the week and hour for the maintenance window for streaming jobs.
- max_
capacity float - The maximum number of AWS Glue data processing units (DPUs) that can be allocated when this job runs.
Required
whenpythonshell
is set, accept either0.0625
or1.0
. Usenumber_of_workers
andworker_type
arguments instead withglue_version
2.0
and above. - max_
retries int - The maximum number of times to retry this job if it fails.
- name str
- The name you assign to this job. It must be unique in your account.
- non_
overridable_ Mapping[str, str]arguments - Non-overridable arguments for this job, specified as name-value pairs.
- notification_
property JobNotification Property Args - Notification property of the job. Defined below.
- number_
of_ intworkers - The number of workers of a defined workerType that are allocated when a job runs.
- security_
configuration str - The name of the Security Configuration to be associated with the job.
- Mapping[str, str]
- Key-value map of resource tags. If configured with a provider
default_tags
configuration block present, tags with matching keys will overwrite those defined at the provider-level. - timeout int
- The job timeout in minutes. The default is 2880 minutes (48 hours) for
glueetl
andpythonshell
jobs, and null (unlimited) forgluestreaming
jobs. - worker_
type str - The type of predefined worker that is allocated when a job runs. Accepts a value of Standard, G.1X, G.2X, or G.025X for Spark jobs. Accepts the value Z.2X for Ray jobs.
- For the Standard worker type, each worker provides 4 vCPU, 16 GB of memory and a 50GB disk, and 2 executors per worker.
- For the G.1X worker type, each worker maps to 1 DPU (4 vCPU, 16 GB of memory, 64 GB disk), and provides 1 executor per worker. Recommended for memory-intensive jobs.
- For the G.2X worker type, each worker maps to 2 DPU (8 vCPU, 32 GB of memory, 128 GB disk), and provides 1 executor per worker. Recommended for memory-intensive jobs.
- For the G.4X worker type, each worker maps to 4 DPU (16 vCPUs, 64 GB of memory) with 256GB disk (approximately 235GB free), and provides 1 executor per worker. Recommended for memory-intensive jobs. Only available for Glue version 3.0. Available AWS Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm).
- For the G.8X worker type, each worker maps to 8 DPU (32 vCPUs, 128 GB of memory) with 512GB disk (approximately 487GB free), and provides 1 executor per worker. Recommended for memory-intensive jobs. Only available for Glue version 3.0. Available AWS Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm).
- For the G.025X worker type, each worker maps to 0.25 DPU (2 vCPU, 4GB of memory, 64 GB disk), and provides 1 executor per worker. Recommended for low volume streaming jobs. Only available for Glue version 3.0.
- For the Z.2X worker type, each worker maps to 2 M-DPU (8vCPU, 64 GB of m emory, 128 GB disk), and provides up to 8 Ray workers based on the autoscaler.
- command Property Map
- The command of the job. Defined below.
- role
Arn String - The ARN of the IAM role associated with this job.
- connections List<String>
- The list of connections used for this job.
- default
Arguments Map<String> - The map of default arguments for this job. You can specify arguments here that your own job-execution script consumes, as well as arguments that AWS Glue itself consumes. For information about how to specify and consume your own Job arguments, see the Calling AWS Glue APIs in Python topic in the developer guide. For information about the key-value pairs that AWS Glue consumes to set up your job, see the Special Parameters Used by AWS Glue topic in the developer guide.
- description String
- Description of the job.
- execution
Class String - Indicates whether the job is run with a standard or flexible execution class. The standard execution class is ideal for time-sensitive workloads that require fast job startup and dedicated resources. Valid value:
FLEX
,STANDARD
. - execution
Property Property Map - Execution property of the job. Defined below.
- glue
Version String - The version of glue to use, for example "1.0". Ray jobs should set this to 4.0 or greater. For information about available versions, see the AWS Glue Release Notes.
- job
Run BooleanQueuing Enabled - Specifies whether job run queuing is enabled for the job runs for this job. A value of true means job run queuing is enabled for the job runs. If false or not populated, the job runs will not be considered for queueing.
- maintenance
Window String - Specifies the day of the week and hour for the maintenance window for streaming jobs.
- max
Capacity Number - The maximum number of AWS Glue data processing units (DPUs) that can be allocated when this job runs.
Required
whenpythonshell
is set, accept either0.0625
or1.0
. Usenumber_of_workers
andworker_type
arguments instead withglue_version
2.0
and above. - max
Retries Number - The maximum number of times to retry this job if it fails.
- name String
- The name you assign to this job. It must be unique in your account.
- non
Overridable Map<String>Arguments - Non-overridable arguments for this job, specified as name-value pairs.
- notification
Property Property Map - Notification property of the job. Defined below.
- number
Of NumberWorkers - The number of workers of a defined workerType that are allocated when a job runs.
- security
Configuration String - The name of the Security Configuration to be associated with the job.
- Map<String>
- Key-value map of resource tags. If configured with a provider
default_tags
configuration block present, tags with matching keys will overwrite those defined at the provider-level. - timeout Number
- The job timeout in minutes. The default is 2880 minutes (48 hours) for
glueetl
andpythonshell
jobs, and null (unlimited) forgluestreaming
jobs. - worker
Type String - The type of predefined worker that is allocated when a job runs. Accepts a value of Standard, G.1X, G.2X, or G.025X for Spark jobs. Accepts the value Z.2X for Ray jobs.
- For the Standard worker type, each worker provides 4 vCPU, 16 GB of memory and a 50GB disk, and 2 executors per worker.
- For the G.1X worker type, each worker maps to 1 DPU (4 vCPU, 16 GB of memory, 64 GB disk), and provides 1 executor per worker. Recommended for memory-intensive jobs.
- For the G.2X worker type, each worker maps to 2 DPU (8 vCPU, 32 GB of memory, 128 GB disk), and provides 1 executor per worker. Recommended for memory-intensive jobs.
- For the G.4X worker type, each worker maps to 4 DPU (16 vCPUs, 64 GB of memory) with 256GB disk (approximately 235GB free), and provides 1 executor per worker. Recommended for memory-intensive jobs. Only available for Glue version 3.0. Available AWS Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm).
- For the G.8X worker type, each worker maps to 8 DPU (32 vCPUs, 128 GB of memory) with 512GB disk (approximately 487GB free), and provides 1 executor per worker. Recommended for memory-intensive jobs. Only available for Glue version 3.0. Available AWS Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm).
- For the G.025X worker type, each worker maps to 0.25 DPU (2 vCPU, 4GB of memory, 64 GB disk), and provides 1 executor per worker. Recommended for low volume streaming jobs. Only available for Glue version 3.0.
- For the Z.2X worker type, each worker maps to 2 M-DPU (8vCPU, 64 GB of m emory, 128 GB disk), and provides up to 8 Ray workers based on the autoscaler.
Outputs
All input properties are implicitly available as output properties. Additionally, the Job resource produces the following output properties:
Look up Existing Job Resource
Get an existing Job resource’s state with the given name, ID, and optional extra properties used to qualify the lookup.
public static get(name: string, id: Input<ID>, state?: JobState, opts?: CustomResourceOptions): Job
@staticmethod
def get(resource_name: str,
id: str,
opts: Optional[ResourceOptions] = None,
arn: Optional[str] = None,
command: Optional[JobCommandArgs] = None,
connections: Optional[Sequence[str]] = None,
default_arguments: Optional[Mapping[str, str]] = None,
description: Optional[str] = None,
execution_class: Optional[str] = None,
execution_property: Optional[JobExecutionPropertyArgs] = None,
glue_version: Optional[str] = None,
job_run_queuing_enabled: Optional[bool] = None,
maintenance_window: Optional[str] = None,
max_capacity: Optional[float] = None,
max_retries: Optional[int] = None,
name: Optional[str] = None,
non_overridable_arguments: Optional[Mapping[str, str]] = None,
notification_property: Optional[JobNotificationPropertyArgs] = None,
number_of_workers: Optional[int] = None,
role_arn: Optional[str] = None,
security_configuration: Optional[str] = None,
tags: Optional[Mapping[str, str]] = None,
tags_all: Optional[Mapping[str, str]] = None,
timeout: Optional[int] = None,
worker_type: Optional[str] = None) -> Job
func GetJob(ctx *Context, name string, id IDInput, state *JobState, opts ...ResourceOption) (*Job, error)
public static Job Get(string name, Input<string> id, JobState? state, CustomResourceOptions? opts = null)
public static Job get(String name, Output<String> id, JobState state, CustomResourceOptions options)
Resource lookup is not supported in YAML
- name
- The unique name of the resulting resource.
- id
- The unique provider ID of the resource to lookup.
- state
- Any extra arguments used during the lookup.
- opts
- A bag of options that control this resource's behavior.
- resource_name
- The unique name of the resulting resource.
- id
- The unique provider ID of the resource to lookup.
- name
- The unique name of the resulting resource.
- id
- The unique provider ID of the resource to lookup.
- state
- Any extra arguments used during the lookup.
- opts
- A bag of options that control this resource's behavior.
- name
- The unique name of the resulting resource.
- id
- The unique provider ID of the resource to lookup.
- state
- Any extra arguments used during the lookup.
- opts
- A bag of options that control this resource's behavior.
- name
- The unique name of the resulting resource.
- id
- The unique provider ID of the resource to lookup.
- state
- Any extra arguments used during the lookup.
- opts
- A bag of options that control this resource's behavior.
- Arn string
- Amazon Resource Name (ARN) of Glue Job
- Command
Job
Command - The command of the job. Defined below.
- Connections List<string>
- The list of connections used for this job.
- Default
Arguments Dictionary<string, string> - The map of default arguments for this job. You can specify arguments here that your own job-execution script consumes, as well as arguments that AWS Glue itself consumes. For information about how to specify and consume your own Job arguments, see the Calling AWS Glue APIs in Python topic in the developer guide. For information about the key-value pairs that AWS Glue consumes to set up your job, see the Special Parameters Used by AWS Glue topic in the developer guide.
- Description string
- Description of the job.
- Execution
Class string - Indicates whether the job is run with a standard or flexible execution class. The standard execution class is ideal for time-sensitive workloads that require fast job startup and dedicated resources. Valid value:
FLEX
,STANDARD
. - Execution
Property JobExecution Property - Execution property of the job. Defined below.
- Glue
Version string - The version of glue to use, for example "1.0". Ray jobs should set this to 4.0 or greater. For information about available versions, see the AWS Glue Release Notes.
- Job
Run boolQueuing Enabled - Specifies whether job run queuing is enabled for the job runs for this job. A value of true means job run queuing is enabled for the job runs. If false or not populated, the job runs will not be considered for queueing.
- Maintenance
Window string - Specifies the day of the week and hour for the maintenance window for streaming jobs.
- Max
Capacity double - The maximum number of AWS Glue data processing units (DPUs) that can be allocated when this job runs.
Required
whenpythonshell
is set, accept either0.0625
or1.0
. Usenumber_of_workers
andworker_type
arguments instead withglue_version
2.0
and above. - Max
Retries int - The maximum number of times to retry this job if it fails.
- Name string
- The name you assign to this job. It must be unique in your account.
- Non
Overridable Dictionary<string, string>Arguments - Non-overridable arguments for this job, specified as name-value pairs.
- Notification
Property JobNotification Property - Notification property of the job. Defined below.
- Number
Of intWorkers - The number of workers of a defined workerType that are allocated when a job runs.
- Role
Arn string - The ARN of the IAM role associated with this job.
- Security
Configuration string - The name of the Security Configuration to be associated with the job.
- Dictionary<string, string>
- Key-value map of resource tags. If configured with a provider
default_tags
configuration block present, tags with matching keys will overwrite those defined at the provider-level. - Dictionary<string, string>
- A map of tags assigned to the resource, including those inherited from the provider
default_tags
configuration block. - Timeout int
- The job timeout in minutes. The default is 2880 minutes (48 hours) for
glueetl
andpythonshell
jobs, and null (unlimited) forgluestreaming
jobs. - Worker
Type string - The type of predefined worker that is allocated when a job runs. Accepts a value of Standard, G.1X, G.2X, or G.025X for Spark jobs. Accepts the value Z.2X for Ray jobs.
- For the Standard worker type, each worker provides 4 vCPU, 16 GB of memory and a 50GB disk, and 2 executors per worker.
- For the G.1X worker type, each worker maps to 1 DPU (4 vCPU, 16 GB of memory, 64 GB disk), and provides 1 executor per worker. Recommended for memory-intensive jobs.
- For the G.2X worker type, each worker maps to 2 DPU (8 vCPU, 32 GB of memory, 128 GB disk), and provides 1 executor per worker. Recommended for memory-intensive jobs.
- For the G.4X worker type, each worker maps to 4 DPU (16 vCPUs, 64 GB of memory) with 256GB disk (approximately 235GB free), and provides 1 executor per worker. Recommended for memory-intensive jobs. Only available for Glue version 3.0. Available AWS Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm).
- For the G.8X worker type, each worker maps to 8 DPU (32 vCPUs, 128 GB of memory) with 512GB disk (approximately 487GB free), and provides 1 executor per worker. Recommended for memory-intensive jobs. Only available for Glue version 3.0. Available AWS Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm).
- For the G.025X worker type, each worker maps to 0.25 DPU (2 vCPU, 4GB of memory, 64 GB disk), and provides 1 executor per worker. Recommended for low volume streaming jobs. Only available for Glue version 3.0.
- For the Z.2X worker type, each worker maps to 2 M-DPU (8vCPU, 64 GB of m emory, 128 GB disk), and provides up to 8 Ray workers based on the autoscaler.
- Arn string
- Amazon Resource Name (ARN) of Glue Job
- Command
Job
Command Args - The command of the job. Defined below.
- Connections []string
- The list of connections used for this job.
- Default
Arguments map[string]string - The map of default arguments for this job. You can specify arguments here that your own job-execution script consumes, as well as arguments that AWS Glue itself consumes. For information about how to specify and consume your own Job arguments, see the Calling AWS Glue APIs in Python topic in the developer guide. For information about the key-value pairs that AWS Glue consumes to set up your job, see the Special Parameters Used by AWS Glue topic in the developer guide.
- Description string
- Description of the job.
- Execution
Class string - Indicates whether the job is run with a standard or flexible execution class. The standard execution class is ideal for time-sensitive workloads that require fast job startup and dedicated resources. Valid value:
FLEX
,STANDARD
. - Execution
Property JobExecution Property Args - Execution property of the job. Defined below.
- Glue
Version string - The version of glue to use, for example "1.0". Ray jobs should set this to 4.0 or greater. For information about available versions, see the AWS Glue Release Notes.
- Job
Run boolQueuing Enabled - Specifies whether job run queuing is enabled for the job runs for this job. A value of true means job run queuing is enabled for the job runs. If false or not populated, the job runs will not be considered for queueing.
- Maintenance
Window string - Specifies the day of the week and hour for the maintenance window for streaming jobs.
- Max
Capacity float64 - The maximum number of AWS Glue data processing units (DPUs) that can be allocated when this job runs.
Required
whenpythonshell
is set, accept either0.0625
or1.0
. Usenumber_of_workers
andworker_type
arguments instead withglue_version
2.0
and above. - Max
Retries int - The maximum number of times to retry this job if it fails.
- Name string
- The name you assign to this job. It must be unique in your account.
- Non
Overridable map[string]stringArguments - Non-overridable arguments for this job, specified as name-value pairs.
- Notification
Property JobNotification Property Args - Notification property of the job. Defined below.
- Number
Of intWorkers - The number of workers of a defined workerType that are allocated when a job runs.
- Role
Arn string - The ARN of the IAM role associated with this job.
- Security
Configuration string - The name of the Security Configuration to be associated with the job.
- map[string]string
- Key-value map of resource tags. If configured with a provider
default_tags
configuration block present, tags with matching keys will overwrite those defined at the provider-level. - map[string]string
- A map of tags assigned to the resource, including those inherited from the provider
default_tags
configuration block. - Timeout int
- The job timeout in minutes. The default is 2880 minutes (48 hours) for
glueetl
andpythonshell
jobs, and null (unlimited) forgluestreaming
jobs. - Worker
Type string - The type of predefined worker that is allocated when a job runs. Accepts a value of Standard, G.1X, G.2X, or G.025X for Spark jobs. Accepts the value Z.2X for Ray jobs.
- For the Standard worker type, each worker provides 4 vCPU, 16 GB of memory and a 50GB disk, and 2 executors per worker.
- For the G.1X worker type, each worker maps to 1 DPU (4 vCPU, 16 GB of memory, 64 GB disk), and provides 1 executor per worker. Recommended for memory-intensive jobs.
- For the G.2X worker type, each worker maps to 2 DPU (8 vCPU, 32 GB of memory, 128 GB disk), and provides 1 executor per worker. Recommended for memory-intensive jobs.
- For the G.4X worker type, each worker maps to 4 DPU (16 vCPUs, 64 GB of memory) with 256GB disk (approximately 235GB free), and provides 1 executor per worker. Recommended for memory-intensive jobs. Only available for Glue version 3.0. Available AWS Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm).
- For the G.8X worker type, each worker maps to 8 DPU (32 vCPUs, 128 GB of memory) with 512GB disk (approximately 487GB free), and provides 1 executor per worker. Recommended for memory-intensive jobs. Only available for Glue version 3.0. Available AWS Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm).
- For the G.025X worker type, each worker maps to 0.25 DPU (2 vCPU, 4GB of memory, 64 GB disk), and provides 1 executor per worker. Recommended for low volume streaming jobs. Only available for Glue version 3.0.
- For the Z.2X worker type, each worker maps to 2 M-DPU (8vCPU, 64 GB of m emory, 128 GB disk), and provides up to 8 Ray workers based on the autoscaler.
- arn String
- Amazon Resource Name (ARN) of Glue Job
- command
Job
Command - The command of the job. Defined below.
- connections List<String>
- The list of connections used for this job.
- default
Arguments Map<String,String> - The map of default arguments for this job. You can specify arguments here that your own job-execution script consumes, as well as arguments that AWS Glue itself consumes. For information about how to specify and consume your own Job arguments, see the Calling AWS Glue APIs in Python topic in the developer guide. For information about the key-value pairs that AWS Glue consumes to set up your job, see the Special Parameters Used by AWS Glue topic in the developer guide.
- description String
- Description of the job.
- execution
Class String - Indicates whether the job is run with a standard or flexible execution class. The standard execution class is ideal for time-sensitive workloads that require fast job startup and dedicated resources. Valid value:
FLEX
,STANDARD
. - execution
Property JobExecution Property - Execution property of the job. Defined below.
- glue
Version String - The version of glue to use, for example "1.0". Ray jobs should set this to 4.0 or greater. For information about available versions, see the AWS Glue Release Notes.
- job
Run BooleanQueuing Enabled - Specifies whether job run queuing is enabled for the job runs for this job. A value of true means job run queuing is enabled for the job runs. If false or not populated, the job runs will not be considered for queueing.
- maintenance
Window String - Specifies the day of the week and hour for the maintenance window for streaming jobs.
- max
Capacity Double - The maximum number of AWS Glue data processing units (DPUs) that can be allocated when this job runs.
Required
whenpythonshell
is set, accept either0.0625
or1.0
. Usenumber_of_workers
andworker_type
arguments instead withglue_version
2.0
and above. - max
Retries Integer - The maximum number of times to retry this job if it fails.
- name String
- The name you assign to this job. It must be unique in your account.
- non
Overridable Map<String,String>Arguments - Non-overridable arguments for this job, specified as name-value pairs.
- notification
Property JobNotification Property - Notification property of the job. Defined below.
- number
Of IntegerWorkers - The number of workers of a defined workerType that are allocated when a job runs.
- role
Arn String - The ARN of the IAM role associated with this job.
- security
Configuration String - The name of the Security Configuration to be associated with the job.
- Map<String,String>
- Key-value map of resource tags. If configured with a provider
default_tags
configuration block present, tags with matching keys will overwrite those defined at the provider-level. - Map<String,String>
- A map of tags assigned to the resource, including those inherited from the provider
default_tags
configuration block. - timeout Integer
- The job timeout in minutes. The default is 2880 minutes (48 hours) for
glueetl
andpythonshell
jobs, and null (unlimited) forgluestreaming
jobs. - worker
Type String - The type of predefined worker that is allocated when a job runs. Accepts a value of Standard, G.1X, G.2X, or G.025X for Spark jobs. Accepts the value Z.2X for Ray jobs.
- For the Standard worker type, each worker provides 4 vCPU, 16 GB of memory and a 50GB disk, and 2 executors per worker.
- For the G.1X worker type, each worker maps to 1 DPU (4 vCPU, 16 GB of memory, 64 GB disk), and provides 1 executor per worker. Recommended for memory-intensive jobs.
- For the G.2X worker type, each worker maps to 2 DPU (8 vCPU, 32 GB of memory, 128 GB disk), and provides 1 executor per worker. Recommended for memory-intensive jobs.
- For the G.4X worker type, each worker maps to 4 DPU (16 vCPUs, 64 GB of memory) with 256GB disk (approximately 235GB free), and provides 1 executor per worker. Recommended for memory-intensive jobs. Only available for Glue version 3.0. Available AWS Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm).
- For the G.8X worker type, each worker maps to 8 DPU (32 vCPUs, 128 GB of memory) with 512GB disk (approximately 487GB free), and provides 1 executor per worker. Recommended for memory-intensive jobs. Only available for Glue version 3.0. Available AWS Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm).
- For the G.025X worker type, each worker maps to 0.25 DPU (2 vCPU, 4GB of memory, 64 GB disk), and provides 1 executor per worker. Recommended for low volume streaming jobs. Only available for Glue version 3.0.
- For the Z.2X worker type, each worker maps to 2 M-DPU (8vCPU, 64 GB of m emory, 128 GB disk), and provides up to 8 Ray workers based on the autoscaler.
- arn string
- Amazon Resource Name (ARN) of Glue Job
- command
Job
Command - The command of the job. Defined below.
- connections string[]
- The list of connections used for this job.
- default
Arguments {[key: string]: string} - The map of default arguments for this job. You can specify arguments here that your own job-execution script consumes, as well as arguments that AWS Glue itself consumes. For information about how to specify and consume your own Job arguments, see the Calling AWS Glue APIs in Python topic in the developer guide. For information about the key-value pairs that AWS Glue consumes to set up your job, see the Special Parameters Used by AWS Glue topic in the developer guide.
- description string
- Description of the job.
- execution
Class string - Indicates whether the job is run with a standard or flexible execution class. The standard execution class is ideal for time-sensitive workloads that require fast job startup and dedicated resources. Valid value:
FLEX
,STANDARD
. - execution
Property JobExecution Property - Execution property of the job. Defined below.
- glue
Version string - The version of glue to use, for example "1.0". Ray jobs should set this to 4.0 or greater. For information about available versions, see the AWS Glue Release Notes.
- job
Run booleanQueuing Enabled - Specifies whether job run queuing is enabled for the job runs for this job. A value of true means job run queuing is enabled for the job runs. If false or not populated, the job runs will not be considered for queueing.
- maintenance
Window string - Specifies the day of the week and hour for the maintenance window for streaming jobs.
- max
Capacity number - The maximum number of AWS Glue data processing units (DPUs) that can be allocated when this job runs.
Required
whenpythonshell
is set, accept either0.0625
or1.0
. Usenumber_of_workers
andworker_type
arguments instead withglue_version
2.0
and above. - max
Retries number - The maximum number of times to retry this job if it fails.
- name string
- The name you assign to this job. It must be unique in your account.
- non
Overridable {[key: string]: string}Arguments - Non-overridable arguments for this job, specified as name-value pairs.
- notification
Property JobNotification Property - Notification property of the job. Defined below.
- number
Of numberWorkers - The number of workers of a defined workerType that are allocated when a job runs.
- role
Arn string - The ARN of the IAM role associated with this job.
- security
Configuration string - The name of the Security Configuration to be associated with the job.
- {[key: string]: string}
- Key-value map of resource tags. If configured with a provider
default_tags
configuration block present, tags with matching keys will overwrite those defined at the provider-level. - {[key: string]: string}
- A map of tags assigned to the resource, including those inherited from the provider
default_tags
configuration block. - timeout number
- The job timeout in minutes. The default is 2880 minutes (48 hours) for
glueetl
andpythonshell
jobs, and null (unlimited) forgluestreaming
jobs. - worker
Type string - The type of predefined worker that is allocated when a job runs. Accepts a value of Standard, G.1X, G.2X, or G.025X for Spark jobs. Accepts the value Z.2X for Ray jobs.
- For the Standard worker type, each worker provides 4 vCPU, 16 GB of memory and a 50GB disk, and 2 executors per worker.
- For the G.1X worker type, each worker maps to 1 DPU (4 vCPU, 16 GB of memory, 64 GB disk), and provides 1 executor per worker. Recommended for memory-intensive jobs.
- For the G.2X worker type, each worker maps to 2 DPU (8 vCPU, 32 GB of memory, 128 GB disk), and provides 1 executor per worker. Recommended for memory-intensive jobs.
- For the G.4X worker type, each worker maps to 4 DPU (16 vCPUs, 64 GB of memory) with 256GB disk (approximately 235GB free), and provides 1 executor per worker. Recommended for memory-intensive jobs. Only available for Glue version 3.0. Available AWS Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm).
- For the G.8X worker type, each worker maps to 8 DPU (32 vCPUs, 128 GB of memory) with 512GB disk (approximately 487GB free), and provides 1 executor per worker. Recommended for memory-intensive jobs. Only available for Glue version 3.0. Available AWS Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm).
- For the G.025X worker type, each worker maps to 0.25 DPU (2 vCPU, 4GB of memory, 64 GB disk), and provides 1 executor per worker. Recommended for low volume streaming jobs. Only available for Glue version 3.0.
- For the Z.2X worker type, each worker maps to 2 M-DPU (8vCPU, 64 GB of m emory, 128 GB disk), and provides up to 8 Ray workers based on the autoscaler.
- arn str
- Amazon Resource Name (ARN) of Glue Job
- command
Job
Command Args - The command of the job. Defined below.
- connections Sequence[str]
- The list of connections used for this job.
- default_
arguments Mapping[str, str] - The map of default arguments for this job. You can specify arguments here that your own job-execution script consumes, as well as arguments that AWS Glue itself consumes. For information about how to specify and consume your own Job arguments, see the Calling AWS Glue APIs in Python topic in the developer guide. For information about the key-value pairs that AWS Glue consumes to set up your job, see the Special Parameters Used by AWS Glue topic in the developer guide.
- description str
- Description of the job.
- execution_
class str - Indicates whether the job is run with a standard or flexible execution class. The standard execution class is ideal for time-sensitive workloads that require fast job startup and dedicated resources. Valid value:
FLEX
,STANDARD
. - execution_
property JobExecution Property Args - Execution property of the job. Defined below.
- glue_
version str - The version of glue to use, for example "1.0". Ray jobs should set this to 4.0 or greater. For information about available versions, see the AWS Glue Release Notes.
- job_
run_ boolqueuing_ enabled - Specifies whether job run queuing is enabled for the job runs for this job. A value of true means job run queuing is enabled for the job runs. If false or not populated, the job runs will not be considered for queueing.
- maintenance_
window str - Specifies the day of the week and hour for the maintenance window for streaming jobs.
- max_
capacity float - The maximum number of AWS Glue data processing units (DPUs) that can be allocated when this job runs.
Required
whenpythonshell
is set, accept either0.0625
or1.0
. Usenumber_of_workers
andworker_type
arguments instead withglue_version
2.0
and above. - max_
retries int - The maximum number of times to retry this job if it fails.
- name str
- The name you assign to this job. It must be unique in your account.
- non_
overridable_ Mapping[str, str]arguments - Non-overridable arguments for this job, specified as name-value pairs.
- notification_
property JobNotification Property Args - Notification property of the job. Defined below.
- number_
of_ intworkers - The number of workers of a defined workerType that are allocated when a job runs.
- role_
arn str - The ARN of the IAM role associated with this job.
- security_
configuration str - The name of the Security Configuration to be associated with the job.
- Mapping[str, str]
- Key-value map of resource tags. If configured with a provider
default_tags
configuration block present, tags with matching keys will overwrite those defined at the provider-level. - Mapping[str, str]
- A map of tags assigned to the resource, including those inherited from the provider
default_tags
configuration block. - timeout int
- The job timeout in minutes. The default is 2880 minutes (48 hours) for
glueetl
andpythonshell
jobs, and null (unlimited) forgluestreaming
jobs. - worker_
type str - The type of predefined worker that is allocated when a job runs. Accepts a value of Standard, G.1X, G.2X, or G.025X for Spark jobs. Accepts the value Z.2X for Ray jobs.
- For the Standard worker type, each worker provides 4 vCPU, 16 GB of memory and a 50GB disk, and 2 executors per worker.
- For the G.1X worker type, each worker maps to 1 DPU (4 vCPU, 16 GB of memory, 64 GB disk), and provides 1 executor per worker. Recommended for memory-intensive jobs.
- For the G.2X worker type, each worker maps to 2 DPU (8 vCPU, 32 GB of memory, 128 GB disk), and provides 1 executor per worker. Recommended for memory-intensive jobs.
- For the G.4X worker type, each worker maps to 4 DPU (16 vCPUs, 64 GB of memory) with 256GB disk (approximately 235GB free), and provides 1 executor per worker. Recommended for memory-intensive jobs. Only available for Glue version 3.0. Available AWS Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm).
- For the G.8X worker type, each worker maps to 8 DPU (32 vCPUs, 128 GB of memory) with 512GB disk (approximately 487GB free), and provides 1 executor per worker. Recommended for memory-intensive jobs. Only available for Glue version 3.0. Available AWS Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm).
- For the G.025X worker type, each worker maps to 0.25 DPU (2 vCPU, 4GB of memory, 64 GB disk), and provides 1 executor per worker. Recommended for low volume streaming jobs. Only available for Glue version 3.0.
- For the Z.2X worker type, each worker maps to 2 M-DPU (8vCPU, 64 GB of m emory, 128 GB disk), and provides up to 8 Ray workers based on the autoscaler.
- arn String
- Amazon Resource Name (ARN) of Glue Job
- command Property Map
- The command of the job. Defined below.
- connections List<String>
- The list of connections used for this job.
- default
Arguments Map<String> - The map of default arguments for this job. You can specify arguments here that your own job-execution script consumes, as well as arguments that AWS Glue itself consumes. For information about how to specify and consume your own Job arguments, see the Calling AWS Glue APIs in Python topic in the developer guide. For information about the key-value pairs that AWS Glue consumes to set up your job, see the Special Parameters Used by AWS Glue topic in the developer guide.
- description String
- Description of the job.
- execution
Class String - Indicates whether the job is run with a standard or flexible execution class. The standard execution class is ideal for time-sensitive workloads that require fast job startup and dedicated resources. Valid value:
FLEX
,STANDARD
. - execution
Property Property Map - Execution property of the job. Defined below.
- glue
Version String - The version of glue to use, for example "1.0". Ray jobs should set this to 4.0 or greater. For information about available versions, see the AWS Glue Release Notes.
- job
Run BooleanQueuing Enabled - Specifies whether job run queuing is enabled for the job runs for this job. A value of true means job run queuing is enabled for the job runs. If false or not populated, the job runs will not be considered for queueing.
- maintenance
Window String - Specifies the day of the week and hour for the maintenance window for streaming jobs.
- max
Capacity Number - The maximum number of AWS Glue data processing units (DPUs) that can be allocated when this job runs.
Required
whenpythonshell
is set, accept either0.0625
or1.0
. Usenumber_of_workers
andworker_type
arguments instead withglue_version
2.0
and above. - max
Retries Number - The maximum number of times to retry this job if it fails.
- name String
- The name you assign to this job. It must be unique in your account.
- non
Overridable Map<String>Arguments - Non-overridable arguments for this job, specified as name-value pairs.
- notification
Property Property Map - Notification property of the job. Defined below.
- number
Of NumberWorkers - The number of workers of a defined workerType that are allocated when a job runs.
- role
Arn String - The ARN of the IAM role associated with this job.
- security
Configuration String - The name of the Security Configuration to be associated with the job.
- Map<String>
- Key-value map of resource tags. If configured with a provider
default_tags
configuration block present, tags with matching keys will overwrite those defined at the provider-level. - Map<String>
- A map of tags assigned to the resource, including those inherited from the provider
default_tags
configuration block. - timeout Number
- The job timeout in minutes. The default is 2880 minutes (48 hours) for
glueetl
andpythonshell
jobs, and null (unlimited) forgluestreaming
jobs. - worker
Type String - The type of predefined worker that is allocated when a job runs. Accepts a value of Standard, G.1X, G.2X, or G.025X for Spark jobs. Accepts the value Z.2X for Ray jobs.
- For the Standard worker type, each worker provides 4 vCPU, 16 GB of memory and a 50GB disk, and 2 executors per worker.
- For the G.1X worker type, each worker maps to 1 DPU (4 vCPU, 16 GB of memory, 64 GB disk), and provides 1 executor per worker. Recommended for memory-intensive jobs.
- For the G.2X worker type, each worker maps to 2 DPU (8 vCPU, 32 GB of memory, 128 GB disk), and provides 1 executor per worker. Recommended for memory-intensive jobs.
- For the G.4X worker type, each worker maps to 4 DPU (16 vCPUs, 64 GB of memory) with 256GB disk (approximately 235GB free), and provides 1 executor per worker. Recommended for memory-intensive jobs. Only available for Glue version 3.0. Available AWS Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm).
- For the G.8X worker type, each worker maps to 8 DPU (32 vCPUs, 128 GB of memory) with 512GB disk (approximately 487GB free), and provides 1 executor per worker. Recommended for memory-intensive jobs. Only available for Glue version 3.0. Available AWS Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm).
- For the G.025X worker type, each worker maps to 0.25 DPU (2 vCPU, 4GB of memory, 64 GB disk), and provides 1 executor per worker. Recommended for low volume streaming jobs. Only available for Glue version 3.0.
- For the Z.2X worker type, each worker maps to 2 M-DPU (8vCPU, 64 GB of m emory, 128 GB disk), and provides up to 8 Ray workers based on the autoscaler.
Supporting Types
JobCommand, JobCommandArgs
- Script
Location string - Specifies the S3 path to a script that executes a job.
- Name string
- The name of the job command. Defaults to
glueetl
. Usepythonshell
for Python Shell Job Type,glueray
for Ray Job Type, orgluestreaming
for Streaming Job Type.max_capacity
needs to be set ifpythonshell
is chosen. - Python
Version string - The Python version being used to execute a Python shell job. Allowed values are 2, 3 or 3.9. Version 3 refers to Python 3.6.
- Runtime string
- In Ray jobs, runtime is used to specify the versions of Ray, Python and additional libraries available in your environment. This field is not used in other job types. For supported runtime environment values, see Working with Ray jobs in the Glue Developer Guide.
- Script
Location string - Specifies the S3 path to a script that executes a job.
- Name string
- The name of the job command. Defaults to
glueetl
. Usepythonshell
for Python Shell Job Type,glueray
for Ray Job Type, orgluestreaming
for Streaming Job Type.max_capacity
needs to be set ifpythonshell
is chosen. - Python
Version string - The Python version being used to execute a Python shell job. Allowed values are 2, 3 or 3.9. Version 3 refers to Python 3.6.
- Runtime string
- In Ray jobs, runtime is used to specify the versions of Ray, Python and additional libraries available in your environment. This field is not used in other job types. For supported runtime environment values, see Working with Ray jobs in the Glue Developer Guide.
- script
Location String - Specifies the S3 path to a script that executes a job.
- name String
- The name of the job command. Defaults to
glueetl
. Usepythonshell
for Python Shell Job Type,glueray
for Ray Job Type, orgluestreaming
for Streaming Job Type.max_capacity
needs to be set ifpythonshell
is chosen. - python
Version String - The Python version being used to execute a Python shell job. Allowed values are 2, 3 or 3.9. Version 3 refers to Python 3.6.
- runtime String
- In Ray jobs, runtime is used to specify the versions of Ray, Python and additional libraries available in your environment. This field is not used in other job types. For supported runtime environment values, see Working with Ray jobs in the Glue Developer Guide.
- script
Location string - Specifies the S3 path to a script that executes a job.
- name string
- The name of the job command. Defaults to
glueetl
. Usepythonshell
for Python Shell Job Type,glueray
for Ray Job Type, orgluestreaming
for Streaming Job Type.max_capacity
needs to be set ifpythonshell
is chosen. - python
Version string - The Python version being used to execute a Python shell job. Allowed values are 2, 3 or 3.9. Version 3 refers to Python 3.6.
- runtime string
- In Ray jobs, runtime is used to specify the versions of Ray, Python and additional libraries available in your environment. This field is not used in other job types. For supported runtime environment values, see Working with Ray jobs in the Glue Developer Guide.
- script_
location str - Specifies the S3 path to a script that executes a job.
- name str
- The name of the job command. Defaults to
glueetl
. Usepythonshell
for Python Shell Job Type,glueray
for Ray Job Type, orgluestreaming
for Streaming Job Type.max_capacity
needs to be set ifpythonshell
is chosen. - python_
version str - The Python version being used to execute a Python shell job. Allowed values are 2, 3 or 3.9. Version 3 refers to Python 3.6.
- runtime str
- In Ray jobs, runtime is used to specify the versions of Ray, Python and additional libraries available in your environment. This field is not used in other job types. For supported runtime environment values, see Working with Ray jobs in the Glue Developer Guide.
- script
Location String - Specifies the S3 path to a script that executes a job.
- name String
- The name of the job command. Defaults to
glueetl
. Usepythonshell
for Python Shell Job Type,glueray
for Ray Job Type, orgluestreaming
for Streaming Job Type.max_capacity
needs to be set ifpythonshell
is chosen. - python
Version String - The Python version being used to execute a Python shell job. Allowed values are 2, 3 or 3.9. Version 3 refers to Python 3.6.
- runtime String
- In Ray jobs, runtime is used to specify the versions of Ray, Python and additional libraries available in your environment. This field is not used in other job types. For supported runtime environment values, see Working with Ray jobs in the Glue Developer Guide.
JobExecutionProperty, JobExecutionPropertyArgs
- Max
Concurrent intRuns - The maximum number of concurrent runs allowed for a job. The default is 1.
- Max
Concurrent intRuns - The maximum number of concurrent runs allowed for a job. The default is 1.
- max
Concurrent IntegerRuns - The maximum number of concurrent runs allowed for a job. The default is 1.
- max
Concurrent numberRuns - The maximum number of concurrent runs allowed for a job. The default is 1.
- max_
concurrent_ intruns - The maximum number of concurrent runs allowed for a job. The default is 1.
- max
Concurrent NumberRuns - The maximum number of concurrent runs allowed for a job. The default is 1.
JobNotificationProperty, JobNotificationPropertyArgs
- Notify
Delay intAfter - After a job run starts, the number of minutes to wait before sending a job run delay notification.
- Notify
Delay intAfter - After a job run starts, the number of minutes to wait before sending a job run delay notification.
- notify
Delay IntegerAfter - After a job run starts, the number of minutes to wait before sending a job run delay notification.
- notify
Delay numberAfter - After a job run starts, the number of minutes to wait before sending a job run delay notification.
- notify_
delay_ intafter - After a job run starts, the number of minutes to wait before sending a job run delay notification.
- notify
Delay NumberAfter - After a job run starts, the number of minutes to wait before sending a job run delay notification.
Import
Using pulumi import
, import Glue Jobs using name
. For example:
$ pulumi import aws:glue/job:Job MyJob MyJob
To learn more about importing existing cloud resources, see Importing resources.
Package Details
- Repository
- AWS Classic pulumi/pulumi-aws
- License
- Apache-2.0
- Notes
- This Pulumi package is based on the
aws
Terraform Provider.