published on Wednesday, Nov 18, 2026 by Pulumi
published on Wednesday, Nov 18, 2026 by Pulumi
An OnlineEvaluator contains the configuration for an Online Evaluation. It periodically samples traces or sessions emitted by an agent and evaluates them using the configured metric sources.
To get more information about OnlineEvaluator, see:
- API documentation
- How-to Guides
Example Usage
Vertex Ai Online Evaluator Basic
import * as pulumi from "@pulumi/pulumi";
import * as gcp from "@pulumi/gcp";
const engine = new gcp.vertex.AiReasoningEngine("engine", {
displayName: "reasoning-engine",
description: "Reasoning engine evaluated by the online evaluator",
region: "us-central1",
});
const evaluator = new gcp.vertex.AiOnlineEvaluator("evaluator", {
region: "us-central1",
displayName: "online-evaluator",
agentResource: engine.id,
config: {
maxEvaluatedSamplesPerRun: "100",
randomSampling: {
percentage: 10,
},
},
cloudObservability: {
openTelemetry: {
semconvVersion: "1.39.0",
},
traceScope: {
filters: [{
duration: {
comparisonOperator: "GREATER",
value: 0,
},
}],
},
},
metricSources: [
{
metric: JSON.stringify({
predefinedMetricSpec: {
metricSpecName: "safety_v1",
},
}),
},
{
metric: JSON.stringify({
predefinedMetricSpec: {
metricSpecName: "final_response_quality_v1",
},
}),
},
],
});
import pulumi
import json
import pulumi_gcp as gcp
engine = gcp.vertex.AiReasoningEngine("engine",
display_name="reasoning-engine",
description="Reasoning engine evaluated by the online evaluator",
region="us-central1")
evaluator = gcp.vertex.AiOnlineEvaluator("evaluator",
region="us-central1",
display_name="online-evaluator",
agent_resource=engine.id,
config={
"max_evaluated_samples_per_run": "100",
"random_sampling": {
"percentage": 10,
},
},
cloud_observability={
"open_telemetry": {
"semconv_version": "1.39.0",
},
"trace_scope": {
"filters": [{
"duration": {
"comparison_operator": "GREATER",
"value": float(0),
},
}],
},
},
metric_sources=[
{
"metric": json.dumps({
"predefinedMetricSpec": {
"metricSpecName": "safety_v1",
},
}),
},
{
"metric": json.dumps({
"predefinedMetricSpec": {
"metricSpecName": "final_response_quality_v1",
},
}),
},
])
package main
import (
"encoding/json"
"github.com/pulumi/pulumi-gcp/sdk/v10/go/gcp/vertex"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
engine, err := vertex.NewAiReasoningEngine(ctx, "engine", &vertex.AiReasoningEngineArgs{
DisplayName: pulumi.String("reasoning-engine"),
Description: pulumi.String("Reasoning engine evaluated by the online evaluator"),
Region: pulumi.String("us-central1"),
})
if err != nil {
return err
}
tmpJSON0, err := json.Marshal(map[string]map[string]string{
"predefinedMetricSpec": map[string]string{
"metricSpecName": "safety_v1",
},
})
if err != nil {
return err
}
json0 := string(tmpJSON0)
tmpJSON1, err := json.Marshal(map[string]map[string]string{
"predefinedMetricSpec": map[string]string{
"metricSpecName": "final_response_quality_v1",
},
})
if err != nil {
return err
}
json1 := string(tmpJSON1)
_, err = vertex.NewAiOnlineEvaluator(ctx, "evaluator", &vertex.AiOnlineEvaluatorArgs{
Region: pulumi.String("us-central1"),
DisplayName: pulumi.String("online-evaluator"),
AgentResource: engine.ID(),
Config: &vertex.AiOnlineEvaluatorConfigArgs{
MaxEvaluatedSamplesPerRun: pulumi.String("100"),
RandomSampling: &vertex.AiOnlineEvaluatorConfigRandomSamplingArgs{
Percentage: pulumi.Int(10),
},
},
CloudObservability: &vertex.AiOnlineEvaluatorCloudObservabilityArgs{
OpenTelemetry: &vertex.AiOnlineEvaluatorCloudObservabilityOpenTelemetryArgs{
SemconvVersion: pulumi.String("1.39.0"),
},
TraceScope: &vertex.AiOnlineEvaluatorCloudObservabilityTraceScopeArgs{
Filters: vertex.AiOnlineEvaluatorCloudObservabilityTraceScopeFilterArray{
&vertex.AiOnlineEvaluatorCloudObservabilityTraceScopeFilterArgs{
Duration: &vertex.AiOnlineEvaluatorCloudObservabilityTraceScopeFilterDurationArgs{
ComparisonOperator: pulumi.String("GREATER"),
Value: pulumi.Float64(0),
},
},
},
},
},
MetricSources: vertex.AiOnlineEvaluatorMetricSourceArray{
&vertex.AiOnlineEvaluatorMetricSourceArgs{
Metric: pulumi.String(json0),
},
&vertex.AiOnlineEvaluatorMetricSourceArgs{
Metric: pulumi.String(json1),
},
},
})
if err != nil {
return err
}
return nil
})
}
using System.Collections.Generic;
using System.Linq;
using System.Text.Json;
using Pulumi;
using Gcp = Pulumi.Gcp;
return await Deployment.RunAsync(() =>
{
var engine = new Gcp.Vertex.AiReasoningEngine("engine", new()
{
DisplayName = "reasoning-engine",
Description = "Reasoning engine evaluated by the online evaluator",
Region = "us-central1",
});
var evaluator = new Gcp.Vertex.AiOnlineEvaluator("evaluator", new()
{
Region = "us-central1",
DisplayName = "online-evaluator",
AgentResource = engine.Id,
Config = new Gcp.Vertex.Inputs.AiOnlineEvaluatorConfigArgs
{
MaxEvaluatedSamplesPerRun = "100",
RandomSampling = new Gcp.Vertex.Inputs.AiOnlineEvaluatorConfigRandomSamplingArgs
{
Percentage = 10,
},
},
CloudObservability = new Gcp.Vertex.Inputs.AiOnlineEvaluatorCloudObservabilityArgs
{
OpenTelemetry = new Gcp.Vertex.Inputs.AiOnlineEvaluatorCloudObservabilityOpenTelemetryArgs
{
SemconvVersion = "1.39.0",
},
TraceScope = new Gcp.Vertex.Inputs.AiOnlineEvaluatorCloudObservabilityTraceScopeArgs
{
Filters = new[]
{
new Gcp.Vertex.Inputs.AiOnlineEvaluatorCloudObservabilityTraceScopeFilterArgs
{
Duration = new Gcp.Vertex.Inputs.AiOnlineEvaluatorCloudObservabilityTraceScopeFilterDurationArgs
{
ComparisonOperator = "GREATER",
Value = 0.0,
},
},
},
},
},
MetricSources = new[]
{
new Gcp.Vertex.Inputs.AiOnlineEvaluatorMetricSourceArgs
{
Metric = JsonSerializer.Serialize(new Dictionary<string, object?>
{
["predefinedMetricSpec"] = new Dictionary<string, object?>
{
["metricSpecName"] = "safety_v1",
},
}),
},
new Gcp.Vertex.Inputs.AiOnlineEvaluatorMetricSourceArgs
{
Metric = JsonSerializer.Serialize(new Dictionary<string, object?>
{
["predefinedMetricSpec"] = new Dictionary<string, object?>
{
["metricSpecName"] = "final_response_quality_v1",
},
}),
},
},
});
});
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.gcp.vertex.AiReasoningEngine;
import com.pulumi.gcp.vertex.AiReasoningEngineArgs;
import com.pulumi.gcp.vertex.AiOnlineEvaluator;
import com.pulumi.gcp.vertex.AiOnlineEvaluatorArgs;
import com.pulumi.gcp.vertex.inputs.AiOnlineEvaluatorConfigArgs;
import com.pulumi.gcp.vertex.inputs.AiOnlineEvaluatorConfigRandomSamplingArgs;
import com.pulumi.gcp.vertex.inputs.AiOnlineEvaluatorCloudObservabilityArgs;
import com.pulumi.gcp.vertex.inputs.AiOnlineEvaluatorCloudObservabilityOpenTelemetryArgs;
import com.pulumi.gcp.vertex.inputs.AiOnlineEvaluatorCloudObservabilityTraceScopeArgs;
import com.pulumi.gcp.vertex.inputs.AiOnlineEvaluatorCloudObservabilityTraceScopeFilterArgs;
import com.pulumi.gcp.vertex.inputs.AiOnlineEvaluatorCloudObservabilityTraceScopeFilterDurationArgs;
import com.pulumi.gcp.vertex.inputs.AiOnlineEvaluatorMetricSourceArgs;
import static com.pulumi.codegen.internal.Serialization.*;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var engine = new AiReasoningEngine("engine", AiReasoningEngineArgs.builder()
.displayName("reasoning-engine")
.description("Reasoning engine evaluated by the online evaluator")
.region("us-central1")
.build());
var evaluator = new AiOnlineEvaluator("evaluator", AiOnlineEvaluatorArgs.builder()
.region("us-central1")
.displayName("online-evaluator")
.agentResource(engine.id())
.config(AiOnlineEvaluatorConfigArgs.builder()
.maxEvaluatedSamplesPerRun("100")
.randomSampling(AiOnlineEvaluatorConfigRandomSamplingArgs.builder()
.percentage(10)
.build())
.build())
.cloudObservability(AiOnlineEvaluatorCloudObservabilityArgs.builder()
.openTelemetry(AiOnlineEvaluatorCloudObservabilityOpenTelemetryArgs.builder()
.semconvVersion("1.39.0")
.build())
.traceScope(AiOnlineEvaluatorCloudObservabilityTraceScopeArgs.builder()
.filters(AiOnlineEvaluatorCloudObservabilityTraceScopeFilterArgs.builder()
.duration(AiOnlineEvaluatorCloudObservabilityTraceScopeFilterDurationArgs.builder()
.comparisonOperator("GREATER")
.value(0.0)
.build())
.build())
.build())
.build())
.metricSources(
AiOnlineEvaluatorMetricSourceArgs.builder()
.metric(serializeJson(
jsonObject(
jsonProperty("predefinedMetricSpec", jsonObject(
jsonProperty("metricSpecName", "safety_v1")
))
)))
.build(),
AiOnlineEvaluatorMetricSourceArgs.builder()
.metric(serializeJson(
jsonObject(
jsonProperty("predefinedMetricSpec", jsonObject(
jsonProperty("metricSpecName", "final_response_quality_v1")
))
)))
.build())
.build());
}
}
resources:
engine:
type: gcp:vertex:AiReasoningEngine
properties:
displayName: reasoning-engine
description: Reasoning engine evaluated by the online evaluator
region: us-central1
evaluator:
type: gcp:vertex:AiOnlineEvaluator
properties:
region: us-central1
displayName: online-evaluator
agentResource: ${engine.id}
config:
maxEvaluatedSamplesPerRun: '100'
randomSampling:
percentage: 10
cloudObservability:
openTelemetry:
semconvVersion: 1.39.0
traceScope:
filters:
- duration:
comparisonOperator: GREATER
value: 0
metricSources:
- metric:
fn::toJSON:
predefinedMetricSpec:
metricSpecName: safety_v1
- metric:
fn::toJSON:
predefinedMetricSpec:
metricSpecName: final_response_quality_v1
terraform {
required_providers {
gcp = {
source = "pulumi/gcp"
}
}
}
resource "gcp_vertex_ai_reasoning_engine" "engine" {
lifecycle {
create_before_destroy = true
}
display_name = "reasoning-engine"
description = "Reasoning engine evaluated by the online evaluator"
region = "us-central1"
}
resource "gcp_vertex_ai_online_evaluator" "evaluator" {
lifecycle {
create_before_destroy = true
}
region = "us-central1"
display_name = "online-evaluator"
agent_resource = gcp_vertex_ai_reasoning_engine.engine.id
config = {
max_evaluated_samples_per_run = "100"
random_sampling = {
percentage = 10
}
}
cloud_observability = {
open_telemetry = {
semconv_version = "1.39.0"
}
trace_scope = {
filters = [{
duration = {
comparison_operator = "GREATER"
value = 0
}
}]
}
}
metric_sources {
metric = jsonencode({
"predefinedMetricSpec" = {
"metricSpecName" = "safety_v1"
}
})
}
metric_sources {
metric = jsonencode({
"predefinedMetricSpec" = {
"metricSpecName" = "final_response_quality_v1"
}
})
}
}
Vertex Ai Online Evaluator Full
import * as pulumi from "@pulumi/pulumi";
import * as gcp from "@pulumi/gcp";
const engine = new gcp.vertex.AiReasoningEngine("engine", {
displayName: "reasoning-engine",
description: "Reasoning engine evaluated by the online evaluator",
region: "us-central1",
});
const metric = new gcp.vertex.AiEvaluationMetric("metric", {
evaluationMetricId: "eval-metric",
region: "us-central1",
displayName: "Custom response quality",
metric: JSON.stringify({
llmBasedMetricSpec: {
metricPromptTemplate: "Rate the quality of the following response on a scale of 1 to 5. Response: {response}",
},
metadata: {
title: "custom-response-quality",
scoreRange: {
min: 1,
max: 5,
},
},
}),
});
const evaluator = new gcp.vertex.AiOnlineEvaluator("evaluator", {
region: "us-central1",
displayName: "online-evaluator",
agentResource: engine.id,
config: {
randomSampling: {
percentage: 100,
},
},
cloudObservability: {
openTelemetry: {
semconvVersion: "1.39.0",
},
sessionScope: {
inactivityTrigger: {
threshold: "3600s",
},
filters: [
{
totalTokenUsage: {
comparisonOperator: "GREATER",
value: 0,
},
},
{
userTurns: {
comparisonOperator: "GREATER",
value: 0,
},
},
{
modelCalls: {
comparisonOperator: "GREATER",
value: 0,
},
},
{
modelCallErrors: {
comparisonOperator: "GREATER_OR_EQUAL",
value: 0,
},
},
{
toolCalls: {
comparisonOperator: "GREATER",
value: 0,
},
},
{
toolCallErrors: {
comparisonOperator: "GREATER_OR_EQUAL",
value: 0,
},
},
{
duration: {
comparisonOperator: "GREATER",
value: 0,
},
},
],
},
},
metricSources: [{
metricResourceName: metric.id,
}],
});
import pulumi
import json
import pulumi_gcp as gcp
engine = gcp.vertex.AiReasoningEngine("engine",
display_name="reasoning-engine",
description="Reasoning engine evaluated by the online evaluator",
region="us-central1")
metric = gcp.vertex.AiEvaluationMetric("metric",
evaluation_metric_id="eval-metric",
region="us-central1",
display_name="Custom response quality",
metric=json.dumps({
"llmBasedMetricSpec": {
"metricPromptTemplate": "Rate the quality of the following response on a scale of 1 to 5. Response: {response}",
},
"metadata": {
"title": "custom-response-quality",
"scoreRange": {
"min": 1,
"max": 5,
},
},
}))
evaluator = gcp.vertex.AiOnlineEvaluator("evaluator",
region="us-central1",
display_name="online-evaluator",
agent_resource=engine.id,
config={
"random_sampling": {
"percentage": 100,
},
},
cloud_observability={
"open_telemetry": {
"semconv_version": "1.39.0",
},
"session_scope": {
"inactivity_trigger": {
"threshold": "3600s",
},
"filters": [
{
"total_token_usage": {
"comparison_operator": "GREATER",
"value": float(0),
},
},
{
"user_turns": {
"comparison_operator": "GREATER",
"value": float(0),
},
},
{
"model_calls": {
"comparison_operator": "GREATER",
"value": float(0),
},
},
{
"model_call_errors": {
"comparison_operator": "GREATER_OR_EQUAL",
"value": float(0),
},
},
{
"tool_calls": {
"comparison_operator": "GREATER",
"value": float(0),
},
},
{
"tool_call_errors": {
"comparison_operator": "GREATER_OR_EQUAL",
"value": float(0),
},
},
{
"duration": {
"comparison_operator": "GREATER",
"value": float(0),
},
},
],
},
},
metric_sources=[{
"metric_resource_name": metric.id,
}])
package main
import (
"encoding/json"
"github.com/pulumi/pulumi-gcp/sdk/v10/go/gcp/vertex"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
engine, err := vertex.NewAiReasoningEngine(ctx, "engine", &vertex.AiReasoningEngineArgs{
DisplayName: pulumi.String("reasoning-engine"),
Description: pulumi.String("Reasoning engine evaluated by the online evaluator"),
Region: pulumi.String("us-central1"),
})
if err != nil {
return err
}
tmpJSON0, err := json.Marshal(map[string]interface{}{
"llmBasedMetricSpec": map[string]string{
"metricPromptTemplate": "Rate the quality of the following response on a scale of 1 to 5. Response: {response}",
},
"metadata": map[string]interface{}{
"title": "custom-response-quality",
"scoreRange": map[string]int{
"min": 1,
"max": 5,
},
},
})
if err != nil {
return err
}
json0 := string(tmpJSON0)
metric, err := vertex.NewAiEvaluationMetric(ctx, "metric", &vertex.AiEvaluationMetricArgs{
EvaluationMetricId: pulumi.String("eval-metric"),
Region: pulumi.String("us-central1"),
DisplayName: pulumi.String("Custom response quality"),
Metric: pulumi.String(json0),
})
if err != nil {
return err
}
_, err = vertex.NewAiOnlineEvaluator(ctx, "evaluator", &vertex.AiOnlineEvaluatorArgs{
Region: pulumi.String("us-central1"),
DisplayName: pulumi.String("online-evaluator"),
AgentResource: engine.ID(),
Config: &vertex.AiOnlineEvaluatorConfigArgs{
RandomSampling: &vertex.AiOnlineEvaluatorConfigRandomSamplingArgs{
Percentage: pulumi.Int(100),
},
},
CloudObservability: &vertex.AiOnlineEvaluatorCloudObservabilityArgs{
OpenTelemetry: &vertex.AiOnlineEvaluatorCloudObservabilityOpenTelemetryArgs{
SemconvVersion: pulumi.String("1.39.0"),
},
SessionScope: &vertex.AiOnlineEvaluatorCloudObservabilitySessionScopeArgs{
InactivityTrigger: &vertex.AiOnlineEvaluatorCloudObservabilitySessionScopeInactivityTriggerArgs{
Threshold: pulumi.String("3600s"),
},
Filters: vertex.AiOnlineEvaluatorCloudObservabilitySessionScopeFilterArray{
&vertex.AiOnlineEvaluatorCloudObservabilitySessionScopeFilterArgs{
TotalTokenUsage: &vertex.AiOnlineEvaluatorCloudObservabilitySessionScopeFilterTotalTokenUsageArgs{
ComparisonOperator: pulumi.String("GREATER"),
Value: pulumi.Float64(0),
},
},
&vertex.AiOnlineEvaluatorCloudObservabilitySessionScopeFilterArgs{
UserTurns: &vertex.AiOnlineEvaluatorCloudObservabilitySessionScopeFilterUserTurnsArgs{
ComparisonOperator: pulumi.String("GREATER"),
Value: pulumi.Float64(0),
},
},
&vertex.AiOnlineEvaluatorCloudObservabilitySessionScopeFilterArgs{
ModelCalls: &vertex.AiOnlineEvaluatorCloudObservabilitySessionScopeFilterModelCallsArgs{
ComparisonOperator: pulumi.String("GREATER"),
Value: pulumi.Float64(0),
},
},
&vertex.AiOnlineEvaluatorCloudObservabilitySessionScopeFilterArgs{
ModelCallErrors: &vertex.AiOnlineEvaluatorCloudObservabilitySessionScopeFilterModelCallErrorsArgs{
ComparisonOperator: pulumi.String("GREATER_OR_EQUAL"),
Value: pulumi.Float64(0),
},
},
&vertex.AiOnlineEvaluatorCloudObservabilitySessionScopeFilterArgs{
ToolCalls: &vertex.AiOnlineEvaluatorCloudObservabilitySessionScopeFilterToolCallsArgs{
ComparisonOperator: pulumi.String("GREATER"),
Value: pulumi.Float64(0),
},
},
&vertex.AiOnlineEvaluatorCloudObservabilitySessionScopeFilterArgs{
ToolCallErrors: &vertex.AiOnlineEvaluatorCloudObservabilitySessionScopeFilterToolCallErrorsArgs{
ComparisonOperator: pulumi.String("GREATER_OR_EQUAL"),
Value: pulumi.Float64(0),
},
},
&vertex.AiOnlineEvaluatorCloudObservabilitySessionScopeFilterArgs{
Duration: &vertex.AiOnlineEvaluatorCloudObservabilitySessionScopeFilterDurationArgs{
ComparisonOperator: pulumi.String("GREATER"),
Value: pulumi.Float64(0),
},
},
},
},
},
MetricSources: vertex.AiOnlineEvaluatorMetricSourceArray{
&vertex.AiOnlineEvaluatorMetricSourceArgs{
MetricResourceName: metric.ID(),
},
},
})
if err != nil {
return err
}
return nil
})
}
using System.Collections.Generic;
using System.Linq;
using System.Text.Json;
using Pulumi;
using Gcp = Pulumi.Gcp;
return await Deployment.RunAsync(() =>
{
var engine = new Gcp.Vertex.AiReasoningEngine("engine", new()
{
DisplayName = "reasoning-engine",
Description = "Reasoning engine evaluated by the online evaluator",
Region = "us-central1",
});
var metric = new Gcp.Vertex.AiEvaluationMetric("metric", new()
{
EvaluationMetricId = "eval-metric",
Region = "us-central1",
DisplayName = "Custom response quality",
Metric = JsonSerializer.Serialize(new Dictionary<string, object?>
{
["llmBasedMetricSpec"] = new Dictionary<string, object?>
{
["metricPromptTemplate"] = "Rate the quality of the following response on a scale of 1 to 5. Response: {response}",
},
["metadata"] = new Dictionary<string, object?>
{
["title"] = "custom-response-quality",
["scoreRange"] = new Dictionary<string, object?>
{
["min"] = 1,
["max"] = 5,
},
},
}),
});
var evaluator = new Gcp.Vertex.AiOnlineEvaluator("evaluator", new()
{
Region = "us-central1",
DisplayName = "online-evaluator",
AgentResource = engine.Id,
Config = new Gcp.Vertex.Inputs.AiOnlineEvaluatorConfigArgs
{
RandomSampling = new Gcp.Vertex.Inputs.AiOnlineEvaluatorConfigRandomSamplingArgs
{
Percentage = 100,
},
},
CloudObservability = new Gcp.Vertex.Inputs.AiOnlineEvaluatorCloudObservabilityArgs
{
OpenTelemetry = new Gcp.Vertex.Inputs.AiOnlineEvaluatorCloudObservabilityOpenTelemetryArgs
{
SemconvVersion = "1.39.0",
},
SessionScope = new Gcp.Vertex.Inputs.AiOnlineEvaluatorCloudObservabilitySessionScopeArgs
{
InactivityTrigger = new Gcp.Vertex.Inputs.AiOnlineEvaluatorCloudObservabilitySessionScopeInactivityTriggerArgs
{
Threshold = "3600s",
},
Filters = new[]
{
new Gcp.Vertex.Inputs.AiOnlineEvaluatorCloudObservabilitySessionScopeFilterArgs
{
TotalTokenUsage = new Gcp.Vertex.Inputs.AiOnlineEvaluatorCloudObservabilitySessionScopeFilterTotalTokenUsageArgs
{
ComparisonOperator = "GREATER",
Value = 0.0,
},
},
new Gcp.Vertex.Inputs.AiOnlineEvaluatorCloudObservabilitySessionScopeFilterArgs
{
UserTurns = new Gcp.Vertex.Inputs.AiOnlineEvaluatorCloudObservabilitySessionScopeFilterUserTurnsArgs
{
ComparisonOperator = "GREATER",
Value = 0.0,
},
},
new Gcp.Vertex.Inputs.AiOnlineEvaluatorCloudObservabilitySessionScopeFilterArgs
{
ModelCalls = new Gcp.Vertex.Inputs.AiOnlineEvaluatorCloudObservabilitySessionScopeFilterModelCallsArgs
{
ComparisonOperator = "GREATER",
Value = 0.0,
},
},
new Gcp.Vertex.Inputs.AiOnlineEvaluatorCloudObservabilitySessionScopeFilterArgs
{
ModelCallErrors = new Gcp.Vertex.Inputs.AiOnlineEvaluatorCloudObservabilitySessionScopeFilterModelCallErrorsArgs
{
ComparisonOperator = "GREATER_OR_EQUAL",
Value = 0.0,
},
},
new Gcp.Vertex.Inputs.AiOnlineEvaluatorCloudObservabilitySessionScopeFilterArgs
{
ToolCalls = new Gcp.Vertex.Inputs.AiOnlineEvaluatorCloudObservabilitySessionScopeFilterToolCallsArgs
{
ComparisonOperator = "GREATER",
Value = 0.0,
},
},
new Gcp.Vertex.Inputs.AiOnlineEvaluatorCloudObservabilitySessionScopeFilterArgs
{
ToolCallErrors = new Gcp.Vertex.Inputs.AiOnlineEvaluatorCloudObservabilitySessionScopeFilterToolCallErrorsArgs
{
ComparisonOperator = "GREATER_OR_EQUAL",
Value = 0.0,
},
},
new Gcp.Vertex.Inputs.AiOnlineEvaluatorCloudObservabilitySessionScopeFilterArgs
{
Duration = new Gcp.Vertex.Inputs.AiOnlineEvaluatorCloudObservabilitySessionScopeFilterDurationArgs
{
ComparisonOperator = "GREATER",
Value = 0.0,
},
},
},
},
},
MetricSources = new[]
{
new Gcp.Vertex.Inputs.AiOnlineEvaluatorMetricSourceArgs
{
MetricResourceName = metric.Id,
},
},
});
});
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.gcp.vertex.AiReasoningEngine;
import com.pulumi.gcp.vertex.AiReasoningEngineArgs;
import com.pulumi.gcp.vertex.AiEvaluationMetric;
import com.pulumi.gcp.vertex.AiEvaluationMetricArgs;
import com.pulumi.gcp.vertex.AiOnlineEvaluator;
import com.pulumi.gcp.vertex.AiOnlineEvaluatorArgs;
import com.pulumi.gcp.vertex.inputs.AiOnlineEvaluatorConfigArgs;
import com.pulumi.gcp.vertex.inputs.AiOnlineEvaluatorConfigRandomSamplingArgs;
import com.pulumi.gcp.vertex.inputs.AiOnlineEvaluatorCloudObservabilityArgs;
import com.pulumi.gcp.vertex.inputs.AiOnlineEvaluatorCloudObservabilityOpenTelemetryArgs;
import com.pulumi.gcp.vertex.inputs.AiOnlineEvaluatorCloudObservabilitySessionScopeArgs;
import com.pulumi.gcp.vertex.inputs.AiOnlineEvaluatorCloudObservabilitySessionScopeInactivityTriggerArgs;
import com.pulumi.gcp.vertex.inputs.AiOnlineEvaluatorCloudObservabilitySessionScopeFilterArgs;
import com.pulumi.gcp.vertex.inputs.AiOnlineEvaluatorCloudObservabilitySessionScopeFilterTotalTokenUsageArgs;
import com.pulumi.gcp.vertex.inputs.AiOnlineEvaluatorCloudObservabilitySessionScopeFilterUserTurnsArgs;
import com.pulumi.gcp.vertex.inputs.AiOnlineEvaluatorCloudObservabilitySessionScopeFilterModelCallsArgs;
import com.pulumi.gcp.vertex.inputs.AiOnlineEvaluatorCloudObservabilitySessionScopeFilterModelCallErrorsArgs;
import com.pulumi.gcp.vertex.inputs.AiOnlineEvaluatorCloudObservabilitySessionScopeFilterToolCallsArgs;
import com.pulumi.gcp.vertex.inputs.AiOnlineEvaluatorCloudObservabilitySessionScopeFilterToolCallErrorsArgs;
import com.pulumi.gcp.vertex.inputs.AiOnlineEvaluatorCloudObservabilitySessionScopeFilterDurationArgs;
import com.pulumi.gcp.vertex.inputs.AiOnlineEvaluatorMetricSourceArgs;
import static com.pulumi.codegen.internal.Serialization.*;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var engine = new AiReasoningEngine("engine", AiReasoningEngineArgs.builder()
.displayName("reasoning-engine")
.description("Reasoning engine evaluated by the online evaluator")
.region("us-central1")
.build());
var metric = new AiEvaluationMetric("metric", AiEvaluationMetricArgs.builder()
.evaluationMetricId("eval-metric")
.region("us-central1")
.displayName("Custom response quality")
.metric(serializeJson(
jsonObject(
jsonProperty("llmBasedMetricSpec", jsonObject(
jsonProperty("metricPromptTemplate", "Rate the quality of the following response on a scale of 1 to 5. Response: {response}")
)),
jsonProperty("metadata", jsonObject(
jsonProperty("title", "custom-response-quality"),
jsonProperty("scoreRange", jsonObject(
jsonProperty("min", 1),
jsonProperty("max", 5)
))
))
)))
.build());
var evaluator = new AiOnlineEvaluator("evaluator", AiOnlineEvaluatorArgs.builder()
.region("us-central1")
.displayName("online-evaluator")
.agentResource(engine.id())
.config(AiOnlineEvaluatorConfigArgs.builder()
.randomSampling(AiOnlineEvaluatorConfigRandomSamplingArgs.builder()
.percentage(100)
.build())
.build())
.cloudObservability(AiOnlineEvaluatorCloudObservabilityArgs.builder()
.openTelemetry(AiOnlineEvaluatorCloudObservabilityOpenTelemetryArgs.builder()
.semconvVersion("1.39.0")
.build())
.sessionScope(AiOnlineEvaluatorCloudObservabilitySessionScopeArgs.builder()
.inactivityTrigger(AiOnlineEvaluatorCloudObservabilitySessionScopeInactivityTriggerArgs.builder()
.threshold("3600s")
.build())
.filters(
AiOnlineEvaluatorCloudObservabilitySessionScopeFilterArgs.builder()
.totalTokenUsage(AiOnlineEvaluatorCloudObservabilitySessionScopeFilterTotalTokenUsageArgs.builder()
.comparisonOperator("GREATER")
.value(0.0)
.build())
.build(),
AiOnlineEvaluatorCloudObservabilitySessionScopeFilterArgs.builder()
.userTurns(AiOnlineEvaluatorCloudObservabilitySessionScopeFilterUserTurnsArgs.builder()
.comparisonOperator("GREATER")
.value(0.0)
.build())
.build(),
AiOnlineEvaluatorCloudObservabilitySessionScopeFilterArgs.builder()
.modelCalls(AiOnlineEvaluatorCloudObservabilitySessionScopeFilterModelCallsArgs.builder()
.comparisonOperator("GREATER")
.value(0.0)
.build())
.build(),
AiOnlineEvaluatorCloudObservabilitySessionScopeFilterArgs.builder()
.modelCallErrors(AiOnlineEvaluatorCloudObservabilitySessionScopeFilterModelCallErrorsArgs.builder()
.comparisonOperator("GREATER_OR_EQUAL")
.value(0.0)
.build())
.build(),
AiOnlineEvaluatorCloudObservabilitySessionScopeFilterArgs.builder()
.toolCalls(AiOnlineEvaluatorCloudObservabilitySessionScopeFilterToolCallsArgs.builder()
.comparisonOperator("GREATER")
.value(0.0)
.build())
.build(),
AiOnlineEvaluatorCloudObservabilitySessionScopeFilterArgs.builder()
.toolCallErrors(AiOnlineEvaluatorCloudObservabilitySessionScopeFilterToolCallErrorsArgs.builder()
.comparisonOperator("GREATER_OR_EQUAL")
.value(0.0)
.build())
.build(),
AiOnlineEvaluatorCloudObservabilitySessionScopeFilterArgs.builder()
.duration(AiOnlineEvaluatorCloudObservabilitySessionScopeFilterDurationArgs.builder()
.comparisonOperator("GREATER")
.value(0.0)
.build())
.build())
.build())
.build())
.metricSources(AiOnlineEvaluatorMetricSourceArgs.builder()
.metricResourceName(metric.id())
.build())
.build());
}
}
resources:
engine:
type: gcp:vertex:AiReasoningEngine
properties:
displayName: reasoning-engine
description: Reasoning engine evaluated by the online evaluator
region: us-central1
metric:
type: gcp:vertex:AiEvaluationMetric
properties:
evaluationMetricId: eval-metric
region: us-central1
displayName: Custom response quality
metric:
fn::toJSON:
llmBasedMetricSpec:
metricPromptTemplate: 'Rate the quality of the following response on a scale of 1 to 5. Response: {response}'
metadata:
title: custom-response-quality
scoreRange:
min: 1
max: 5
evaluator:
type: gcp:vertex:AiOnlineEvaluator
properties:
region: us-central1
displayName: online-evaluator
agentResource: ${engine.id}
config:
randomSampling:
percentage: 100
cloudObservability:
openTelemetry:
semconvVersion: 1.39.0
sessionScope:
inactivityTrigger:
threshold: 3600s
filters:
- totalTokenUsage:
comparisonOperator: GREATER
value: 0
- userTurns:
comparisonOperator: GREATER
value: 0
- modelCalls:
comparisonOperator: GREATER
value: 0
- modelCallErrors:
comparisonOperator: GREATER_OR_EQUAL
value: 0
- toolCalls:
comparisonOperator: GREATER
value: 0
- toolCallErrors:
comparisonOperator: GREATER_OR_EQUAL
value: 0
- duration:
comparisonOperator: GREATER
value: 0
metricSources:
- metricResourceName: ${metric.id}
terraform {
required_providers {
gcp = {
source = "pulumi/gcp"
}
}
}
resource "gcp_vertex_ai_reasoning_engine" "engine" {
lifecycle {
create_before_destroy = true
}
display_name = "reasoning-engine"
description = "Reasoning engine evaluated by the online evaluator"
region = "us-central1"
}
resource "gcp_vertex_ai_evaluation_metric" "metric" {
lifecycle {
create_before_destroy = true
}
evaluation_metric_id = "eval-metric"
region = "us-central1"
display_name = "Custom response quality"
metric = jsonencode({
"llmBasedMetricSpec" = {
"metricPromptTemplate" = "Rate the quality of the following response on a scale of 1 to 5. Response: {response}"
}
"metadata" = {
"title" = "custom-response-quality"
"scoreRange" = {
"min" = 1
"max" = 5
}
}
})
}
resource "gcp_vertex_ai_online_evaluator" "evaluator" {
lifecycle {
create_before_destroy = true
}
region = "us-central1"
display_name = "online-evaluator"
agent_resource = gcp_vertex_ai_reasoning_engine.engine.id
config = {
random_sampling = {
percentage = 100
}
}
cloud_observability = {
open_telemetry = {
semconv_version = "1.39.0"
}
session_scope = {
inactivity_trigger = {
threshold = "3600s"
}
filters = [{
total_token_usage = {
comparison_operator = "GREATER"
value = 0
}
}, {
user_turns = {
comparison_operator = "GREATER"
value = 0
}
}, {
model_calls = {
comparison_operator = "GREATER"
value = 0
}
}, {
model_call_errors = {
comparison_operator = "GREATER_OR_EQUAL"
value = 0
}
}, {
tool_calls = {
comparison_operator = "GREATER"
value = 0
}
}, {
tool_call_errors = {
comparison_operator = "GREATER_OR_EQUAL"
value = 0
}
}, {
duration = {
comparison_operator = "GREATER"
value = 0
}
}]
}
}
metric_sources {
metric_resource_name = gcp_vertex_ai_evaluation_metric.metric.id
}
}
Create AiOnlineEvaluator Resource
Resources are created with functions called constructors. To learn more about declaring and configuring resources, see Resources.
Constructor syntax
new AiOnlineEvaluator(name: string, args: AiOnlineEvaluatorArgs, opts?: CustomResourceOptions);@overload
def AiOnlineEvaluator(resource_name: str,
args: AiOnlineEvaluatorArgs,
opts: Optional[ResourceOptions] = None)
@overload
def AiOnlineEvaluator(resource_name: str,
opts: Optional[ResourceOptions] = None,
agent_resource: Optional[str] = None,
cloud_observability: Optional[AiOnlineEvaluatorCloudObservabilityArgs] = None,
config: Optional[AiOnlineEvaluatorConfigArgs] = None,
metric_sources: Optional[Sequence[AiOnlineEvaluatorMetricSourceArgs]] = None,
deletion_policy: Optional[str] = None,
display_name: Optional[str] = None,
project: Optional[str] = None,
region: Optional[str] = None)func NewAiOnlineEvaluator(ctx *Context, name string, args AiOnlineEvaluatorArgs, opts ...ResourceOption) (*AiOnlineEvaluator, error)public AiOnlineEvaluator(string name, AiOnlineEvaluatorArgs args, CustomResourceOptions? opts = null)
public AiOnlineEvaluator(String name, AiOnlineEvaluatorArgs args)
public AiOnlineEvaluator(String name, AiOnlineEvaluatorArgs args, CustomResourceOptions options)
type: gcp:vertex:AiOnlineEvaluator
properties: # The arguments to resource properties.
options: # Bag of options to control resource's behavior.
resource "gcp_vertex_ai_online_evaluator" "name" {
# resource properties
}Parameters
- name string
- The unique name of the resource.
- args AiOnlineEvaluatorArgs
- The arguments to resource properties.
- opts CustomResourceOptions
- Bag of options to control resource's behavior.
- resource_name str
- The unique name of the resource.
- args AiOnlineEvaluatorArgs
- The arguments to resource properties.
- opts ResourceOptions
- Bag of options to control resource's behavior.
- ctx Context
- Context object for the current deployment.
- name string
- The unique name of the resource.
- args AiOnlineEvaluatorArgs
- The arguments to resource properties.
- opts ResourceOption
- Bag of options to control resource's behavior.
- name string
- The unique name of the resource.
- args AiOnlineEvaluatorArgs
- The arguments to resource properties.
- opts CustomResourceOptions
- Bag of options to control resource's behavior.
- name String
- The unique name of the resource.
- args AiOnlineEvaluatorArgs
- The arguments to resource properties.
- options CustomResourceOptions
- Bag of options to control resource's behavior.
Constructor example
The following reference example uses placeholder values for all input properties.
var aiOnlineEvaluatorResource = new Gcp.Vertex.AiOnlineEvaluator("aiOnlineEvaluatorResource", new()
{
AgentResource = "string",
CloudObservability = new Gcp.Vertex.Inputs.AiOnlineEvaluatorCloudObservabilityArgs
{
OpenTelemetry = new Gcp.Vertex.Inputs.AiOnlineEvaluatorCloudObservabilityOpenTelemetryArgs
{
SemconvVersion = "string",
},
LogView = "string",
SessionScope = new Gcp.Vertex.Inputs.AiOnlineEvaluatorCloudObservabilitySessionScopeArgs
{
Filters = new[]
{
new Gcp.Vertex.Inputs.AiOnlineEvaluatorCloudObservabilitySessionScopeFilterArgs
{
Duration = new Gcp.Vertex.Inputs.AiOnlineEvaluatorCloudObservabilitySessionScopeFilterDurationArgs
{
ComparisonOperator = "string",
Value = 0.0,
},
ModelCallErrors = new Gcp.Vertex.Inputs.AiOnlineEvaluatorCloudObservabilitySessionScopeFilterModelCallErrorsArgs
{
ComparisonOperator = "string",
Value = 0.0,
},
ModelCalls = new Gcp.Vertex.Inputs.AiOnlineEvaluatorCloudObservabilitySessionScopeFilterModelCallsArgs
{
ComparisonOperator = "string",
Value = 0.0,
},
ToolCallErrors = new Gcp.Vertex.Inputs.AiOnlineEvaluatorCloudObservabilitySessionScopeFilterToolCallErrorsArgs
{
ComparisonOperator = "string",
Value = 0.0,
},
ToolCalls = new Gcp.Vertex.Inputs.AiOnlineEvaluatorCloudObservabilitySessionScopeFilterToolCallsArgs
{
ComparisonOperator = "string",
Value = 0.0,
},
TotalTokenUsage = new Gcp.Vertex.Inputs.AiOnlineEvaluatorCloudObservabilitySessionScopeFilterTotalTokenUsageArgs
{
ComparisonOperator = "string",
Value = 0.0,
},
UserTurns = new Gcp.Vertex.Inputs.AiOnlineEvaluatorCloudObservabilitySessionScopeFilterUserTurnsArgs
{
ComparisonOperator = "string",
Value = 0.0,
},
},
},
InactivityTrigger = new Gcp.Vertex.Inputs.AiOnlineEvaluatorCloudObservabilitySessionScopeInactivityTriggerArgs
{
Threshold = "string",
},
},
TraceScope = new Gcp.Vertex.Inputs.AiOnlineEvaluatorCloudObservabilityTraceScopeArgs
{
Filters = new[]
{
new Gcp.Vertex.Inputs.AiOnlineEvaluatorCloudObservabilityTraceScopeFilterArgs
{
Duration = new Gcp.Vertex.Inputs.AiOnlineEvaluatorCloudObservabilityTraceScopeFilterDurationArgs
{
ComparisonOperator = "string",
Value = 0.0,
},
TotalTokenUsage = new Gcp.Vertex.Inputs.AiOnlineEvaluatorCloudObservabilityTraceScopeFilterTotalTokenUsageArgs
{
ComparisonOperator = "string",
Value = 0.0,
},
},
},
},
TraceView = "string",
},
Config = new Gcp.Vertex.Inputs.AiOnlineEvaluatorConfigArgs
{
MaxEvaluatedSamplesPerRun = "string",
RandomSampling = new Gcp.Vertex.Inputs.AiOnlineEvaluatorConfigRandomSamplingArgs
{
Percentage = 0,
},
},
MetricSources = new[]
{
new Gcp.Vertex.Inputs.AiOnlineEvaluatorMetricSourceArgs
{
Metric = "string",
MetricResourceName = "string",
},
},
DeletionPolicy = "string",
DisplayName = "string",
Project = "string",
Region = "string",
});
example, err := vertex.NewAiOnlineEvaluator(ctx, "aiOnlineEvaluatorResource", &vertex.AiOnlineEvaluatorArgs{
AgentResource: pulumi.String("string"),
CloudObservability: &vertex.AiOnlineEvaluatorCloudObservabilityArgs{
OpenTelemetry: &vertex.AiOnlineEvaluatorCloudObservabilityOpenTelemetryArgs{
SemconvVersion: pulumi.String("string"),
},
LogView: pulumi.String("string"),
SessionScope: &vertex.AiOnlineEvaluatorCloudObservabilitySessionScopeArgs{
Filters: vertex.AiOnlineEvaluatorCloudObservabilitySessionScopeFilterArray{
&vertex.AiOnlineEvaluatorCloudObservabilitySessionScopeFilterArgs{
Duration: &vertex.AiOnlineEvaluatorCloudObservabilitySessionScopeFilterDurationArgs{
ComparisonOperator: pulumi.String("string"),
Value: pulumi.Float64(0),
},
ModelCallErrors: &vertex.AiOnlineEvaluatorCloudObservabilitySessionScopeFilterModelCallErrorsArgs{
ComparisonOperator: pulumi.String("string"),
Value: pulumi.Float64(0),
},
ModelCalls: &vertex.AiOnlineEvaluatorCloudObservabilitySessionScopeFilterModelCallsArgs{
ComparisonOperator: pulumi.String("string"),
Value: pulumi.Float64(0),
},
ToolCallErrors: &vertex.AiOnlineEvaluatorCloudObservabilitySessionScopeFilterToolCallErrorsArgs{
ComparisonOperator: pulumi.String("string"),
Value: pulumi.Float64(0),
},
ToolCalls: &vertex.AiOnlineEvaluatorCloudObservabilitySessionScopeFilterToolCallsArgs{
ComparisonOperator: pulumi.String("string"),
Value: pulumi.Float64(0),
},
TotalTokenUsage: &vertex.AiOnlineEvaluatorCloudObservabilitySessionScopeFilterTotalTokenUsageArgs{
ComparisonOperator: pulumi.String("string"),
Value: pulumi.Float64(0),
},
UserTurns: &vertex.AiOnlineEvaluatorCloudObservabilitySessionScopeFilterUserTurnsArgs{
ComparisonOperator: pulumi.String("string"),
Value: pulumi.Float64(0),
},
},
},
InactivityTrigger: &vertex.AiOnlineEvaluatorCloudObservabilitySessionScopeInactivityTriggerArgs{
Threshold: pulumi.String("string"),
},
},
TraceScope: &vertex.AiOnlineEvaluatorCloudObservabilityTraceScopeArgs{
Filters: vertex.AiOnlineEvaluatorCloudObservabilityTraceScopeFilterArray{
&vertex.AiOnlineEvaluatorCloudObservabilityTraceScopeFilterArgs{
Duration: &vertex.AiOnlineEvaluatorCloudObservabilityTraceScopeFilterDurationArgs{
ComparisonOperator: pulumi.String("string"),
Value: pulumi.Float64(0),
},
TotalTokenUsage: &vertex.AiOnlineEvaluatorCloudObservabilityTraceScopeFilterTotalTokenUsageArgs{
ComparisonOperator: pulumi.String("string"),
Value: pulumi.Float64(0),
},
},
},
},
TraceView: pulumi.String("string"),
},
Config: &vertex.AiOnlineEvaluatorConfigArgs{
MaxEvaluatedSamplesPerRun: pulumi.String("string"),
RandomSampling: &vertex.AiOnlineEvaluatorConfigRandomSamplingArgs{
Percentage: pulumi.Int(0),
},
},
MetricSources: vertex.AiOnlineEvaluatorMetricSourceArray{
&vertex.AiOnlineEvaluatorMetricSourceArgs{
Metric: pulumi.String("string"),
MetricResourceName: pulumi.String("string"),
},
},
DeletionPolicy: pulumi.String("string"),
DisplayName: pulumi.String("string"),
Project: pulumi.String("string"),
Region: pulumi.String("string"),
})
resource "gcp_vertex_ai_online_evaluator" "aiOnlineEvaluatorResource" {
lifecycle {
create_before_destroy = true
}
agent_resource = "string"
cloud_observability = {
open_telemetry = {
semconv_version = "string"
}
log_view = "string"
session_scope = {
filters = [{
duration = {
comparison_operator = "string"
value = 0
}
model_call_errors = {
comparison_operator = "string"
value = 0
}
model_calls = {
comparison_operator = "string"
value = 0
}
tool_call_errors = {
comparison_operator = "string"
value = 0
}
tool_calls = {
comparison_operator = "string"
value = 0
}
total_token_usage = {
comparison_operator = "string"
value = 0
}
user_turns = {
comparison_operator = "string"
value = 0
}
}]
inactivity_trigger = {
threshold = "string"
}
}
trace_scope = {
filters = [{
duration = {
comparison_operator = "string"
value = 0
}
total_token_usage = {
comparison_operator = "string"
value = 0
}
}]
}
trace_view = "string"
}
config = {
max_evaluated_samples_per_run = "string"
random_sampling = {
percentage = 0
}
}
metric_sources {
metric = "string"
metric_resource_name = "string"
}
deletion_policy = "string"
display_name = "string"
project = "string"
region = "string"
}
var aiOnlineEvaluatorResource = new AiOnlineEvaluator("aiOnlineEvaluatorResource", AiOnlineEvaluatorArgs.builder()
.agentResource("string")
.cloudObservability(AiOnlineEvaluatorCloudObservabilityArgs.builder()
.openTelemetry(AiOnlineEvaluatorCloudObservabilityOpenTelemetryArgs.builder()
.semconvVersion("string")
.build())
.logView("string")
.sessionScope(AiOnlineEvaluatorCloudObservabilitySessionScopeArgs.builder()
.filters(AiOnlineEvaluatorCloudObservabilitySessionScopeFilterArgs.builder()
.duration(AiOnlineEvaluatorCloudObservabilitySessionScopeFilterDurationArgs.builder()
.comparisonOperator("string")
.value(0.0)
.build())
.modelCallErrors(AiOnlineEvaluatorCloudObservabilitySessionScopeFilterModelCallErrorsArgs.builder()
.comparisonOperator("string")
.value(0.0)
.build())
.modelCalls(AiOnlineEvaluatorCloudObservabilitySessionScopeFilterModelCallsArgs.builder()
.comparisonOperator("string")
.value(0.0)
.build())
.toolCallErrors(AiOnlineEvaluatorCloudObservabilitySessionScopeFilterToolCallErrorsArgs.builder()
.comparisonOperator("string")
.value(0.0)
.build())
.toolCalls(AiOnlineEvaluatorCloudObservabilitySessionScopeFilterToolCallsArgs.builder()
.comparisonOperator("string")
.value(0.0)
.build())
.totalTokenUsage(AiOnlineEvaluatorCloudObservabilitySessionScopeFilterTotalTokenUsageArgs.builder()
.comparisonOperator("string")
.value(0.0)
.build())
.userTurns(AiOnlineEvaluatorCloudObservabilitySessionScopeFilterUserTurnsArgs.builder()
.comparisonOperator("string")
.value(0.0)
.build())
.build())
.inactivityTrigger(AiOnlineEvaluatorCloudObservabilitySessionScopeInactivityTriggerArgs.builder()
.threshold("string")
.build())
.build())
.traceScope(AiOnlineEvaluatorCloudObservabilityTraceScopeArgs.builder()
.filters(AiOnlineEvaluatorCloudObservabilityTraceScopeFilterArgs.builder()
.duration(AiOnlineEvaluatorCloudObservabilityTraceScopeFilterDurationArgs.builder()
.comparisonOperator("string")
.value(0.0)
.build())
.totalTokenUsage(AiOnlineEvaluatorCloudObservabilityTraceScopeFilterTotalTokenUsageArgs.builder()
.comparisonOperator("string")
.value(0.0)
.build())
.build())
.build())
.traceView("string")
.build())
.config(AiOnlineEvaluatorConfigArgs.builder()
.maxEvaluatedSamplesPerRun("string")
.randomSampling(AiOnlineEvaluatorConfigRandomSamplingArgs.builder()
.percentage(0)
.build())
.build())
.metricSources(AiOnlineEvaluatorMetricSourceArgs.builder()
.metric("string")
.metricResourceName("string")
.build())
.deletionPolicy("string")
.displayName("string")
.project("string")
.region("string")
.build());
ai_online_evaluator_resource = gcp.vertex.AiOnlineEvaluator("aiOnlineEvaluatorResource",
agent_resource="string",
cloud_observability={
"open_telemetry": {
"semconv_version": "string",
},
"log_view": "string",
"session_scope": {
"filters": [{
"duration": {
"comparison_operator": "string",
"value": float(0),
},
"model_call_errors": {
"comparison_operator": "string",
"value": float(0),
},
"model_calls": {
"comparison_operator": "string",
"value": float(0),
},
"tool_call_errors": {
"comparison_operator": "string",
"value": float(0),
},
"tool_calls": {
"comparison_operator": "string",
"value": float(0),
},
"total_token_usage": {
"comparison_operator": "string",
"value": float(0),
},
"user_turns": {
"comparison_operator": "string",
"value": float(0),
},
}],
"inactivity_trigger": {
"threshold": "string",
},
},
"trace_scope": {
"filters": [{
"duration": {
"comparison_operator": "string",
"value": float(0),
},
"total_token_usage": {
"comparison_operator": "string",
"value": float(0),
},
}],
},
"trace_view": "string",
},
config={
"max_evaluated_samples_per_run": "string",
"random_sampling": {
"percentage": 0,
},
},
metric_sources=[{
"metric": "string",
"metric_resource_name": "string",
}],
deletion_policy="string",
display_name="string",
project="string",
region="string")
const aiOnlineEvaluatorResource = new gcp.vertex.AiOnlineEvaluator("aiOnlineEvaluatorResource", {
agentResource: "string",
cloudObservability: {
openTelemetry: {
semconvVersion: "string",
},
logView: "string",
sessionScope: {
filters: [{
duration: {
comparisonOperator: "string",
value: 0,
},
modelCallErrors: {
comparisonOperator: "string",
value: 0,
},
modelCalls: {
comparisonOperator: "string",
value: 0,
},
toolCallErrors: {
comparisonOperator: "string",
value: 0,
},
toolCalls: {
comparisonOperator: "string",
value: 0,
},
totalTokenUsage: {
comparisonOperator: "string",
value: 0,
},
userTurns: {
comparisonOperator: "string",
value: 0,
},
}],
inactivityTrigger: {
threshold: "string",
},
},
traceScope: {
filters: [{
duration: {
comparisonOperator: "string",
value: 0,
},
totalTokenUsage: {
comparisonOperator: "string",
value: 0,
},
}],
},
traceView: "string",
},
config: {
maxEvaluatedSamplesPerRun: "string",
randomSampling: {
percentage: 0,
},
},
metricSources: [{
metric: "string",
metricResourceName: "string",
}],
deletionPolicy: "string",
displayName: "string",
project: "string",
region: "string",
});
type: gcp:vertex:AiOnlineEvaluator
properties:
agentResource: string
cloudObservability:
logView: string
openTelemetry:
semconvVersion: string
sessionScope:
filters:
- duration:
comparisonOperator: string
value: 0
modelCallErrors:
comparisonOperator: string
value: 0
modelCalls:
comparisonOperator: string
value: 0
toolCallErrors:
comparisonOperator: string
value: 0
toolCalls:
comparisonOperator: string
value: 0
totalTokenUsage:
comparisonOperator: string
value: 0
userTurns:
comparisonOperator: string
value: 0
inactivityTrigger:
threshold: string
traceScope:
filters:
- duration:
comparisonOperator: string
value: 0
totalTokenUsage:
comparisonOperator: string
value: 0
traceView: string
config:
maxEvaluatedSamplesPerRun: string
randomSampling:
percentage: 0
deletionPolicy: string
displayName: string
metricSources:
- metric: string
metricResourceName: string
project: string
region: string
AiOnlineEvaluator Resource Properties
To learn more about resource properties and how to use them, see Inputs and Outputs in the Architecture and Concepts docs.
Inputs
In Python, inputs that are objects can be passed either as argument classes or as dictionary literals.
The AiOnlineEvaluator resource accepts the following input properties:
- Agent
Resource string - The name of the agent that the OnlineEvaluator evaluates periodically. This value is used to filter the traces with a matching cloud.resource_id and link the evaluation results with relevant dashboards/UIs. This field is immutable; once set, it cannot be changed.
- Cloud
Observability AiOnline Evaluator Cloud Observability - Data source for the OnlineEvaluator, based on the Google Cloud
Observability stack (Cloud Trace & Cloud Logging). The data source type is
immutable once set. Within
cloudObservability,logViewandtraceViewcan be updated; the eval scope (traceScope/sessionScope) and the convention (openTelemetry) are immutable. Structure is documented below. - Config
Ai
Online Evaluator Config - Configuration for sampling behavior of the OnlineEvaluator. The OnlineEvaluator runs at a fixed interval of 10 minutes. Structure is documented below.
- Metric
Sources List<AiOnline Evaluator Metric Source> - A list of metric sources to be used for evaluating samples. At least one
metric source must be provided. Only predefined metrics (inline
metric) and registered metrics (metricResourceName) are supported. Every registered metric must have adisplayName(ortitle) and ascoreRangedefined, otherwise the evaluations will fail. The maximum number of metric sources is 25. Exactly one ofmetricormetricResourceNamemust be set per metric source. Structure is documented below. - Deletion
Policy string - Whether Terraform will be prevented from destroying the resource. Defaults to DELETE. When a 'terraform destroy' or 'pulumi up' would delete the resource, the command will fail if this field is set to "PREVENT" in Terraform state. When set to "ABANDON", the command will remove the resource from Terraform management without updating or deleting the resource in the API. When set to "DELETE", deleting the resource is allowed.
- Display
Name string - Human-readable name for the OnlineEvaluator. The name doesn't have to be unique. The name can consist of any UTF-8 characters. The maximum length is 63 characters.
- Project string
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
- Region string
- The region of the OnlineEvaluator (e.g. us-central1). If unset, the provider's region is used.
- Agent
Resource string - The name of the agent that the OnlineEvaluator evaluates periodically. This value is used to filter the traces with a matching cloud.resource_id and link the evaluation results with relevant dashboards/UIs. This field is immutable; once set, it cannot be changed.
- Cloud
Observability AiOnline Evaluator Cloud Observability Args - Data source for the OnlineEvaluator, based on the Google Cloud
Observability stack (Cloud Trace & Cloud Logging). The data source type is
immutable once set. Within
cloudObservability,logViewandtraceViewcan be updated; the eval scope (traceScope/sessionScope) and the convention (openTelemetry) are immutable. Structure is documented below. - Config
Ai
Online Evaluator Config Args - Configuration for sampling behavior of the OnlineEvaluator. The OnlineEvaluator runs at a fixed interval of 10 minutes. Structure is documented below.
- Metric
Sources []AiOnline Evaluator Metric Source Args - A list of metric sources to be used for evaluating samples. At least one
metric source must be provided. Only predefined metrics (inline
metric) and registered metrics (metricResourceName) are supported. Every registered metric must have adisplayName(ortitle) and ascoreRangedefined, otherwise the evaluations will fail. The maximum number of metric sources is 25. Exactly one ofmetricormetricResourceNamemust be set per metric source. Structure is documented below. - Deletion
Policy string - Whether Terraform will be prevented from destroying the resource. Defaults to DELETE. When a 'terraform destroy' or 'pulumi up' would delete the resource, the command will fail if this field is set to "PREVENT" in Terraform state. When set to "ABANDON", the command will remove the resource from Terraform management without updating or deleting the resource in the API. When set to "DELETE", deleting the resource is allowed.
- Display
Name string - Human-readable name for the OnlineEvaluator. The name doesn't have to be unique. The name can consist of any UTF-8 characters. The maximum length is 63 characters.
- Project string
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
- Region string
- The region of the OnlineEvaluator (e.g. us-central1). If unset, the provider's region is used.
- agent_
resource string - The name of the agent that the OnlineEvaluator evaluates periodically. This value is used to filter the traces with a matching cloud.resource_id and link the evaluation results with relevant dashboards/UIs. This field is immutable; once set, it cannot be changed.
- cloud_
observability object - Data source for the OnlineEvaluator, based on the Google Cloud
Observability stack (Cloud Trace & Cloud Logging). The data source type is
immutable once set. Within
cloudObservability,logViewandtraceViewcan be updated; the eval scope (traceScope/sessionScope) and the convention (openTelemetry) are immutable. Structure is documented below. - config object
- Configuration for sampling behavior of the OnlineEvaluator. The OnlineEvaluator runs at a fixed interval of 10 minutes. Structure is documented below.
- metric_
sources list(object) - A list of metric sources to be used for evaluating samples. At least one
metric source must be provided. Only predefined metrics (inline
metric) and registered metrics (metricResourceName) are supported. Every registered metric must have adisplayName(ortitle) and ascoreRangedefined, otherwise the evaluations will fail. The maximum number of metric sources is 25. Exactly one ofmetricormetricResourceNamemust be set per metric source. Structure is documented below. - deletion_
policy string - Whether Terraform will be prevented from destroying the resource. Defaults to DELETE. When a 'terraform destroy' or 'pulumi up' would delete the resource, the command will fail if this field is set to "PREVENT" in Terraform state. When set to "ABANDON", the command will remove the resource from Terraform management without updating or deleting the resource in the API. When set to "DELETE", deleting the resource is allowed.
- display_
name string - Human-readable name for the OnlineEvaluator. The name doesn't have to be unique. The name can consist of any UTF-8 characters. The maximum length is 63 characters.
- project string
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
- region string
- The region of the OnlineEvaluator (e.g. us-central1). If unset, the provider's region is used.
- agent
Resource String - The name of the agent that the OnlineEvaluator evaluates periodically. This value is used to filter the traces with a matching cloud.resource_id and link the evaluation results with relevant dashboards/UIs. This field is immutable; once set, it cannot be changed.
- cloud
Observability AiOnline Evaluator Cloud Observability - Data source for the OnlineEvaluator, based on the Google Cloud
Observability stack (Cloud Trace & Cloud Logging). The data source type is
immutable once set. Within
cloudObservability,logViewandtraceViewcan be updated; the eval scope (traceScope/sessionScope) and the convention (openTelemetry) are immutable. Structure is documented below. - config
Ai
Online Evaluator Config - Configuration for sampling behavior of the OnlineEvaluator. The OnlineEvaluator runs at a fixed interval of 10 minutes. Structure is documented below.
- metric
Sources List<AiOnline Evaluator Metric Source> - A list of metric sources to be used for evaluating samples. At least one
metric source must be provided. Only predefined metrics (inline
metric) and registered metrics (metricResourceName) are supported. Every registered metric must have adisplayName(ortitle) and ascoreRangedefined, otherwise the evaluations will fail. The maximum number of metric sources is 25. Exactly one ofmetricormetricResourceNamemust be set per metric source. Structure is documented below. - deletion
Policy String - Whether Terraform will be prevented from destroying the resource. Defaults to DELETE. When a 'terraform destroy' or 'pulumi up' would delete the resource, the command will fail if this field is set to "PREVENT" in Terraform state. When set to "ABANDON", the command will remove the resource from Terraform management without updating or deleting the resource in the API. When set to "DELETE", deleting the resource is allowed.
- display
Name String - Human-readable name for the OnlineEvaluator. The name doesn't have to be unique. The name can consist of any UTF-8 characters. The maximum length is 63 characters.
- project String
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
- region String
- The region of the OnlineEvaluator (e.g. us-central1). If unset, the provider's region is used.
- agent
Resource string - The name of the agent that the OnlineEvaluator evaluates periodically. This value is used to filter the traces with a matching cloud.resource_id and link the evaluation results with relevant dashboards/UIs. This field is immutable; once set, it cannot be changed.
- cloud
Observability AiOnline Evaluator Cloud Observability - Data source for the OnlineEvaluator, based on the Google Cloud
Observability stack (Cloud Trace & Cloud Logging). The data source type is
immutable once set. Within
cloudObservability,logViewandtraceViewcan be updated; the eval scope (traceScope/sessionScope) and the convention (openTelemetry) are immutable. Structure is documented below. - config
Ai
Online Evaluator Config - Configuration for sampling behavior of the OnlineEvaluator. The OnlineEvaluator runs at a fixed interval of 10 minutes. Structure is documented below.
- metric
Sources AiOnline Evaluator Metric Source[] - A list of metric sources to be used for evaluating samples. At least one
metric source must be provided. Only predefined metrics (inline
metric) and registered metrics (metricResourceName) are supported. Every registered metric must have adisplayName(ortitle) and ascoreRangedefined, otherwise the evaluations will fail. The maximum number of metric sources is 25. Exactly one ofmetricormetricResourceNamemust be set per metric source. Structure is documented below. - deletion
Policy string - Whether Terraform will be prevented from destroying the resource. Defaults to DELETE. When a 'terraform destroy' or 'pulumi up' would delete the resource, the command will fail if this field is set to "PREVENT" in Terraform state. When set to "ABANDON", the command will remove the resource from Terraform management without updating or deleting the resource in the API. When set to "DELETE", deleting the resource is allowed.
- display
Name string - Human-readable name for the OnlineEvaluator. The name doesn't have to be unique. The name can consist of any UTF-8 characters. The maximum length is 63 characters.
- project string
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
- region string
- The region of the OnlineEvaluator (e.g. us-central1). If unset, the provider's region is used.
- agent_
resource str - The name of the agent that the OnlineEvaluator evaluates periodically. This value is used to filter the traces with a matching cloud.resource_id and link the evaluation results with relevant dashboards/UIs. This field is immutable; once set, it cannot be changed.
- cloud_
observability AiOnline Evaluator Cloud Observability Args - Data source for the OnlineEvaluator, based on the Google Cloud
Observability stack (Cloud Trace & Cloud Logging). The data source type is
immutable once set. Within
cloudObservability,logViewandtraceViewcan be updated; the eval scope (traceScope/sessionScope) and the convention (openTelemetry) are immutable. Structure is documented below. - config
Ai
Online Evaluator Config Args - Configuration for sampling behavior of the OnlineEvaluator. The OnlineEvaluator runs at a fixed interval of 10 minutes. Structure is documented below.
- metric_
sources Sequence[AiOnline Evaluator Metric Source Args] - A list of metric sources to be used for evaluating samples. At least one
metric source must be provided. Only predefined metrics (inline
metric) and registered metrics (metricResourceName) are supported. Every registered metric must have adisplayName(ortitle) and ascoreRangedefined, otherwise the evaluations will fail. The maximum number of metric sources is 25. Exactly one ofmetricormetricResourceNamemust be set per metric source. Structure is documented below. - deletion_
policy str - Whether Terraform will be prevented from destroying the resource. Defaults to DELETE. When a 'terraform destroy' or 'pulumi up' would delete the resource, the command will fail if this field is set to "PREVENT" in Terraform state. When set to "ABANDON", the command will remove the resource from Terraform management without updating or deleting the resource in the API. When set to "DELETE", deleting the resource is allowed.
- display_
name str - Human-readable name for the OnlineEvaluator. The name doesn't have to be unique. The name can consist of any UTF-8 characters. The maximum length is 63 characters.
- project str
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
- region str
- The region of the OnlineEvaluator (e.g. us-central1). If unset, the provider's region is used.
- agent
Resource String - The name of the agent that the OnlineEvaluator evaluates periodically. This value is used to filter the traces with a matching cloud.resource_id and link the evaluation results with relevant dashboards/UIs. This field is immutable; once set, it cannot be changed.
- cloud
Observability Property Map - Data source for the OnlineEvaluator, based on the Google Cloud
Observability stack (Cloud Trace & Cloud Logging). The data source type is
immutable once set. Within
cloudObservability,logViewandtraceViewcan be updated; the eval scope (traceScope/sessionScope) and the convention (openTelemetry) are immutable. Structure is documented below. - config Property Map
- Configuration for sampling behavior of the OnlineEvaluator. The OnlineEvaluator runs at a fixed interval of 10 minutes. Structure is documented below.
- metric
Sources List<Property Map> - A list of metric sources to be used for evaluating samples. At least one
metric source must be provided. Only predefined metrics (inline
metric) and registered metrics (metricResourceName) are supported. Every registered metric must have adisplayName(ortitle) and ascoreRangedefined, otherwise the evaluations will fail. The maximum number of metric sources is 25. Exactly one ofmetricormetricResourceNamemust be set per metric source. Structure is documented below. - deletion
Policy String - Whether Terraform will be prevented from destroying the resource. Defaults to DELETE. When a 'terraform destroy' or 'pulumi up' would delete the resource, the command will fail if this field is set to "PREVENT" in Terraform state. When set to "ABANDON", the command will remove the resource from Terraform management without updating or deleting the resource in the API. When set to "DELETE", deleting the resource is allowed.
- display
Name String - Human-readable name for the OnlineEvaluator. The name doesn't have to be unique. The name can consist of any UTF-8 characters. The maximum length is 63 characters.
- project String
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
- region String
- The region of the OnlineEvaluator (e.g. us-central1). If unset, the provider's region is used.
Outputs
All input properties are implicitly available as output properties. Additionally, the AiOnlineEvaluator resource produces the following output properties:
- Create
Time string - Timestamp when the OnlineEvaluator was created.
- Id string
- The provider-assigned unique ID for this managed resource.
- Name string
- The resource name of the OnlineEvaluator. This is the final component of the resource path. Format: projects/{{project}}/locations/{{region}}/onlineEvaluators/{{id}}.
- State string
- The state of the OnlineEvaluator. Possible values are: ACTIVE, SUSPENDED, FAILED, WARNING.
- State
Details List<AiOnline Evaluator State Detail> - Contains additional information about the state of the OnlineEvaluator. This is used to provide more details in the event of a failure. Structure is documented below.
- Update
Time string - Timestamp when the OnlineEvaluator was last updated.
- Create
Time string - Timestamp when the OnlineEvaluator was created.
- Id string
- The provider-assigned unique ID for this managed resource.
- Name string
- The resource name of the OnlineEvaluator. This is the final component of the resource path. Format: projects/{{project}}/locations/{{region}}/onlineEvaluators/{{id}}.
- State string
- The state of the OnlineEvaluator. Possible values are: ACTIVE, SUSPENDED, FAILED, WARNING.
- State
Details []AiOnline Evaluator State Detail - Contains additional information about the state of the OnlineEvaluator. This is used to provide more details in the event of a failure. Structure is documented below.
- Update
Time string - Timestamp when the OnlineEvaluator was last updated.
- create_
time string - Timestamp when the OnlineEvaluator was created.
- id string
- The provider-assigned unique ID for this managed resource.
- name string
- The resource name of the OnlineEvaluator. This is the final component of the resource path. Format: projects/{{project}}/locations/{{region}}/onlineEvaluators/{{id}}.
- state string
- The state of the OnlineEvaluator. Possible values are: ACTIVE, SUSPENDED, FAILED, WARNING.
- state_
details list(object) - Contains additional information about the state of the OnlineEvaluator. This is used to provide more details in the event of a failure. Structure is documented below.
- update_
time string - Timestamp when the OnlineEvaluator was last updated.
- create
Time String - Timestamp when the OnlineEvaluator was created.
- id String
- The provider-assigned unique ID for this managed resource.
- name String
- The resource name of the OnlineEvaluator. This is the final component of the resource path. Format: projects/{{project}}/locations/{{region}}/onlineEvaluators/{{id}}.
- state String
- The state of the OnlineEvaluator. Possible values are: ACTIVE, SUSPENDED, FAILED, WARNING.
- state
Details List<AiOnline Evaluator State Detail> - Contains additional information about the state of the OnlineEvaluator. This is used to provide more details in the event of a failure. Structure is documented below.
- update
Time String - Timestamp when the OnlineEvaluator was last updated.
- create
Time string - Timestamp when the OnlineEvaluator was created.
- id string
- The provider-assigned unique ID for this managed resource.
- name string
- The resource name of the OnlineEvaluator. This is the final component of the resource path. Format: projects/{{project}}/locations/{{region}}/onlineEvaluators/{{id}}.
- state string
- The state of the OnlineEvaluator. Possible values are: ACTIVE, SUSPENDED, FAILED, WARNING.
- state
Details AiOnline Evaluator State Detail[] - Contains additional information about the state of the OnlineEvaluator. This is used to provide more details in the event of a failure. Structure is documented below.
- update
Time string - Timestamp when the OnlineEvaluator was last updated.
- create_
time str - Timestamp when the OnlineEvaluator was created.
- id str
- The provider-assigned unique ID for this managed resource.
- name str
- The resource name of the OnlineEvaluator. This is the final component of the resource path. Format: projects/{{project}}/locations/{{region}}/onlineEvaluators/{{id}}.
- state str
- The state of the OnlineEvaluator. Possible values are: ACTIVE, SUSPENDED, FAILED, WARNING.
- state_
details Sequence[AiOnline Evaluator State Detail] - Contains additional information about the state of the OnlineEvaluator. This is used to provide more details in the event of a failure. Structure is documented below.
- update_
time str - Timestamp when the OnlineEvaluator was last updated.
- create
Time String - Timestamp when the OnlineEvaluator was created.
- id String
- The provider-assigned unique ID for this managed resource.
- name String
- The resource name of the OnlineEvaluator. This is the final component of the resource path. Format: projects/{{project}}/locations/{{region}}/onlineEvaluators/{{id}}.
- state String
- The state of the OnlineEvaluator. Possible values are: ACTIVE, SUSPENDED, FAILED, WARNING.
- state
Details List<Property Map> - Contains additional information about the state of the OnlineEvaluator. This is used to provide more details in the event of a failure. Structure is documented below.
- update
Time String - Timestamp when the OnlineEvaluator was last updated.
Look up Existing AiOnlineEvaluator Resource
Get an existing AiOnlineEvaluator resource’s state with the given name, ID, and optional extra properties used to qualify the lookup.
public static get(name: string, id: Input<ID>, state?: AiOnlineEvaluatorState, opts?: CustomResourceOptions): AiOnlineEvaluator@staticmethod
def get(resource_name: str,
id: str,
opts: Optional[ResourceOptions] = None,
agent_resource: Optional[str] = None,
cloud_observability: Optional[AiOnlineEvaluatorCloudObservabilityArgs] = None,
config: Optional[AiOnlineEvaluatorConfigArgs] = None,
create_time: Optional[str] = None,
deletion_policy: Optional[str] = None,
display_name: Optional[str] = None,
metric_sources: Optional[Sequence[AiOnlineEvaluatorMetricSourceArgs]] = None,
name: Optional[str] = None,
project: Optional[str] = None,
region: Optional[str] = None,
state: Optional[str] = None,
state_details: Optional[Sequence[AiOnlineEvaluatorStateDetailArgs]] = None,
update_time: Optional[str] = None) -> AiOnlineEvaluatorfunc GetAiOnlineEvaluator(ctx *Context, name string, id IDInput, state *AiOnlineEvaluatorState, opts ...ResourceOption) (*AiOnlineEvaluator, error)public static AiOnlineEvaluator Get(string name, Input<string> id, AiOnlineEvaluatorState? state, CustomResourceOptions? opts = null)public static AiOnlineEvaluator get(String name, Output<String> id, AiOnlineEvaluatorState state, CustomResourceOptions options)resources: existingAiOnlineEvaluator: type: gcp:vertex:AiOnlineEvaluator get: id: ${id}import {
to = gcp_vertex_ai_online_evaluator.example
id = "${id}"
}
- name string
- The unique name of the resulting resource.
- id Input<ID>
- The unique provider ID of the resource to look up.
- state AiOnlineEvaluatorState
- Known property values the provider needs, along with the ID, to find the resource. Usually not required.
- opts CustomResourceOptions
- Bag of options to control resource's behavior.
- resource_name str
- The unique name of the resulting resource.
- id str
- The unique provider ID of the resource to look up.
- opts ResourceOptions
- Bag of options to control resource's behavior.
- ctx Context
- Context object for the current deployment.
- name string
- The unique name of the resulting resource.
- id IDInput
- The unique provider ID of the resource to look up.
- state AiOnlineEvaluatorState
- Known property values the provider needs, along with the ID, to find the resource. Usually not required.
- opts ResourceOption
- Bag of options to control resource's behavior.
- name string
- The unique name of the resulting resource.
- id Input<string>
- The unique provider ID of the resource to look up.
- state AiOnlineEvaluatorState
- Known property values the provider needs, along with the ID, to find the resource. Usually not required.
- opts CustomResourceOptions
- Bag of options to control resource's behavior.
- name String
- The unique name of the resulting resource.
- id Output<String>
- The unique provider ID of the resource to look up.
- state AiOnlineEvaluatorState
- Known property values the provider needs, along with the ID, to find the resource. Usually not required.
- options CustomResourceOptions
- Bag of options to control resource's behavior.
- Agent
Resource string - The name of the agent that the OnlineEvaluator evaluates periodically. This value is used to filter the traces with a matching cloud.resource_id and link the evaluation results with relevant dashboards/UIs. This field is immutable; once set, it cannot be changed.
- Cloud
Observability AiOnline Evaluator Cloud Observability - Data source for the OnlineEvaluator, based on the Google Cloud
Observability stack (Cloud Trace & Cloud Logging). The data source type is
immutable once set. Within
cloudObservability,logViewandtraceViewcan be updated; the eval scope (traceScope/sessionScope) and the convention (openTelemetry) are immutable. Structure is documented below. - Config
Ai
Online Evaluator Config - Configuration for sampling behavior of the OnlineEvaluator. The OnlineEvaluator runs at a fixed interval of 10 minutes. Structure is documented below.
- Create
Time string - Timestamp when the OnlineEvaluator was created.
- Deletion
Policy string - Whether Terraform will be prevented from destroying the resource. Defaults to DELETE. When a 'terraform destroy' or 'pulumi up' would delete the resource, the command will fail if this field is set to "PREVENT" in Terraform state. When set to "ABANDON", the command will remove the resource from Terraform management without updating or deleting the resource in the API. When set to "DELETE", deleting the resource is allowed.
- Display
Name string - Human-readable name for the OnlineEvaluator. The name doesn't have to be unique. The name can consist of any UTF-8 characters. The maximum length is 63 characters.
- Metric
Sources List<AiOnline Evaluator Metric Source> - A list of metric sources to be used for evaluating samples. At least one
metric source must be provided. Only predefined metrics (inline
metric) and registered metrics (metricResourceName) are supported. Every registered metric must have adisplayName(ortitle) and ascoreRangedefined, otherwise the evaluations will fail. The maximum number of metric sources is 25. Exactly one ofmetricormetricResourceNamemust be set per metric source. Structure is documented below. - Name string
- The resource name of the OnlineEvaluator. This is the final component of the resource path. Format: projects/{{project}}/locations/{{region}}/onlineEvaluators/{{id}}.
- Project string
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
- Region string
- The region of the OnlineEvaluator (e.g. us-central1). If unset, the provider's region is used.
- State string
- The state of the OnlineEvaluator. Possible values are: ACTIVE, SUSPENDED, FAILED, WARNING.
- State
Details List<AiOnline Evaluator State Detail> - Contains additional information about the state of the OnlineEvaluator. This is used to provide more details in the event of a failure. Structure is documented below.
- Update
Time string - Timestamp when the OnlineEvaluator was last updated.
- Agent
Resource string - The name of the agent that the OnlineEvaluator evaluates periodically. This value is used to filter the traces with a matching cloud.resource_id and link the evaluation results with relevant dashboards/UIs. This field is immutable; once set, it cannot be changed.
- Cloud
Observability AiOnline Evaluator Cloud Observability Args - Data source for the OnlineEvaluator, based on the Google Cloud
Observability stack (Cloud Trace & Cloud Logging). The data source type is
immutable once set. Within
cloudObservability,logViewandtraceViewcan be updated; the eval scope (traceScope/sessionScope) and the convention (openTelemetry) are immutable. Structure is documented below. - Config
Ai
Online Evaluator Config Args - Configuration for sampling behavior of the OnlineEvaluator. The OnlineEvaluator runs at a fixed interval of 10 minutes. Structure is documented below.
- Create
Time string - Timestamp when the OnlineEvaluator was created.
- Deletion
Policy string - Whether Terraform will be prevented from destroying the resource. Defaults to DELETE. When a 'terraform destroy' or 'pulumi up' would delete the resource, the command will fail if this field is set to "PREVENT" in Terraform state. When set to "ABANDON", the command will remove the resource from Terraform management without updating or deleting the resource in the API. When set to "DELETE", deleting the resource is allowed.
- Display
Name string - Human-readable name for the OnlineEvaluator. The name doesn't have to be unique. The name can consist of any UTF-8 characters. The maximum length is 63 characters.
- Metric
Sources []AiOnline Evaluator Metric Source Args - A list of metric sources to be used for evaluating samples. At least one
metric source must be provided. Only predefined metrics (inline
metric) and registered metrics (metricResourceName) are supported. Every registered metric must have adisplayName(ortitle) and ascoreRangedefined, otherwise the evaluations will fail. The maximum number of metric sources is 25. Exactly one ofmetricormetricResourceNamemust be set per metric source. Structure is documented below. - Name string
- The resource name of the OnlineEvaluator. This is the final component of the resource path. Format: projects/{{project}}/locations/{{region}}/onlineEvaluators/{{id}}.
- Project string
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
- Region string
- The region of the OnlineEvaluator (e.g. us-central1). If unset, the provider's region is used.
- State string
- The state of the OnlineEvaluator. Possible values are: ACTIVE, SUSPENDED, FAILED, WARNING.
- State
Details []AiOnline Evaluator State Detail Args - Contains additional information about the state of the OnlineEvaluator. This is used to provide more details in the event of a failure. Structure is documented below.
- Update
Time string - Timestamp when the OnlineEvaluator was last updated.
- agent_
resource string - The name of the agent that the OnlineEvaluator evaluates periodically. This value is used to filter the traces with a matching cloud.resource_id and link the evaluation results with relevant dashboards/UIs. This field is immutable; once set, it cannot be changed.
- cloud_
observability object - Data source for the OnlineEvaluator, based on the Google Cloud
Observability stack (Cloud Trace & Cloud Logging). The data source type is
immutable once set. Within
cloudObservability,logViewandtraceViewcan be updated; the eval scope (traceScope/sessionScope) and the convention (openTelemetry) are immutable. Structure is documented below. - config object
- Configuration for sampling behavior of the OnlineEvaluator. The OnlineEvaluator runs at a fixed interval of 10 minutes. Structure is documented below.
- create_
time string - Timestamp when the OnlineEvaluator was created.
- deletion_
policy string - Whether Terraform will be prevented from destroying the resource. Defaults to DELETE. When a 'terraform destroy' or 'pulumi up' would delete the resource, the command will fail if this field is set to "PREVENT" in Terraform state. When set to "ABANDON", the command will remove the resource from Terraform management without updating or deleting the resource in the API. When set to "DELETE", deleting the resource is allowed.
- display_
name string - Human-readable name for the OnlineEvaluator. The name doesn't have to be unique. The name can consist of any UTF-8 characters. The maximum length is 63 characters.
- metric_
sources list(object) - A list of metric sources to be used for evaluating samples. At least one
metric source must be provided. Only predefined metrics (inline
metric) and registered metrics (metricResourceName) are supported. Every registered metric must have adisplayName(ortitle) and ascoreRangedefined, otherwise the evaluations will fail. The maximum number of metric sources is 25. Exactly one ofmetricormetricResourceNamemust be set per metric source. Structure is documented below. - name string
- The resource name of the OnlineEvaluator. This is the final component of the resource path. Format: projects/{{project}}/locations/{{region}}/onlineEvaluators/{{id}}.
- project string
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
- region string
- The region of the OnlineEvaluator (e.g. us-central1). If unset, the provider's region is used.
- state string
- The state of the OnlineEvaluator. Possible values are: ACTIVE, SUSPENDED, FAILED, WARNING.
- state_
details list(object) - Contains additional information about the state of the OnlineEvaluator. This is used to provide more details in the event of a failure. Structure is documented below.
- update_
time string - Timestamp when the OnlineEvaluator was last updated.
- agent
Resource String - The name of the agent that the OnlineEvaluator evaluates periodically. This value is used to filter the traces with a matching cloud.resource_id and link the evaluation results with relevant dashboards/UIs. This field is immutable; once set, it cannot be changed.
- cloud
Observability AiOnline Evaluator Cloud Observability - Data source for the OnlineEvaluator, based on the Google Cloud
Observability stack (Cloud Trace & Cloud Logging). The data source type is
immutable once set. Within
cloudObservability,logViewandtraceViewcan be updated; the eval scope (traceScope/sessionScope) and the convention (openTelemetry) are immutable. Structure is documented below. - config
Ai
Online Evaluator Config - Configuration for sampling behavior of the OnlineEvaluator. The OnlineEvaluator runs at a fixed interval of 10 minutes. Structure is documented below.
- create
Time String - Timestamp when the OnlineEvaluator was created.
- deletion
Policy String - Whether Terraform will be prevented from destroying the resource. Defaults to DELETE. When a 'terraform destroy' or 'pulumi up' would delete the resource, the command will fail if this field is set to "PREVENT" in Terraform state. When set to "ABANDON", the command will remove the resource from Terraform management without updating or deleting the resource in the API. When set to "DELETE", deleting the resource is allowed.
- display
Name String - Human-readable name for the OnlineEvaluator. The name doesn't have to be unique. The name can consist of any UTF-8 characters. The maximum length is 63 characters.
- metric
Sources List<AiOnline Evaluator Metric Source> - A list of metric sources to be used for evaluating samples. At least one
metric source must be provided. Only predefined metrics (inline
metric) and registered metrics (metricResourceName) are supported. Every registered metric must have adisplayName(ortitle) and ascoreRangedefined, otherwise the evaluations will fail. The maximum number of metric sources is 25. Exactly one ofmetricormetricResourceNamemust be set per metric source. Structure is documented below. - name String
- The resource name of the OnlineEvaluator. This is the final component of the resource path. Format: projects/{{project}}/locations/{{region}}/onlineEvaluators/{{id}}.
- project String
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
- region String
- The region of the OnlineEvaluator (e.g. us-central1). If unset, the provider's region is used.
- state String
- The state of the OnlineEvaluator. Possible values are: ACTIVE, SUSPENDED, FAILED, WARNING.
- state
Details List<AiOnline Evaluator State Detail> - Contains additional information about the state of the OnlineEvaluator. This is used to provide more details in the event of a failure. Structure is documented below.
- update
Time String - Timestamp when the OnlineEvaluator was last updated.
- agent
Resource string - The name of the agent that the OnlineEvaluator evaluates periodically. This value is used to filter the traces with a matching cloud.resource_id and link the evaluation results with relevant dashboards/UIs. This field is immutable; once set, it cannot be changed.
- cloud
Observability AiOnline Evaluator Cloud Observability - Data source for the OnlineEvaluator, based on the Google Cloud
Observability stack (Cloud Trace & Cloud Logging). The data source type is
immutable once set. Within
cloudObservability,logViewandtraceViewcan be updated; the eval scope (traceScope/sessionScope) and the convention (openTelemetry) are immutable. Structure is documented below. - config
Ai
Online Evaluator Config - Configuration for sampling behavior of the OnlineEvaluator. The OnlineEvaluator runs at a fixed interval of 10 minutes. Structure is documented below.
- create
Time string - Timestamp when the OnlineEvaluator was created.
- deletion
Policy string - Whether Terraform will be prevented from destroying the resource. Defaults to DELETE. When a 'terraform destroy' or 'pulumi up' would delete the resource, the command will fail if this field is set to "PREVENT" in Terraform state. When set to "ABANDON", the command will remove the resource from Terraform management without updating or deleting the resource in the API. When set to "DELETE", deleting the resource is allowed.
- display
Name string - Human-readable name for the OnlineEvaluator. The name doesn't have to be unique. The name can consist of any UTF-8 characters. The maximum length is 63 characters.
- metric
Sources AiOnline Evaluator Metric Source[] - A list of metric sources to be used for evaluating samples. At least one
metric source must be provided. Only predefined metrics (inline
metric) and registered metrics (metricResourceName) are supported. Every registered metric must have adisplayName(ortitle) and ascoreRangedefined, otherwise the evaluations will fail. The maximum number of metric sources is 25. Exactly one ofmetricormetricResourceNamemust be set per metric source. Structure is documented below. - name string
- The resource name of the OnlineEvaluator. This is the final component of the resource path. Format: projects/{{project}}/locations/{{region}}/onlineEvaluators/{{id}}.
- project string
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
- region string
- The region of the OnlineEvaluator (e.g. us-central1). If unset, the provider's region is used.
- state string
- The state of the OnlineEvaluator. Possible values are: ACTIVE, SUSPENDED, FAILED, WARNING.
- state
Details AiOnline Evaluator State Detail[] - Contains additional information about the state of the OnlineEvaluator. This is used to provide more details in the event of a failure. Structure is documented below.
- update
Time string - Timestamp when the OnlineEvaluator was last updated.
- agent_
resource str - The name of the agent that the OnlineEvaluator evaluates periodically. This value is used to filter the traces with a matching cloud.resource_id and link the evaluation results with relevant dashboards/UIs. This field is immutable; once set, it cannot be changed.
- cloud_
observability AiOnline Evaluator Cloud Observability Args - Data source for the OnlineEvaluator, based on the Google Cloud
Observability stack (Cloud Trace & Cloud Logging). The data source type is
immutable once set. Within
cloudObservability,logViewandtraceViewcan be updated; the eval scope (traceScope/sessionScope) and the convention (openTelemetry) are immutable. Structure is documented below. - config
Ai
Online Evaluator Config Args - Configuration for sampling behavior of the OnlineEvaluator. The OnlineEvaluator runs at a fixed interval of 10 minutes. Structure is documented below.
- create_
time str - Timestamp when the OnlineEvaluator was created.
- deletion_
policy str - Whether Terraform will be prevented from destroying the resource. Defaults to DELETE. When a 'terraform destroy' or 'pulumi up' would delete the resource, the command will fail if this field is set to "PREVENT" in Terraform state. When set to "ABANDON", the command will remove the resource from Terraform management without updating or deleting the resource in the API. When set to "DELETE", deleting the resource is allowed.
- display_
name str - Human-readable name for the OnlineEvaluator. The name doesn't have to be unique. The name can consist of any UTF-8 characters. The maximum length is 63 characters.
- metric_
sources Sequence[AiOnline Evaluator Metric Source Args] - A list of metric sources to be used for evaluating samples. At least one
metric source must be provided. Only predefined metrics (inline
metric) and registered metrics (metricResourceName) are supported. Every registered metric must have adisplayName(ortitle) and ascoreRangedefined, otherwise the evaluations will fail. The maximum number of metric sources is 25. Exactly one ofmetricormetricResourceNamemust be set per metric source. Structure is documented below. - name str
- The resource name of the OnlineEvaluator. This is the final component of the resource path. Format: projects/{{project}}/locations/{{region}}/onlineEvaluators/{{id}}.
- project str
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
- region str
- The region of the OnlineEvaluator (e.g. us-central1). If unset, the provider's region is used.
- state str
- The state of the OnlineEvaluator. Possible values are: ACTIVE, SUSPENDED, FAILED, WARNING.
- state_
details Sequence[AiOnline Evaluator State Detail Args] - Contains additional information about the state of the OnlineEvaluator. This is used to provide more details in the event of a failure. Structure is documented below.
- update_
time str - Timestamp when the OnlineEvaluator was last updated.
- agent
Resource String - The name of the agent that the OnlineEvaluator evaluates periodically. This value is used to filter the traces with a matching cloud.resource_id and link the evaluation results with relevant dashboards/UIs. This field is immutable; once set, it cannot be changed.
- cloud
Observability Property Map - Data source for the OnlineEvaluator, based on the Google Cloud
Observability stack (Cloud Trace & Cloud Logging). The data source type is
immutable once set. Within
cloudObservability,logViewandtraceViewcan be updated; the eval scope (traceScope/sessionScope) and the convention (openTelemetry) are immutable. Structure is documented below. - config Property Map
- Configuration for sampling behavior of the OnlineEvaluator. The OnlineEvaluator runs at a fixed interval of 10 minutes. Structure is documented below.
- create
Time String - Timestamp when the OnlineEvaluator was created.
- deletion
Policy String - Whether Terraform will be prevented from destroying the resource. Defaults to DELETE. When a 'terraform destroy' or 'pulumi up' would delete the resource, the command will fail if this field is set to "PREVENT" in Terraform state. When set to "ABANDON", the command will remove the resource from Terraform management without updating or deleting the resource in the API. When set to "DELETE", deleting the resource is allowed.
- display
Name String - Human-readable name for the OnlineEvaluator. The name doesn't have to be unique. The name can consist of any UTF-8 characters. The maximum length is 63 characters.
- metric
Sources List<Property Map> - A list of metric sources to be used for evaluating samples. At least one
metric source must be provided. Only predefined metrics (inline
metric) and registered metrics (metricResourceName) are supported. Every registered metric must have adisplayName(ortitle) and ascoreRangedefined, otherwise the evaluations will fail. The maximum number of metric sources is 25. Exactly one ofmetricormetricResourceNamemust be set per metric source. Structure is documented below. - name String
- The resource name of the OnlineEvaluator. This is the final component of the resource path. Format: projects/{{project}}/locations/{{region}}/onlineEvaluators/{{id}}.
- project String
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
- region String
- The region of the OnlineEvaluator (e.g. us-central1). If unset, the provider's region is used.
- state String
- The state of the OnlineEvaluator. Possible values are: ACTIVE, SUSPENDED, FAILED, WARNING.
- state
Details List<Property Map> - Contains additional information about the state of the OnlineEvaluator. This is used to provide more details in the event of a failure. Structure is documented below.
- update
Time String - Timestamp when the OnlineEvaluator was last updated.
Supporting Types
AiOnlineEvaluatorCloudObservability, AiOnlineEvaluatorCloudObservabilityArgs
- Open
Telemetry AiOnline Evaluator Cloud Observability Open Telemetry - Configuration for a data source following the OpenTelemetry convention. This is the convention that the data source follows. Structure is documented below.
- Log
View string - Optional log view that will be used to query logs. If empty, the
project's default view (
projects/{{project_id}}) will be used. - Session
Scope AiOnline Evaluator Cloud Observability Session Scope - If chosen, the OnlineEvaluator will evaluate sessions matching the
specified filter. A session is a group of traces with a common
gen_ai.conversation.id. Exactly one oftraceScopeorsessionScopemust be set. Structure is documented below. - Trace
Scope AiOnline Evaluator Cloud Observability Trace Scope - If chosen, the OnlineEvaluator will evaluate single traces matching
the specified filter. Exactly one of
traceScopeorsessionScopemust be set. Structure is documented below. - Trace
View string - Optional trace view that will be used to query traces. If empty, the
_AllSpansview from the_TraceUS bucket will be used.
- Open
Telemetry AiOnline Evaluator Cloud Observability Open Telemetry - Configuration for a data source following the OpenTelemetry convention. This is the convention that the data source follows. Structure is documented below.
- Log
View string - Optional log view that will be used to query logs. If empty, the
project's default view (
projects/{{project_id}}) will be used. - Session
Scope AiOnline Evaluator Cloud Observability Session Scope - If chosen, the OnlineEvaluator will evaluate sessions matching the
specified filter. A session is a group of traces with a common
gen_ai.conversation.id. Exactly one oftraceScopeorsessionScopemust be set. Structure is documented below. - Trace
Scope AiOnline Evaluator Cloud Observability Trace Scope - If chosen, the OnlineEvaluator will evaluate single traces matching
the specified filter. Exactly one of
traceScopeorsessionScopemust be set. Structure is documented below. - Trace
View string - Optional trace view that will be used to query traces. If empty, the
_AllSpansview from the_TraceUS bucket will be used.
- open_
telemetry object - Configuration for a data source following the OpenTelemetry convention. This is the convention that the data source follows. Structure is documented below.
- log_
view string - Optional log view that will be used to query logs. If empty, the
project's default view (
projects/{{project_id}}) will be used. - session_
scope object - If chosen, the OnlineEvaluator will evaluate sessions matching the
specified filter. A session is a group of traces with a common
gen_ai.conversation.id. Exactly one oftraceScopeorsessionScopemust be set. Structure is documented below. - trace_
scope object - If chosen, the OnlineEvaluator will evaluate single traces matching
the specified filter. Exactly one of
traceScopeorsessionScopemust be set. Structure is documented below. - trace_
view string - Optional trace view that will be used to query traces. If empty, the
_AllSpansview from the_TraceUS bucket will be used.
- open
Telemetry AiOnline Evaluator Cloud Observability Open Telemetry - Configuration for a data source following the OpenTelemetry convention. This is the convention that the data source follows. Structure is documented below.
- log
View String - Optional log view that will be used to query logs. If empty, the
project's default view (
projects/{{project_id}}) will be used. - session
Scope AiOnline Evaluator Cloud Observability Session Scope - If chosen, the OnlineEvaluator will evaluate sessions matching the
specified filter. A session is a group of traces with a common
gen_ai.conversation.id. Exactly one oftraceScopeorsessionScopemust be set. Structure is documented below. - trace
Scope AiOnline Evaluator Cloud Observability Trace Scope - If chosen, the OnlineEvaluator will evaluate single traces matching
the specified filter. Exactly one of
traceScopeorsessionScopemust be set. Structure is documented below. - trace
View String - Optional trace view that will be used to query traces. If empty, the
_AllSpansview from the_TraceUS bucket will be used.
- open
Telemetry AiOnline Evaluator Cloud Observability Open Telemetry - Configuration for a data source following the OpenTelemetry convention. This is the convention that the data source follows. Structure is documented below.
- log
View string - Optional log view that will be used to query logs. If empty, the
project's default view (
projects/{{project_id}}) will be used. - session
Scope AiOnline Evaluator Cloud Observability Session Scope - If chosen, the OnlineEvaluator will evaluate sessions matching the
specified filter. A session is a group of traces with a common
gen_ai.conversation.id. Exactly one oftraceScopeorsessionScopemust be set. Structure is documented below. - trace
Scope AiOnline Evaluator Cloud Observability Trace Scope - If chosen, the OnlineEvaluator will evaluate single traces matching
the specified filter. Exactly one of
traceScopeorsessionScopemust be set. Structure is documented below. - trace
View string - Optional trace view that will be used to query traces. If empty, the
_AllSpansview from the_TraceUS bucket will be used.
- open_
telemetry AiOnline Evaluator Cloud Observability Open Telemetry - Configuration for a data source following the OpenTelemetry convention. This is the convention that the data source follows. Structure is documented below.
- log_
view str - Optional log view that will be used to query logs. If empty, the
project's default view (
projects/{{project_id}}) will be used. - session_
scope AiOnline Evaluator Cloud Observability Session Scope - If chosen, the OnlineEvaluator will evaluate sessions matching the
specified filter. A session is a group of traces with a common
gen_ai.conversation.id. Exactly one oftraceScopeorsessionScopemust be set. Structure is documented below. - trace_
scope AiOnline Evaluator Cloud Observability Trace Scope - If chosen, the OnlineEvaluator will evaluate single traces matching
the specified filter. Exactly one of
traceScopeorsessionScopemust be set. Structure is documented below. - trace_
view str - Optional trace view that will be used to query traces. If empty, the
_AllSpansview from the_TraceUS bucket will be used.
- open
Telemetry Property Map - Configuration for a data source following the OpenTelemetry convention. This is the convention that the data source follows. Structure is documented below.
- log
View String - Optional log view that will be used to query logs. If empty, the
project's default view (
projects/{{project_id}}) will be used. - session
Scope Property Map - If chosen, the OnlineEvaluator will evaluate sessions matching the
specified filter. A session is a group of traces with a common
gen_ai.conversation.id. Exactly one oftraceScopeorsessionScopemust be set. Structure is documented below. - trace
Scope Property Map - If chosen, the OnlineEvaluator will evaluate single traces matching
the specified filter. Exactly one of
traceScopeorsessionScopemust be set. Structure is documented below. - trace
View String - Optional trace view that will be used to query traces. If empty, the
_AllSpansview from the_TraceUS bucket will be used.
AiOnlineEvaluatorCloudObservabilityOpenTelemetry, AiOnlineEvaluatorCloudObservabilityOpenTelemetryArgs
- Semconv
Version string - Defines which version of the OTel Semantic Convention the data follows. Can be "1.39.0" or newer.
- Semconv
Version string - Defines which version of the OTel Semantic Convention the data follows. Can be "1.39.0" or newer.
- semconv_
version string - Defines which version of the OTel Semantic Convention the data follows. Can be "1.39.0" or newer.
- semconv
Version String - Defines which version of the OTel Semantic Convention the data follows. Can be "1.39.0" or newer.
- semconv
Version string - Defines which version of the OTel Semantic Convention the data follows. Can be "1.39.0" or newer.
- semconv_
version str - Defines which version of the OTel Semantic Convention the data follows. Can be "1.39.0" or newer.
- semconv
Version String - Defines which version of the OTel Semantic Convention the data follows. Can be "1.39.0" or newer.
AiOnlineEvaluatorCloudObservabilitySessionScope, AiOnlineEvaluatorCloudObservabilitySessionScopeArgs
- Filters
List<Ai
Online Evaluator Cloud Observability Session Scope Filter> - A list of predicates to filter sessions. Multiple predicates are combined using AND. The maximum number of predicates is 10. Structure is documented below.
- Inactivity
Trigger AiOnline Evaluator Cloud Observability Session Scope Inactivity Trigger - Session is considered ready for evaluation when there are no new traces for a specified period of inactivity. Structure is documented below.
- Filters
[]Ai
Online Evaluator Cloud Observability Session Scope Filter - A list of predicates to filter sessions. Multiple predicates are combined using AND. The maximum number of predicates is 10. Structure is documented below.
- Inactivity
Trigger AiOnline Evaluator Cloud Observability Session Scope Inactivity Trigger - Session is considered ready for evaluation when there are no new traces for a specified period of inactivity. Structure is documented below.
- filters list(object)
- A list of predicates to filter sessions. Multiple predicates are combined using AND. The maximum number of predicates is 10. Structure is documented below.
- inactivity_
trigger object - Session is considered ready for evaluation when there are no new traces for a specified period of inactivity. Structure is documented below.
- filters
List<Ai
Online Evaluator Cloud Observability Session Scope Filter> - A list of predicates to filter sessions. Multiple predicates are combined using AND. The maximum number of predicates is 10. Structure is documented below.
- inactivity
Trigger AiOnline Evaluator Cloud Observability Session Scope Inactivity Trigger - Session is considered ready for evaluation when there are no new traces for a specified period of inactivity. Structure is documented below.
- filters
Ai
Online Evaluator Cloud Observability Session Scope Filter[] - A list of predicates to filter sessions. Multiple predicates are combined using AND. The maximum number of predicates is 10. Structure is documented below.
- inactivity
Trigger AiOnline Evaluator Cloud Observability Session Scope Inactivity Trigger - Session is considered ready for evaluation when there are no new traces for a specified period of inactivity. Structure is documented below.
- filters
Sequence[Ai
Online Evaluator Cloud Observability Session Scope Filter] - A list of predicates to filter sessions. Multiple predicates are combined using AND. The maximum number of predicates is 10. Structure is documented below.
- inactivity_
trigger AiOnline Evaluator Cloud Observability Session Scope Inactivity Trigger - Session is considered ready for evaluation when there are no new traces for a specified period of inactivity. Structure is documented below.
- filters List<Property Map>
- A list of predicates to filter sessions. Multiple predicates are combined using AND. The maximum number of predicates is 10. Structure is documented below.
- inactivity
Trigger Property Map - Session is considered ready for evaluation when there are no new traces for a specified period of inactivity. Structure is documented below.
AiOnlineEvaluatorCloudObservabilitySessionScopeFilter, AiOnlineEvaluatorCloudObservabilitySessionScopeFilterArgs
- Duration
Ai
Online Evaluator Cloud Observability Session Scope Filter Duration - Filter on the duration of a session (in seconds). Structure is documented below.
- Model
Call AiErrors Online Evaluator Cloud Observability Session Scope Filter Model Call Errors - Filter on the number of LLM call errors within a session. Structure is documented below.
- Model
Calls AiOnline Evaluator Cloud Observability Session Scope Filter Model Calls - Filter on the number of underlying LLM calls within a session. Structure is documented below.
- Tool
Call AiErrors Online Evaluator Cloud Observability Session Scope Filter Tool Call Errors - Filter on the number of tool call errors within a session. Structure is documented below.
- Tool
Calls AiOnline Evaluator Cloud Observability Session Scope Filter Tool Calls - Filter on the number of underlying tool calls within a session. Structure is documented below.
- Total
Token AiUsage Online Evaluator Cloud Observability Session Scope Filter Total Token Usage - Filter on the total token usage within a session. Structure is documented below.
- User
Turns AiOnline Evaluator Cloud Observability Session Scope Filter User Turns - Filter on the number of user turns within a session. Structure is documented below.
- Duration
Ai
Online Evaluator Cloud Observability Session Scope Filter Duration - Filter on the duration of a session (in seconds). Structure is documented below.
- Model
Call AiErrors Online Evaluator Cloud Observability Session Scope Filter Model Call Errors - Filter on the number of LLM call errors within a session. Structure is documented below.
- Model
Calls AiOnline Evaluator Cloud Observability Session Scope Filter Model Calls - Filter on the number of underlying LLM calls within a session. Structure is documented below.
- Tool
Call AiErrors Online Evaluator Cloud Observability Session Scope Filter Tool Call Errors - Filter on the number of tool call errors within a session. Structure is documented below.
- Tool
Calls AiOnline Evaluator Cloud Observability Session Scope Filter Tool Calls - Filter on the number of underlying tool calls within a session. Structure is documented below.
- Total
Token AiUsage Online Evaluator Cloud Observability Session Scope Filter Total Token Usage - Filter on the total token usage within a session. Structure is documented below.
- User
Turns AiOnline Evaluator Cloud Observability Session Scope Filter User Turns - Filter on the number of user turns within a session. Structure is documented below.
- duration object
- Filter on the duration of a session (in seconds). Structure is documented below.
- model_
call_ objecterrors - Filter on the number of LLM call errors within a session. Structure is documented below.
- model_
calls object - Filter on the number of underlying LLM calls within a session. Structure is documented below.
- tool_
call_ objecterrors - Filter on the number of tool call errors within a session. Structure is documented below.
- tool_
calls object - Filter on the number of underlying tool calls within a session. Structure is documented below.
- total_
token_ objectusage - Filter on the total token usage within a session. Structure is documented below.
- user_
turns object - Filter on the number of user turns within a session. Structure is documented below.
- duration
Ai
Online Evaluator Cloud Observability Session Scope Filter Duration - Filter on the duration of a session (in seconds). Structure is documented below.
- model
Call AiErrors Online Evaluator Cloud Observability Session Scope Filter Model Call Errors - Filter on the number of LLM call errors within a session. Structure is documented below.
- model
Calls AiOnline Evaluator Cloud Observability Session Scope Filter Model Calls - Filter on the number of underlying LLM calls within a session. Structure is documented below.
- tool
Call AiErrors Online Evaluator Cloud Observability Session Scope Filter Tool Call Errors - Filter on the number of tool call errors within a session. Structure is documented below.
- tool
Calls AiOnline Evaluator Cloud Observability Session Scope Filter Tool Calls - Filter on the number of underlying tool calls within a session. Structure is documented below.
- total
Token AiUsage Online Evaluator Cloud Observability Session Scope Filter Total Token Usage - Filter on the total token usage within a session. Structure is documented below.
- user
Turns AiOnline Evaluator Cloud Observability Session Scope Filter User Turns - Filter on the number of user turns within a session. Structure is documented below.
- duration
Ai
Online Evaluator Cloud Observability Session Scope Filter Duration - Filter on the duration of a session (in seconds). Structure is documented below.
- model
Call AiErrors Online Evaluator Cloud Observability Session Scope Filter Model Call Errors - Filter on the number of LLM call errors within a session. Structure is documented below.
- model
Calls AiOnline Evaluator Cloud Observability Session Scope Filter Model Calls - Filter on the number of underlying LLM calls within a session. Structure is documented below.
- tool
Call AiErrors Online Evaluator Cloud Observability Session Scope Filter Tool Call Errors - Filter on the number of tool call errors within a session. Structure is documented below.
- tool
Calls AiOnline Evaluator Cloud Observability Session Scope Filter Tool Calls - Filter on the number of underlying tool calls within a session. Structure is documented below.
- total
Token AiUsage Online Evaluator Cloud Observability Session Scope Filter Total Token Usage - Filter on the total token usage within a session. Structure is documented below.
- user
Turns AiOnline Evaluator Cloud Observability Session Scope Filter User Turns - Filter on the number of user turns within a session. Structure is documented below.
- duration
Ai
Online Evaluator Cloud Observability Session Scope Filter Duration - Filter on the duration of a session (in seconds). Structure is documented below.
- model_
call_ Aierrors Online Evaluator Cloud Observability Session Scope Filter Model Call Errors - Filter on the number of LLM call errors within a session. Structure is documented below.
- model_
calls AiOnline Evaluator Cloud Observability Session Scope Filter Model Calls - Filter on the number of underlying LLM calls within a session. Structure is documented below.
- tool_
call_ Aierrors Online Evaluator Cloud Observability Session Scope Filter Tool Call Errors - Filter on the number of tool call errors within a session. Structure is documented below.
- tool_
calls AiOnline Evaluator Cloud Observability Session Scope Filter Tool Calls - Filter on the number of underlying tool calls within a session. Structure is documented below.
- total_
token_ Aiusage Online Evaluator Cloud Observability Session Scope Filter Total Token Usage - Filter on the total token usage within a session. Structure is documented below.
- user_
turns AiOnline Evaluator Cloud Observability Session Scope Filter User Turns - Filter on the number of user turns within a session. Structure is documented below.
- duration Property Map
- Filter on the duration of a session (in seconds). Structure is documented below.
- model
Call Property MapErrors - Filter on the number of LLM call errors within a session. Structure is documented below.
- model
Calls Property Map - Filter on the number of underlying LLM calls within a session. Structure is documented below.
- tool
Call Property MapErrors - Filter on the number of tool call errors within a session. Structure is documented below.
- tool
Calls Property Map - Filter on the number of underlying tool calls within a session. Structure is documented below.
- total
Token Property MapUsage - Filter on the total token usage within a session. Structure is documented below.
- user
Turns Property Map - Filter on the number of user turns within a session. Structure is documented below.
AiOnlineEvaluatorCloudObservabilitySessionScopeFilterDuration, AiOnlineEvaluatorCloudObservabilitySessionScopeFilterDurationArgs
- Comparison
Operator string - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- Value double
- The value to compare against.
- Comparison
Operator string - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- Value float64
- The value to compare against.
- comparison_
operator string - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- value number
- The value to compare against.
- comparison
Operator String - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- value Double
- The value to compare against.
- comparison
Operator string - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- value number
- The value to compare against.
- comparison_
operator str - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- value float
- The value to compare against.
- comparison
Operator String - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- value Number
- The value to compare against.
AiOnlineEvaluatorCloudObservabilitySessionScopeFilterModelCallErrors, AiOnlineEvaluatorCloudObservabilitySessionScopeFilterModelCallErrorsArgs
- Comparison
Operator string - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- Value double
- The value to compare against.
- Comparison
Operator string - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- Value float64
- The value to compare against.
- comparison_
operator string - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- value number
- The value to compare against.
- comparison
Operator String - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- value Double
- The value to compare against.
- comparison
Operator string - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- value number
- The value to compare against.
- comparison_
operator str - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- value float
- The value to compare against.
- comparison
Operator String - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- value Number
- The value to compare against.
AiOnlineEvaluatorCloudObservabilitySessionScopeFilterModelCalls, AiOnlineEvaluatorCloudObservabilitySessionScopeFilterModelCallsArgs
- Comparison
Operator string - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- Value double
- The value to compare against.
- Comparison
Operator string - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- Value float64
- The value to compare against.
- comparison_
operator string - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- value number
- The value to compare against.
- comparison
Operator String - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- value Double
- The value to compare against.
- comparison
Operator string - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- value number
- The value to compare against.
- comparison_
operator str - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- value float
- The value to compare against.
- comparison
Operator String - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- value Number
- The value to compare against.
AiOnlineEvaluatorCloudObservabilitySessionScopeFilterToolCallErrors, AiOnlineEvaluatorCloudObservabilitySessionScopeFilterToolCallErrorsArgs
- Comparison
Operator string - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- Value double
- The value to compare against.
- Comparison
Operator string - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- Value float64
- The value to compare against.
- comparison_
operator string - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- value number
- The value to compare against.
- comparison
Operator String - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- value Double
- The value to compare against.
- comparison
Operator string - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- value number
- The value to compare against.
- comparison_
operator str - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- value float
- The value to compare against.
- comparison
Operator String - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- value Number
- The value to compare against.
AiOnlineEvaluatorCloudObservabilitySessionScopeFilterToolCalls, AiOnlineEvaluatorCloudObservabilitySessionScopeFilterToolCallsArgs
- Comparison
Operator string - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- Value double
- The value to compare against.
- Comparison
Operator string - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- Value float64
- The value to compare against.
- comparison_
operator string - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- value number
- The value to compare against.
- comparison
Operator String - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- value Double
- The value to compare against.
- comparison
Operator string - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- value number
- The value to compare against.
- comparison_
operator str - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- value float
- The value to compare against.
- comparison
Operator String - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- value Number
- The value to compare against.
AiOnlineEvaluatorCloudObservabilitySessionScopeFilterTotalTokenUsage, AiOnlineEvaluatorCloudObservabilitySessionScopeFilterTotalTokenUsageArgs
- Comparison
Operator string - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- Value double
- The value to compare against.
- Comparison
Operator string - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- Value float64
- The value to compare against.
- comparison_
operator string - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- value number
- The value to compare against.
- comparison
Operator String - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- value Double
- The value to compare against.
- comparison
Operator string - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- value number
- The value to compare against.
- comparison_
operator str - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- value float
- The value to compare against.
- comparison
Operator String - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- value Number
- The value to compare against.
AiOnlineEvaluatorCloudObservabilitySessionScopeFilterUserTurns, AiOnlineEvaluatorCloudObservabilitySessionScopeFilterUserTurnsArgs
- Comparison
Operator string - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- Value double
- The value to compare against.
- Comparison
Operator string - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- Value float64
- The value to compare against.
- comparison_
operator string - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- value number
- The value to compare against.
- comparison
Operator String - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- value Double
- The value to compare against.
- comparison
Operator string - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- value number
- The value to compare against.
- comparison_
operator str - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- value float
- The value to compare against.
- comparison
Operator String - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- value Number
- The value to compare against.
AiOnlineEvaluatorCloudObservabilitySessionScopeInactivityTrigger, AiOnlineEvaluatorCloudObservabilitySessionScopeInactivityTriggerArgs
- Threshold string
- The amount of time that must pass with no new traces before a session is considered ready for evaluation. Must be a positive duration no greater than 7 days (604800 seconds), e.g. "3600s".
- Threshold string
- The amount of time that must pass with no new traces before a session is considered ready for evaluation. Must be a positive duration no greater than 7 days (604800 seconds), e.g. "3600s".
- threshold string
- The amount of time that must pass with no new traces before a session is considered ready for evaluation. Must be a positive duration no greater than 7 days (604800 seconds), e.g. "3600s".
- threshold String
- The amount of time that must pass with no new traces before a session is considered ready for evaluation. Must be a positive duration no greater than 7 days (604800 seconds), e.g. "3600s".
- threshold string
- The amount of time that must pass with no new traces before a session is considered ready for evaluation. Must be a positive duration no greater than 7 days (604800 seconds), e.g. "3600s".
- threshold str
- The amount of time that must pass with no new traces before a session is considered ready for evaluation. Must be a positive duration no greater than 7 days (604800 seconds), e.g. "3600s".
- threshold String
- The amount of time that must pass with no new traces before a session is considered ready for evaluation. Must be a positive duration no greater than 7 days (604800 seconds), e.g. "3600s".
AiOnlineEvaluatorCloudObservabilityTraceScope, AiOnlineEvaluatorCloudObservabilityTraceScopeArgs
- Filters
List<Ai
Online Evaluator Cloud Observability Trace Scope Filter> - A list of predicates to filter traces. Multiple predicates are combined using AND. The maximum number of predicates is 10. Structure is documented below.
- Filters
[]Ai
Online Evaluator Cloud Observability Trace Scope Filter - A list of predicates to filter traces. Multiple predicates are combined using AND. The maximum number of predicates is 10. Structure is documented below.
- filters list(object)
- A list of predicates to filter traces. Multiple predicates are combined using AND. The maximum number of predicates is 10. Structure is documented below.
- filters
List<Ai
Online Evaluator Cloud Observability Trace Scope Filter> - A list of predicates to filter traces. Multiple predicates are combined using AND. The maximum number of predicates is 10. Structure is documented below.
- filters
Ai
Online Evaluator Cloud Observability Trace Scope Filter[] - A list of predicates to filter traces. Multiple predicates are combined using AND. The maximum number of predicates is 10. Structure is documented below.
- filters
Sequence[Ai
Online Evaluator Cloud Observability Trace Scope Filter] - A list of predicates to filter traces. Multiple predicates are combined using AND. The maximum number of predicates is 10. Structure is documented below.
- filters List<Property Map>
- A list of predicates to filter traces. Multiple predicates are combined using AND. The maximum number of predicates is 10. Structure is documented below.
AiOnlineEvaluatorCloudObservabilityTraceScopeFilter, AiOnlineEvaluatorCloudObservabilityTraceScopeFilterArgs
- Duration
Ai
Online Evaluator Cloud Observability Trace Scope Filter Duration - Filter on the duration of a session (in seconds). Structure is documented below.
- Total
Token AiUsage Online Evaluator Cloud Observability Trace Scope Filter Total Token Usage - Filter on the total token usage within a session. Structure is documented below.
- Duration
Ai
Online Evaluator Cloud Observability Trace Scope Filter Duration - Filter on the duration of a session (in seconds). Structure is documented below.
- Total
Token AiUsage Online Evaluator Cloud Observability Trace Scope Filter Total Token Usage - Filter on the total token usage within a session. Structure is documented below.
- duration object
- Filter on the duration of a session (in seconds). Structure is documented below.
- total_
token_ objectusage - Filter on the total token usage within a session. Structure is documented below.
- duration
Ai
Online Evaluator Cloud Observability Trace Scope Filter Duration - Filter on the duration of a session (in seconds). Structure is documented below.
- total
Token AiUsage Online Evaluator Cloud Observability Trace Scope Filter Total Token Usage - Filter on the total token usage within a session. Structure is documented below.
- duration
Ai
Online Evaluator Cloud Observability Trace Scope Filter Duration - Filter on the duration of a session (in seconds). Structure is documented below.
- total
Token AiUsage Online Evaluator Cloud Observability Trace Scope Filter Total Token Usage - Filter on the total token usage within a session. Structure is documented below.
- duration
Ai
Online Evaluator Cloud Observability Trace Scope Filter Duration - Filter on the duration of a session (in seconds). Structure is documented below.
- total_
token_ Aiusage Online Evaluator Cloud Observability Trace Scope Filter Total Token Usage - Filter on the total token usage within a session. Structure is documented below.
- duration Property Map
- Filter on the duration of a session (in seconds). Structure is documented below.
- total
Token Property MapUsage - Filter on the total token usage within a session. Structure is documented below.
AiOnlineEvaluatorCloudObservabilityTraceScopeFilterDuration, AiOnlineEvaluatorCloudObservabilityTraceScopeFilterDurationArgs
- Comparison
Operator string - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- Value double
- The value to compare against.
- Comparison
Operator string - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- Value float64
- The value to compare against.
- comparison_
operator string - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- value number
- The value to compare against.
- comparison
Operator String - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- value Double
- The value to compare against.
- comparison
Operator string - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- value number
- The value to compare against.
- comparison_
operator str - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- value float
- The value to compare against.
- comparison
Operator String - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- value Number
- The value to compare against.
AiOnlineEvaluatorCloudObservabilityTraceScopeFilterTotalTokenUsage, AiOnlineEvaluatorCloudObservabilityTraceScopeFilterTotalTokenUsageArgs
- Comparison
Operator string - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- Value double
- The value to compare against.
- Comparison
Operator string - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- Value float64
- The value to compare against.
- comparison_
operator string - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- value number
- The value to compare against.
- comparison
Operator String - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- value Double
- The value to compare against.
- comparison
Operator string - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- value number
- The value to compare against.
- comparison_
operator str - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- value float
- The value to compare against.
- comparison
Operator String - The comparison operator to apply. Possible values are: LESS, LESS_OR_EQUAL, EQUAL, NOT_EQUAL, GREATER_OR_EQUAL, GREATER.
- value Number
- The value to compare against.
AiOnlineEvaluatorConfig, AiOnlineEvaluatorConfigArgs
- Max
Evaluated stringSamples Per Run - The maximum number of evaluations to perform per run. If set to 0, the number is unbounded.
- Random
Sampling AiOnline Evaluator Config Random Sampling - Configuration for random sampling. This is the sampling method used to select traces for evaluation. Structure is documented below.
- Max
Evaluated stringSamples Per Run - The maximum number of evaluations to perform per run. If set to 0, the number is unbounded.
- Random
Sampling AiOnline Evaluator Config Random Sampling - Configuration for random sampling. This is the sampling method used to select traces for evaluation. Structure is documented below.
- max_
evaluated_ stringsamples_ per_ run - The maximum number of evaluations to perform per run. If set to 0, the number is unbounded.
- random_
sampling object - Configuration for random sampling. This is the sampling method used to select traces for evaluation. Structure is documented below.
- max
Evaluated StringSamples Per Run - The maximum number of evaluations to perform per run. If set to 0, the number is unbounded.
- random
Sampling AiOnline Evaluator Config Random Sampling - Configuration for random sampling. This is the sampling method used to select traces for evaluation. Structure is documented below.
- max
Evaluated stringSamples Per Run - The maximum number of evaluations to perform per run. If set to 0, the number is unbounded.
- random
Sampling AiOnline Evaluator Config Random Sampling - Configuration for random sampling. This is the sampling method used to select traces for evaluation. Structure is documented below.
- max_
evaluated_ strsamples_ per_ run - The maximum number of evaluations to perform per run. If set to 0, the number is unbounded.
- random_
sampling AiOnline Evaluator Config Random Sampling - Configuration for random sampling. This is the sampling method used to select traces for evaluation. Structure is documented below.
- max
Evaluated StringSamples Per Run - The maximum number of evaluations to perform per run. If set to 0, the number is unbounded.
- random
Sampling Property Map - Configuration for random sampling. This is the sampling method used to select traces for evaluation. Structure is documented below.
AiOnlineEvaluatorConfigRandomSampling, AiOnlineEvaluatorConfigRandomSamplingArgs
- Percentage int
- The percentage of traces to sample for evaluation. Must be an integer between 1 and 100.
- Percentage int
- The percentage of traces to sample for evaluation. Must be an integer between 1 and 100.
- percentage number
- The percentage of traces to sample for evaluation. Must be an integer between 1 and 100.
- percentage Integer
- The percentage of traces to sample for evaluation. Must be an integer between 1 and 100.
- percentage number
- The percentage of traces to sample for evaluation. Must be an integer between 1 and 100.
- percentage int
- The percentage of traces to sample for evaluation. Must be an integer between 1 and 100.
- percentage Number
- The percentage of traces to sample for evaluation. Must be an integer between 1 and 100.
AiOnlineEvaluatorMetricSource, AiOnlineEvaluatorMetricSourceArgs
- Metric string
- Inline metric config, provided as a JSON-formatted string using
camelCase field names to match the API format (see the
Metric API documentation).
The suggested predefined
metricSpecNamevalues depend on the evaluation scope configured incloudObservability; a predefined metric must match the scope or the create request will be rejected: - Metric
Resource stringName - Resource name of a registered metric (a
gcp.vertex.AiEvaluationMetric), e.g. projects/{{project}}/locations/{{region}}/evaluationMetrics/{{id}}.
- Metric string
- Inline metric config, provided as a JSON-formatted string using
camelCase field names to match the API format (see the
Metric API documentation).
The suggested predefined
metricSpecNamevalues depend on the evaluation scope configured incloudObservability; a predefined metric must match the scope or the create request will be rejected: - Metric
Resource stringName - Resource name of a registered metric (a
gcp.vertex.AiEvaluationMetric), e.g. projects/{{project}}/locations/{{region}}/evaluationMetrics/{{id}}.
- metric string
- Inline metric config, provided as a JSON-formatted string using
camelCase field names to match the API format (see the
Metric API documentation).
The suggested predefined
metricSpecNamevalues depend on the evaluation scope configured incloudObservability; a predefined metric must match the scope or the create request will be rejected: - metric_
resource_ stringname - Resource name of a registered metric (a
gcp.vertex.AiEvaluationMetric), e.g. projects/{{project}}/locations/{{region}}/evaluationMetrics/{{id}}.
- metric String
- Inline metric config, provided as a JSON-formatted string using
camelCase field names to match the API format (see the
Metric API documentation).
The suggested predefined
metricSpecNamevalues depend on the evaluation scope configured incloudObservability; a predefined metric must match the scope or the create request will be rejected: - metric
Resource StringName - Resource name of a registered metric (a
gcp.vertex.AiEvaluationMetric), e.g. projects/{{project}}/locations/{{region}}/evaluationMetrics/{{id}}.
- metric string
- Inline metric config, provided as a JSON-formatted string using
camelCase field names to match the API format (see the
Metric API documentation).
The suggested predefined
metricSpecNamevalues depend on the evaluation scope configured incloudObservability; a predefined metric must match the scope or the create request will be rejected: - metric
Resource stringName - Resource name of a registered metric (a
gcp.vertex.AiEvaluationMetric), e.g. projects/{{project}}/locations/{{region}}/evaluationMetrics/{{id}}.
- metric str
- Inline metric config, provided as a JSON-formatted string using
camelCase field names to match the API format (see the
Metric API documentation).
The suggested predefined
metricSpecNamevalues depend on the evaluation scope configured incloudObservability; a predefined metric must match the scope or the create request will be rejected: - metric_
resource_ strname - Resource name of a registered metric (a
gcp.vertex.AiEvaluationMetric), e.g. projects/{{project}}/locations/{{region}}/evaluationMetrics/{{id}}.
- metric String
- Inline metric config, provided as a JSON-formatted string using
camelCase field names to match the API format (see the
Metric API documentation).
The suggested predefined
metricSpecNamevalues depend on the evaluation scope configured incloudObservability; a predefined metric must match the scope or the create request will be rejected: - metric
Resource StringName - Resource name of a registered metric (a
gcp.vertex.AiEvaluationMetric), e.g. projects/{{project}}/locations/{{region}}/evaluationMetrics/{{id}}.
AiOnlineEvaluatorStateDetail, AiOnlineEvaluatorStateDetailArgs
- Message string
- (Output) Human-readable message describing the state of the OnlineEvaluator.
- Message string
- (Output) Human-readable message describing the state of the OnlineEvaluator.
- message string
- (Output) Human-readable message describing the state of the OnlineEvaluator.
- message String
- (Output) Human-readable message describing the state of the OnlineEvaluator.
- message string
- (Output) Human-readable message describing the state of the OnlineEvaluator.
- message str
- (Output) Human-readable message describing the state of the OnlineEvaluator.
- message String
- (Output) Human-readable message describing the state of the OnlineEvaluator.
Import
OnlineEvaluator can be imported using any of these accepted formats:
projects/{{project}}/locations/{{region}}/onlineEvaluators/{{name}}{{project}}/{{region}}/{{name}}{{region}}/{{name}}{{name}}
When using the pulumi import command, OnlineEvaluator can be imported using one of the formats above. For example:
$ pulumi import gcp:vertex/aiOnlineEvaluator:AiOnlineEvaluator default projects/{{project}}/locations/{{region}}/onlineEvaluators/{{name}}
$ pulumi import gcp:vertex/aiOnlineEvaluator:AiOnlineEvaluator default {{project}}/{{region}}/{{name}}
$ pulumi import gcp:vertex/aiOnlineEvaluator:AiOnlineEvaluator default {{region}}/{{name}}
$ pulumi import gcp:vertex/aiOnlineEvaluator:AiOnlineEvaluator default {{name}}
To learn more about importing existing cloud resources, see Importing resources.
Package Details
- Repository
- Google Cloud (GCP) pulumi/pulumi-gcp
- License
- Apache-2.0
- Notes
- This Pulumi package is based on the
google-betaTerraform Provider.
published on Wednesday, Nov 18, 2026 by Pulumi